From 11e2382dfd539ae36c4ff3bbff896d69588a99ff Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:00:04 +0200 Subject: [PATCH 01/65] session check is happening during initialization not before register of each plugin --- pype/ftrack/actions/action_attributes_remapper.py | 3 --- pype/ftrack/actions/action_client_review_sort.py | 5 +---- pype/ftrack/actions/action_component_open.py | 6 ------ pype/ftrack/actions/action_create_cust_attrs.py | 6 ------ pype/ftrack/actions/action_create_folders.py | 3 --- .../actions/action_create_project_structure.py | 3 --- pype/ftrack/actions/action_cust_attr_doctor.py | 5 +---- pype/ftrack/actions/action_delete_asset.py | 8 +------- pype/ftrack/actions/action_delete_asset_byname.py | 6 ------ pype/ftrack/actions/action_djvview.py | 2 -- pype/ftrack/actions/action_job_killer.py | 6 ------ pype/ftrack/actions/action_multiple_notes.py | 3 --- pype/ftrack/actions/action_prepare_project.py | 3 --- pype/ftrack/actions/action_rv.py | 2 -- pype/ftrack/actions/action_start_timer.py | 5 +---- .../actions/action_sync_hier_attrs_local.py | 3 --- .../ftrack/actions/action_sync_to_avalon_local.py | 7 ------- pype/ftrack/actions/action_test.py | 3 --- .../actions/action_thumbnail_to_childern.py | 2 -- pype/ftrack/actions/action_thumbnail_to_parent.py | 2 -- pype/ftrack/actions/action_where_run_ask.py | 3 --- pype/ftrack/actions/action_where_run_show.py | 3 --- pype/ftrack/events/action_sync_hier_attrs.py | 5 +---- pype/ftrack/events/action_sync_to_avalon.py | 3 --- .../ftrack/events/event_del_avalon_id_from_new.py | 2 -- pype/ftrack/events/event_next_task_update.py | 2 -- pype/ftrack/events/event_radio_buttons.py | 2 -- pype/ftrack/events/event_sync_hier_attr.py | 2 -- pype/ftrack/events/event_sync_to_avalon.py | 4 ---- pype/ftrack/events/event_test.py | 4 +--- pype/ftrack/events/event_thumbnail_updates.py | 2 -- pype/ftrack/events/event_user_assigment.py | 2 -- .../events/event_version_to_task_statuses.py | 2 -- pype/ftrack/lib/ftrack_base_handler.py | 15 ++++++++++++++- 34 files changed, 20 insertions(+), 114 deletions(-) diff --git a/pype/ftrack/actions/action_attributes_remapper.py b/pype/ftrack/actions/action_attributes_remapper.py index 759b5765e5..a0393ece40 100644 --- a/pype/ftrack/actions/action_attributes_remapper.py +++ b/pype/ftrack/actions/action_attributes_remapper.py @@ -281,7 +281,4 @@ class AttributesRemapper(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - AttributesRemapper(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_client_review_sort.py b/pype/ftrack/actions/action_client_review_sort.py index 6a659ce5e3..91926c2874 100644 --- a/pype/ftrack/actions/action_client_review_sort.py +++ b/pype/ftrack/actions/action_client_review_sort.py @@ -55,11 +55,8 @@ class ClientReviewSort(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - action_handler = ClientReviewSort(session, plugins_presets) - action_handler.register() + ClientReviewSort(session, plugins_presets).register() def main(arguments=None): diff --git a/pype/ftrack/actions/action_component_open.py b/pype/ftrack/actions/action_component_open.py index 33f4d38890..98d773dba6 100644 --- a/pype/ftrack/actions/action_component_open.py +++ b/pype/ftrack/actions/action_component_open.py @@ -68,12 +68,6 @@ class ComponentOpen(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - ComponentOpen(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_cust_attrs.py b/pype/ftrack/actions/action_create_cust_attrs.py index 47a6bb5d5f..ac6dcb0fd7 100644 --- a/pype/ftrack/actions/action_create_cust_attrs.py +++ b/pype/ftrack/actions/action_create_cust_attrs.py @@ -572,12 +572,6 @@ class CustomAttributes(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - CustomAttributes(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_folders.py b/pype/ftrack/actions/action_create_folders.py index 269316e052..44e9741bab 100644 --- a/pype/ftrack/actions/action_create_folders.py +++ b/pype/ftrack/actions/action_create_folders.py @@ -327,9 +327,6 @@ class PartialDict(dict): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CreateFolders(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_project_structure.py b/pype/ftrack/actions/action_create_project_structure.py index 74d458b5f8..c39c717b11 100644 --- a/pype/ftrack/actions/action_create_project_structure.py +++ b/pype/ftrack/actions/action_create_project_structure.py @@ -198,9 +198,6 @@ class CreateProjectFolders(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CreateProjectFolders(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_cust_attr_doctor.py b/pype/ftrack/actions/action_cust_attr_doctor.py index af5fe2dc4a..0469b3a1e6 100644 --- a/pype/ftrack/actions/action_cust_attr_doctor.py +++ b/pype/ftrack/actions/action_cust_attr_doctor.py @@ -9,7 +9,7 @@ from pype.ftrack import BaseAction class CustomAttributeDoctor(BaseAction): - + ignore_me = True #: Action identifier. identifier = 'custom.attributes.doctor' @@ -294,9 +294,6 @@ class CustomAttributeDoctor(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CustomAttributeDoctor(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_delete_asset.py b/pype/ftrack/actions/action_delete_asset.py index 1b1e7fc905..106c81758a 100644 --- a/pype/ftrack/actions/action_delete_asset.py +++ b/pype/ftrack/actions/action_delete_asset.py @@ -85,7 +85,7 @@ class DeleteAsset(BaseAction): 'type': 'asset', 'name': entity['name'] }) - + if av_entity is None: return { 'success': False, @@ -314,12 +314,6 @@ class DeleteAsset(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - DeleteAsset(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_delete_asset_byname.py b/pype/ftrack/actions/action_delete_asset_byname.py index 2431b2311e..4a3807f8f0 100644 --- a/pype/ftrack/actions/action_delete_asset_byname.py +++ b/pype/ftrack/actions/action_delete_asset_byname.py @@ -135,12 +135,6 @@ class AssetsRemover(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - AssetsRemover(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_djvview.py b/pype/ftrack/actions/action_djvview.py index 58914fbc1e..9da12dd67c 100644 --- a/pype/ftrack/actions/action_djvview.py +++ b/pype/ftrack/actions/action_djvview.py @@ -220,8 +220,6 @@ class DJVViewAction(BaseAction): def register(session, plugins_presets={}): """Register hooks.""" - if not isinstance(session, ftrack_api.session.Session): - return DJVViewAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_job_killer.py b/pype/ftrack/actions/action_job_killer.py index 717f87e879..64fb99133d 100644 --- a/pype/ftrack/actions/action_job_killer.py +++ b/pype/ftrack/actions/action_job_killer.py @@ -121,12 +121,6 @@ class JobKiller(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - JobKiller(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_multiple_notes.py b/pype/ftrack/actions/action_multiple_notes.py index 6e28b7bed6..bd51cb2984 100644 --- a/pype/ftrack/actions/action_multiple_notes.py +++ b/pype/ftrack/actions/action_multiple_notes.py @@ -115,9 +115,6 @@ class MultipleNotes(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - MultipleNotes(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_prepare_project.py b/pype/ftrack/actions/action_prepare_project.py index e914fa74f0..d645748f6f 100644 --- a/pype/ftrack/actions/action_prepare_project.py +++ b/pype/ftrack/actions/action_prepare_project.py @@ -372,7 +372,4 @@ class PrepareProject(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - PrepareProject(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_rv.py b/pype/ftrack/actions/action_rv.py index 6b6591355f..69c6624b71 100644 --- a/pype/ftrack/actions/action_rv.py +++ b/pype/ftrack/actions/action_rv.py @@ -328,8 +328,6 @@ class RVAction(BaseAction): def register(session, plugins_presets={}): """Register hooks.""" - if not isinstance(session, ftrack_api.session.Session): - return RVAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_start_timer.py b/pype/ftrack/actions/action_start_timer.py index 36752a1edc..292789e9f3 100644 --- a/pype/ftrack/actions/action_start_timer.py +++ b/pype/ftrack/actions/action_start_timer.py @@ -26,7 +26,7 @@ class StartTimer(BaseAction): user.start_timer(entity, force=True) self.session.commit() - + self.log.info( "Starting Ftrack timer for task: {}".format(entity['name']) ) @@ -37,7 +37,4 @@ class StartTimer(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - StartTimer(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_sync_hier_attrs_local.py b/pype/ftrack/actions/action_sync_hier_attrs_local.py index 05a70461a1..289abd0122 100644 --- a/pype/ftrack/actions/action_sync_hier_attrs_local.py +++ b/pype/ftrack/actions/action_sync_hier_attrs_local.py @@ -309,9 +309,6 @@ class SyncHierarchicalAttrs(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_sync_to_avalon_local.py b/pype/ftrack/actions/action_sync_to_avalon_local.py index ddf8ed6571..61050f9883 100644 --- a/pype/ftrack/actions/action_sync_to_avalon_local.py +++ b/pype/ftrack/actions/action_sync_to_avalon_local.py @@ -263,11 +263,4 @@ class SyncToAvalon(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - SyncToAvalon(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_test.py b/pype/ftrack/actions/action_test.py index a2bc8bf892..58f7210e3b 100644 --- a/pype/ftrack/actions/action_test.py +++ b/pype/ftrack/actions/action_test.py @@ -43,9 +43,6 @@ class TestAction(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - TestAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_thumbnail_to_childern.py b/pype/ftrack/actions/action_thumbnail_to_childern.py index 101b678512..7d189cf652 100644 --- a/pype/ftrack/actions/action_thumbnail_to_childern.py +++ b/pype/ftrack/actions/action_thumbnail_to_childern.py @@ -68,8 +68,6 @@ class ThumbToChildren(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbToChildren(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_thumbnail_to_parent.py b/pype/ftrack/actions/action_thumbnail_to_parent.py index c382d9303c..efafca4a96 100644 --- a/pype/ftrack/actions/action_thumbnail_to_parent.py +++ b/pype/ftrack/actions/action_thumbnail_to_parent.py @@ -90,8 +90,6 @@ class ThumbToParent(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbToParent(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_where_run_ask.py b/pype/ftrack/actions/action_where_run_ask.py index 0351c09909..795c2664cc 100644 --- a/pype/ftrack/actions/action_where_run_ask.py +++ b/pype/ftrack/actions/action_where_run_ask.py @@ -40,7 +40,4 @@ class ActionAskWhereIRun(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - ActionAskWhereIRun(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_where_run_show.py b/pype/ftrack/actions/action_where_run_show.py index 7fea23e3b7..48618f0251 100644 --- a/pype/ftrack/actions/action_where_run_show.py +++ b/pype/ftrack/actions/action_where_run_show.py @@ -80,7 +80,4 @@ class ActionShowWhereIRun(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - ActionShowWhereIRun(session, plugins_presets).register() diff --git a/pype/ftrack/events/action_sync_hier_attrs.py b/pype/ftrack/events/action_sync_hier_attrs.py index c9d968ee5d..23ac319261 100644 --- a/pype/ftrack/events/action_sync_hier_attrs.py +++ b/pype/ftrack/events/action_sync_hier_attrs.py @@ -220,7 +220,7 @@ class SyncHierarchicalAttrs(BaseAction): if job['status'] in ('queued', 'running'): job['status'] = 'failed' session.commit() - + if self.interface_messages: self.show_interface_from_dict( messages=self.interface_messages, @@ -341,9 +341,6 @@ class SyncHierarchicalAttrs(BaseAction): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/events/action_sync_to_avalon.py b/pype/ftrack/events/action_sync_to_avalon.py index 51a4ae9475..7b5f94f216 100644 --- a/pype/ftrack/events/action_sync_to_avalon.py +++ b/pype/ftrack/events/action_sync_to_avalon.py @@ -296,9 +296,6 @@ def register(session, plugins_presets): # Validate that session is an instance of ftrack_api.Session. If not, # assume that register is being called from an old or incompatible API and # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - SyncToAvalon(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_del_avalon_id_from_new.py b/pype/ftrack/events/event_del_avalon_id_from_new.py index 6f6320f51b..e5c4b1be45 100644 --- a/pype/ftrack/events/event_del_avalon_id_from_new.py +++ b/pype/ftrack/events/event_del_avalon_id_from_new.py @@ -53,7 +53,5 @@ class DelAvalonIdFromNew(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return DelAvalonIdFromNew(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_next_task_update.py b/pype/ftrack/events/event_next_task_update.py index 18a7abf328..51ccb2f057 100644 --- a/pype/ftrack/events/event_next_task_update.py +++ b/pype/ftrack/events/event_next_task_update.py @@ -88,7 +88,5 @@ class NextTaskUpdate(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return NextTaskUpdate(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_radio_buttons.py b/pype/ftrack/events/event_radio_buttons.py index 9c6f2d490a..917c7a49e6 100644 --- a/pype/ftrack/events/event_radio_buttons.py +++ b/pype/ftrack/events/event_radio_buttons.py @@ -36,7 +36,5 @@ class Radio_buttons(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return Radio_buttons(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_sync_hier_attr.py b/pype/ftrack/events/event_sync_hier_attr.py index 5ddc2394af..8031ec9e55 100644 --- a/pype/ftrack/events/event_sync_hier_attr.py +++ b/pype/ftrack/events/event_sync_hier_attr.py @@ -209,7 +209,5 @@ class SyncHierarchicalAttrs(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_sync_to_avalon.py b/pype/ftrack/events/event_sync_to_avalon.py index 4e7b208726..a25866be65 100644 --- a/pype/ftrack/events/event_sync_to_avalon.py +++ b/pype/ftrack/events/event_sync_to_avalon.py @@ -122,8 +122,4 @@ class Sync_to_Avalon(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - - if not isinstance(session, ftrack_api.session.Session): - return - Sync_to_Avalon(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_test.py b/pype/ftrack/events/event_test.py index 94d99dbf67..a909aa5510 100644 --- a/pype/ftrack/events/event_test.py +++ b/pype/ftrack/events/event_test.py @@ -8,7 +8,7 @@ from pype.ftrack import BaseEvent class Test_Event(BaseEvent): ignore_me = True - + priority = 10000 def launch(self, session, event): @@ -22,7 +22,5 @@ class Test_Event(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return Test_Event(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_thumbnail_updates.py b/pype/ftrack/events/event_thumbnail_updates.py index 51bb15a4c7..ae6f8adb5e 100644 --- a/pype/ftrack/events/event_thumbnail_updates.py +++ b/pype/ftrack/events/event_thumbnail_updates.py @@ -47,7 +47,5 @@ class ThumbnailEvents(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbnailEvents(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_user_assigment.py b/pype/ftrack/events/event_user_assigment.py index fe8b331629..61699d736c 100644 --- a/pype/ftrack/events/event_user_assigment.py +++ b/pype/ftrack/events/event_user_assigment.py @@ -233,7 +233,5 @@ def register(session, plugins_presets): """ Register plugin. Called when used as an plugin. """ - if not isinstance(session, ftrack_api.session.Session): - return UserAssigmentEvent(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_version_to_task_statuses.py b/pype/ftrack/events/event_version_to_task_statuses.py index 85a31383d5..66a55c0cf7 100644 --- a/pype/ftrack/events/event_version_to_task_statuses.py +++ b/pype/ftrack/events/event_version_to_task_statuses.py @@ -71,7 +71,5 @@ class VersionToTaskStatus(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return VersionToTaskStatus(session, plugins_presets).register() diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 9eda74f0f3..9821bfd81f 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -31,8 +31,21 @@ class BaseHandler(object): def __init__(self, session, plugins_presets={}): '''Expects a ftrack_api.Session instance''' - self._session = session self.log = Logger().get_logger(self.__class__.__name__) + if not isinstance(session, ftrack_api.session.Session): + self.log.warning(( + "Session object entered with args is instance of \"{}\"" + " but expected instance is \"{}\"." + ).format( + str(type(session)), + str(ftrack_api.session.Session.__qualname__) + )) + self.register = self.register_without_session + self._session = None + + return + + self._session = session # Using decorator self.register = self.register_decorator(self.register) From 43f63fd260768b03ceac194d40083e4b188f3043 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:01:43 +0200 Subject: [PATCH 02/65] print traceback when error during plugin registration happens --- pype/ftrack/ftrack_server/ftrack_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/ftrack_server.py b/pype/ftrack/ftrack_server/ftrack_server.py index e1c13cda32..612e89e60f 100644 --- a/pype/ftrack/ftrack_server/ftrack_server.py +++ b/pype/ftrack/ftrack_server/ftrack_server.py @@ -126,7 +126,7 @@ class FtrackServer: msg = '"{}" - register was not successful ({})'.format( function_dict['name'], str(exc) ) - log.warning(msg) + log.warning(msg, exc_info=True) def run_server(self): self.session = ftrack_api.Session(auto_connect_event_hub=True,) From f0e5f39342fadcfdc626e06141cfdeba1af47d2e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:02:22 +0200 Subject: [PATCH 03/65] run server in ftrack server gives ability to use other session and not to load plugins --- pype/ftrack/ftrack_server/ftrack_server.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pype/ftrack/ftrack_server/ftrack_server.py b/pype/ftrack/ftrack_server/ftrack_server.py index 612e89e60f..12b046c510 100644 --- a/pype/ftrack/ftrack_server/ftrack_server.py +++ b/pype/ftrack/ftrack_server/ftrack_server.py @@ -128,21 +128,25 @@ class FtrackServer: ) log.warning(msg, exc_info=True) - def run_server(self): - self.session = ftrack_api.Session(auto_connect_event_hub=True,) + def run_server(self, session=None, load_files=True): + if not session: + session = ftrack_api.Session(auto_connect_event_hub=True) - paths_str = os.environ.get(self.env_key) - if paths_str is None: - log.error(( - "Env var \"{}\" is not set, \"{}\" server won\'t launch" - ).format(self.env_key, self.server_type)) - return + self.session = session - paths = paths_str.split(os.pathsep) - self.set_files(paths) + if load_files: + paths_str = os.environ.get(self.env_key) + if paths_str is None: + log.error(( + "Env var \"{}\" is not set, \"{}\" server won\'t launch" + ).format(self.env_key, self.server_type)) + return - log.info(60*"*") - log.info('Registration of actions/events has finished!') + paths = paths_str.split(os.pathsep) + self.set_files(paths) + + log.info(60*"*") + log.info('Registration of actions/events has finished!') # keep event_hub on session running self.session.event_hub.wait() From 8b8427ca871a5a36e462d929d76a7521d206853c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:11:12 +0200 Subject: [PATCH 04/65] added scripts to be able to be run as subprocess, one for storing, second for processing events --- .../sub_event_processor.py | 46 +++++++++ .../parallel_event_server/sub_event_storer.py | 97 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py new file mode 100644 index 0000000000..bd3118a893 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -0,0 +1,46 @@ +import os +import sys +import datetime +import signal +import socket +import pymongo + +from pype.ftrack.ftrack_server import FtrackServer +from session_processor import ProcessSession +from pypeapp import Logger + +log = Logger().get_logger("Event storer") + + +def main(args): + port = int(args[-1]) + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Connect the socket to the port where the server is listening + server_address = ("localhost", port) + log.debug("Processor connected to {} port {}".format(*server_address)) + sock.connect(server_address) + + sock.sendall(b"CreatedProcess") + try: + session = ProcessSession(auto_connect_event_hub=True, sock=sock) + server = FtrackServer('event') + log.debug("Launched Ftrack Event processor") + server.run_server(session) + + finally: + log.debug("First closing socket") + sock.close() + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + main(sys.argv) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py new file mode 100644 index 0000000000..295504f389 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -0,0 +1,97 @@ +import os +import sys +import datetime +import signal +import socket +import pymongo + +from pype.ftrack.ftrack_server import FtrackServer +from pype.ftrack.lib.custom_db_connector import DbConnector +from pype.ftrack.ftrack_server.parallel_event_server.session_storer import StorerSession +from pypeapp import Logger + +log = Logger().get_logger("Event storer") + + +dbcon = DbConnector( + mongo_url=os.environ["AVALON_MONGO"], + database_name="pype" +) +table_name = "ftrack_events" + +# ignore_topics = ["ftrack.meta.connected"] +ignore_topics = [] + +def install_db(): + dbcon.install() + dbcon.create_table(table_name, capped=False) + dbcon.active_table = table_name + +def launch(event): + if event.get("topic") in ignore_topics: + return + + event_data = event._data + event_id = event["id"] + + event_data["pype_data"] = { + "stored": datetime.datetime.utcnow(), + "is_processed": False + } + + try: + # dbcon.insert_one(event_data) + dbcon.update({"id": event_id}, event_data, upsert=True) + log.debug("Event: {} stored".format(event_id)) + + except pymongo.errors.DuplicateKeyError: + log.debug("Event: {} already exists".format(event_id)) + + except Exception as exc: + log.error( + "Event: {} failed to store".format(event_id), + exc_info=True + ) + + +def register(session): + '''Registers the event, subscribing the discover and launch topics.''' + install_db() + session.event_hub.subscribe("topic=*", launch) + + +def main(args): + port = int(args[-1]) + + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Connect the socket to the port where the server is listening + server_address = ("localhost", port) + log.debug("Storer connected to {} port {}".format(*server_address)) + sock.connect(server_address) + sock.sendall(b"CreatedStore") + + try: + session = StorerSession(auto_connect_event_hub=True, sock=sock) + register(session) + server = FtrackServer("event") + log.info(os.environ["FTRACK_EVENTS_PATH"]) + log.debug("Launched Ftrack Event storer") + server.run_server(session, load_files=False) + + finally: + log.debug("First closing socket") + sock.close() + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + main(sys.argv) From 1ee8ab0dabeec7b4cfb2098e051bbd2bd583e050 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:11:46 +0200 Subject: [PATCH 05/65] added socket thread which is able to start subprocess with connection to specific port with sockets --- .../parallel_event_server/socket_thread.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py new file mode 100644 index 0000000000..deede2ed03 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -0,0 +1,114 @@ +import os +import time +import signal +import socket +import threading +import subprocess +from pypeapp import Logger + + +class SocketThread(threading.Thread): + MAX_TIMEOUT = 30 + def __init__(self, name, port, filepath): + super(SocketThread, self).__init__() + self.log = Logger().get_logger("SocketThread", "Event Thread") + self.setName(name) + self.name = name + self.port = port + self.filepath = filepath + self.sock = None + self.subproc = None + self.connection = None + self._is_running = False + self.finished = False + + def stop(self): + self._is_running = False + super().stop() + + def process_to_die(self): + if not self.connection: + self.stop() + return + + self.connection.sendall(b"ptd") + + def run(self): + self._is_running = True + time_socket = time.time() + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = sock + + # Bind the socket to the port + server_address = ("localhost", self.port) + sock.bind(server_address) + self.log.debug( + "Running Socked thread on {}:{}".format(*server_address) + ) + + self.subproc = subprocess.Popen( + ["python", self.filepath, "-port", str(self.port)], + stdout=subprocess.PIPE + ) + + # Listen for incoming connections + sock.listen(1) + sock.settimeout(1.0) + while True: + if not self._is_running: + break + try: + connection, client_address = sock.accept() + time_socket = time.time() + connection.settimeout(1.0) + self.connection = connection + + except socket.timeout: + if (time.time() - time_socket) > self.MAX_TIMEOUT: + self.log.error("Connection timeout passed. Terminating.") + self._is_running = False + os.kill(self.subproc.pid, signal.SIGINT) + break + continue + + try: + time_con = time.time() + # Receive the data in small chunks and retransmit it + while True: + try: + if not self._is_running: + break + try: + data = connection.recv(16) + time_con = time.time() + + except socket.timeout: + if (time.time() - time_con) > self.MAX_TIMEOUT: + self.log.error( + "Connection timeout passed. Terminating." + ) + self._is_running = False + os.kill(self.subproc.pid, signal.SIGINT) + break + continue + + except ConnectionResetError: + self._is_running = False + break + + if data: + connection.sendall(data) + + except Exception as exc: + self.log.error( + "Event server process failed", exc_info=True + ) + + finally: + # Clean up the connection + connection.close() + if self.subproc.poll() is None: + os.kill(self.subproc.pid, signal.SIGINT) + + self.finished = True From 631dcb56ea4959fd1f592feb6377e284849d612e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:12:15 +0200 Subject: [PATCH 06/65] added modified ftrack sessions, one for storing, second for processing events --- .../session_processor.py | 259 ++++++++++++++++++ .../parallel_event_server/session_storer.py | 257 +++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/session_processor.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/session_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py new file mode 100644 index 0000000000..55bdf0fba9 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py @@ -0,0 +1,259 @@ +import logging +import os +import atexit +import tempfile +import threading +import time +import requests +import queue + +import ftrack_api +import ftrack_api.session +import ftrack_api.cache +import ftrack_api.operation +import ftrack_api._centralized_storage_scenario +import ftrack_api.event +from ftrack_api.logging import LazyLogMessage as L + +from pype.ftrack.lib.custom_db_connector import DbConnector + + +class ProcessEventHub(ftrack_api.event.hub.EventHub): + dbcon = DbConnector( + mongo_url=os.environ["AVALON_MONGO"], + database_name="pype" + ) + table_name = "ftrack_events" + is_table_created = False + + def __init__(self, *args, **kwargs): + self.sock = kwargs.pop("sock") + super(ProcessEventHub, self).__init__(*args, **kwargs) + + def prepare_dbcon(self): + self.dbcon.install() + if not self.is_table_created: + self.dbcon.create_table(self.table_name, capped=False) + self.dbcon.active_table = self.table_name + self.is_table_created = True + + def wait(self, duration=None): + '''Wait for events and handle as they arrive. + + If *duration* is specified, then only process events until duration is + reached. *duration* is in seconds though float values can be used for + smaller values. + + ''' + started = time.time() + + self.prepare_dbcon() + while True: + try: + event = self._event_queue.get(timeout=0.1) + except queue.Empty: + if not self.load_events(): + time.sleep(0.5) + else: + self._handle(event) + self.dbcon.update_one( + {"id": event["id"]}, + {"$set": {"pype_data.is_processed": True}} + ) + # Additional special processing of events. + if event['topic'] == 'ftrack.meta.disconnected': + break + + if duration is not None: + if (time.time() - started) > duration: + break + + def load_events(self): + not_processed_events = self.dbcon.find({"pype_data.is_processed": False}) + found = False + for event_data in not_processed_events: + new_event_data = { + k: v for k, v in event_data.items() + if k not in ["_id", "pype_data"] + } + try: + event = ftrack_api.event.base.Event(**new_event_data) + print(event) + except Exception: + self.logger.exception(L( + 'Failed to convert payload into event: {0}', + event_data + )) + continue + found = True + self._event_queue.put(event) + + return found + + def _handle_packet(self, code, packet_identifier, path, data): + '''Handle packet received from server.''' + # if self.is_waiting: + code_name = self._code_name_mapping[code] + if code_name == "event": + return + if code_name == "heartbeat": + self.sock.sendall(b"processor") + return self._send_packet(self._code_name_mapping["heartbeat"]) + + return super()._handle_packet(code, packet_identifier, path, data) + + +class ProcessSession(ftrack_api.session.Session): + '''An isolated session for interaction with an ftrack server.''' + def __init__( + self, server_url=None, api_key=None, api_user=None, auto_populate=True, + plugin_paths=None, cache=None, cache_key_maker=None, + auto_connect_event_hub=None, schema_cache_path=None, + plugin_arguments=None, sock=None + ): + super(ftrack_api.session.Session, self).__init__() + self.logger = logging.getLogger( + __name__ + '.' + self.__class__.__name__ + ) + self._closed = False + + if server_url is None: + server_url = os.environ.get('FTRACK_SERVER') + + if not server_url: + raise TypeError( + 'Required "server_url" not specified. Pass as argument or set ' + 'in environment variable FTRACK_SERVER.' + ) + + self._server_url = server_url + + if api_key is None: + api_key = os.environ.get( + 'FTRACK_API_KEY', + # Backwards compatibility + os.environ.get('FTRACK_APIKEY') + ) + + if not api_key: + raise TypeError( + 'Required "api_key" not specified. Pass as argument or set in ' + 'environment variable FTRACK_API_KEY.' + ) + + self._api_key = api_key + + if api_user is None: + api_user = os.environ.get('FTRACK_API_USER') + if not api_user: + try: + api_user = getpass.getuser() + except Exception: + pass + + if not api_user: + raise TypeError( + 'Required "api_user" not specified. Pass as argument, set in ' + 'environment variable FTRACK_API_USER or one of the standard ' + 'environment variables used by Python\'s getpass module.' + ) + + self._api_user = api_user + + # Currently pending operations. + self.recorded_operations = ftrack_api.operation.Operations() + self.record_operations = True + + self.cache_key_maker = cache_key_maker + if self.cache_key_maker is None: + self.cache_key_maker = ftrack_api.cache.StringKeyMaker() + + # Enforce always having a memory cache at top level so that the same + # in-memory instance is returned from session. + self.cache = ftrack_api.cache.LayeredCache([ + ftrack_api.cache.MemoryCache() + ]) + + if cache is not None: + if callable(cache): + cache = cache(self) + + if cache is not None: + self.cache.caches.append(cache) + + self._managed_request = None + self._request = requests.Session() + self._request.auth = ftrack_api.session.SessionAuthentication( + self._api_key, self._api_user + ) + + self.auto_populate = auto_populate + + # Fetch server information and in doing so also check credentials. + self._server_information = self._fetch_server_information() + + # Now check compatibility of server based on retrieved information. + self.check_server_compatibility() + + # Construct event hub and load plugins. + self._event_hub = ProcessEventHub( + self._server_url, + self._api_user, + self._api_key, + sock=sock + ) + + self._auto_connect_event_hub_thread = None + if auto_connect_event_hub in (None, True): + # Connect to event hub in background thread so as not to block main + # session usage waiting for event hub connection. + self._auto_connect_event_hub_thread = threading.Thread( + target=self._event_hub.connect + ) + self._auto_connect_event_hub_thread.daemon = True + self._auto_connect_event_hub_thread.start() + + # To help with migration from auto_connect_event_hub default changing + # from True to False. + self._event_hub._deprecation_warning_auto_connect = ( + auto_connect_event_hub is None + ) + + # Register to auto-close session on exit. + atexit.register(self.close) + + self._plugin_paths = plugin_paths + if self._plugin_paths is None: + self._plugin_paths = os.environ.get( + 'FTRACK_EVENT_PLUGIN_PATH', '' + ).split(os.pathsep) + + self._discover_plugins(plugin_arguments=plugin_arguments) + + # TODO: Make schemas read-only and non-mutable (or at least without + # rebuilding types)? + if schema_cache_path is not False: + if schema_cache_path is None: + schema_cache_path = os.environ.get( + 'FTRACK_API_SCHEMA_CACHE_PATH', tempfile.gettempdir() + ) + + schema_cache_path = os.path.join( + schema_cache_path, 'ftrack_api_schema_cache.json' + ) + + self.schemas = self._load_schemas(schema_cache_path) + self.types = self._build_entity_type_classes(self.schemas) + + ftrack_api._centralized_storage_scenario.register(self) + + self._configure_locations() + self.event_hub.publish( + ftrack_api.event.base.Event( + topic='ftrack.api.session.ready', + data=dict( + session=self + ) + ), + synchronous=True + ) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py new file mode 100644 index 0000000000..01b1228d55 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py @@ -0,0 +1,257 @@ +import logging +import os +import atexit +import tempfile +import threading +import requests + +import ftrack_api +import ftrack_api.session +import ftrack_api.cache +import ftrack_api.operation +import ftrack_api._centralized_storage_scenario +import ftrack_api.event +from ftrack_api.logging import LazyLogMessage as L + + +class StorerEventHub(ftrack_api.event.hub.EventHub): + def __init__(self, *args, **kwargs): + self.sock = kwargs.pop("sock") + super(StorerEventHub, self).__init__(*args, **kwargs) + + def _handle_packet(self, code, packet_identifier, path, data): + '''Handle packet received from server.''' + if self._code_name_mapping[code] == "heartbeat": + # Reply with heartbeat. + self.sock.sendall(b"storer") + return self._send_packet(self._code_name_mapping['heartbeat']) + + return super(StorerEventHub, self)._handle_packet( + code, packet_identifier, path, data + ) + + +class StorerSession(ftrack_api.session.Session): + '''An isolated session for interaction with an ftrack server.''' + def __init__( + self, server_url=None, api_key=None, api_user=None, auto_populate=True, + plugin_paths=None, cache=None, cache_key_maker=None, + auto_connect_event_hub=None, schema_cache_path=None, + plugin_arguments=None, sock=None + ): + '''Initialise session. + + *server_url* should be the URL of the ftrack server to connect to + including any port number. If not specified attempt to look up from + :envvar:`FTRACK_SERVER`. + + *api_key* should be the API key to use for authentication whilst + *api_user* should be the username of the user in ftrack to record + operations against. If not specified, *api_key* should be retrieved + from :envvar:`FTRACK_API_KEY` and *api_user* from + :envvar:`FTRACK_API_USER`. + + If *auto_populate* is True (the default), then accessing entity + attributes will cause them to be automatically fetched from the server + if they are not already. This flag can be changed on the session + directly at any time. + + *plugin_paths* should be a list of paths to search for plugins. If not + specified, default to looking up :envvar:`FTRACK_EVENT_PLUGIN_PATH`. + + *cache* should be an instance of a cache that fulfils the + :class:`ftrack_api.cache.Cache` interface and will be used as the cache + for the session. It can also be a callable that will be called with the + session instance as sole argument. The callable should return ``None`` + if a suitable cache could not be configured, but session instantiation + can continue safely. + + .. note:: + + The session will add the specified cache to a pre-configured layered + cache that specifies the top level cache as a + :class:`ftrack_api.cache.MemoryCache`. Therefore, it is unnecessary + to construct a separate memory cache for typical behaviour. Working + around this behaviour or removing the memory cache can lead to + unexpected behaviour. + + *cache_key_maker* should be an instance of a key maker that fulfils the + :class:`ftrack_api.cache.KeyMaker` interface and will be used to + generate keys for objects being stored in the *cache*. If not specified, + a :class:`~ftrack_api.cache.StringKeyMaker` will be used. + + If *auto_connect_event_hub* is True then embedded event hub will be + automatically connected to the event server and allow for publishing and + subscribing to **non-local** events. If False, then only publishing and + subscribing to **local** events will be possible until the hub is + manually connected using :meth:`EventHub.connect + `. + + .. note:: + + The event hub connection is performed in a background thread to + improve session startup time. If a registered plugin requires a + connected event hub then it should check the event hub connection + status explicitly. Subscribing to events does *not* require a + connected event hub. + + Enable schema caching by setting *schema_cache_path* to a folder path. + If not set, :envvar:`FTRACK_API_SCHEMA_CACHE_PATH` will be used to + determine the path to store cache in. If the environment variable is + also not specified then a temporary directory will be used. Set to + `False` to disable schema caching entirely. + + *plugin_arguments* should be an optional mapping (dict) of keyword + arguments to pass to plugin register functions upon discovery. If a + discovered plugin has a signature that is incompatible with the passed + arguments, the discovery mechanism will attempt to reduce the passed + arguments to only those that the plugin accepts. Note that a warning + will be logged in this case. + + ''' + super(ftrack_api.session.Session, self).__init__() + self.logger = logging.getLogger( + __name__ + '.' + self.__class__.__name__ + ) + self._closed = False + + if server_url is None: + server_url = os.environ.get('FTRACK_SERVER') + + if not server_url: + raise TypeError( + 'Required "server_url" not specified. Pass as argument or set ' + 'in environment variable FTRACK_SERVER.' + ) + + self._server_url = server_url + + if api_key is None: + api_key = os.environ.get( + 'FTRACK_API_KEY', + # Backwards compatibility + os.environ.get('FTRACK_APIKEY') + ) + + if not api_key: + raise TypeError( + 'Required "api_key" not specified. Pass as argument or set in ' + 'environment variable FTRACK_API_KEY.' + ) + + self._api_key = api_key + + if api_user is None: + api_user = os.environ.get('FTRACK_API_USER') + if not api_user: + try: + api_user = getpass.getuser() + except Exception: + pass + + if not api_user: + raise TypeError( + 'Required "api_user" not specified. Pass as argument, set in ' + 'environment variable FTRACK_API_USER or one of the standard ' + 'environment variables used by Python\'s getpass module.' + ) + + self._api_user = api_user + + # Currently pending operations. + self.recorded_operations = ftrack_api.operation.Operations() + self.record_operations = True + + self.cache_key_maker = cache_key_maker + if self.cache_key_maker is None: + self.cache_key_maker = ftrack_api.cache.StringKeyMaker() + + # Enforce always having a memory cache at top level so that the same + # in-memory instance is returned from session. + self.cache = ftrack_api.cache.LayeredCache([ + ftrack_api.cache.MemoryCache() + ]) + + if cache is not None: + if callable(cache): + cache = cache(self) + + if cache is not None: + self.cache.caches.append(cache) + + self._managed_request = None + self._request = requests.Session() + self._request.auth = ftrack_api.session.SessionAuthentication( + self._api_key, self._api_user + ) + + self.auto_populate = auto_populate + + # Fetch server information and in doing so also check credentials. + self._server_information = self._fetch_server_information() + + # Now check compatibility of server based on retrieved information. + self.check_server_compatibility() + + # Construct event hub and load plugins. + self._event_hub = StorerEventHub( + self._server_url, + self._api_user, + self._api_key, + sock=sock + ) + + self._auto_connect_event_hub_thread = None + if auto_connect_event_hub in (None, True): + # Connect to event hub in background thread so as not to block main + # session usage waiting for event hub connection. + self._auto_connect_event_hub_thread = threading.Thread( + target=self._event_hub.connect + ) + self._auto_connect_event_hub_thread.daemon = True + self._auto_connect_event_hub_thread.start() + + # To help with migration from auto_connect_event_hub default changing + # from True to False. + self._event_hub._deprecation_warning_auto_connect = ( + auto_connect_event_hub is None + ) + + # Register to auto-close session on exit. + atexit.register(self.close) + + self._plugin_paths = plugin_paths + if self._plugin_paths is None: + self._plugin_paths = os.environ.get( + 'FTRACK_EVENT_PLUGIN_PATH', '' + ).split(os.pathsep) + + self._discover_plugins(plugin_arguments=plugin_arguments) + + # TODO: Make schemas read-only and non-mutable (or at least without + # rebuilding types)? + if schema_cache_path is not False: + if schema_cache_path is None: + schema_cache_path = os.environ.get( + 'FTRACK_API_SCHEMA_CACHE_PATH', tempfile.gettempdir() + ) + + schema_cache_path = os.path.join( + schema_cache_path, 'ftrack_api_schema_cache.json' + ) + + self.schemas = self._load_schemas(schema_cache_path) + self.types = self._build_entity_type_classes(self.schemas) + + ftrack_api._centralized_storage_scenario.register(self) + + self._configure_locations() + self.event_hub.publish( + ftrack_api.event.base.Event( + topic='ftrack.api.session.ready', + data=dict( + session=self + ) + ), + synchronous=True + ) From 5c70f3eb13497dd42c2681097ed0ba99d7b762da Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:12:54 +0200 Subject: [PATCH 07/65] modified event_server_cli to be able run subprocesses and handle their liveability --- .../parallel_event_server/__init__.py | 0 .../parallel_event_server/event_server_cli.py | 380 ++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/__init__.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/__init__.py b/pype/ftrack/ftrack_server/parallel_event_server/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py new file mode 100644 index 0000000000..b1955a9f91 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -0,0 +1,380 @@ +import os +import sys +import signal +import socket +import argparse +import time +from urllib.parse import urlparse + +import requests +from pype.vendor import ftrack_api +from pype.ftrack.lib import credentials +from pype.ftrack.ftrack_server import FtrackServer +from pypeapp import Logger +import socket_thread + +log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") + + +def check_ftrack_url(url, log_errors=True): + if not url: + log.error('Ftrack URL is not set!') + return None + + url = url.strip('/ ') + + if 'http' not in url: + if url.endswith('ftrackapp.com'): + url = 'https://' + url + else: + url = 'https://{0}.ftrackapp.com'.format(url) + try: + result = requests.get(url, allow_redirects=False) + except requests.exceptions.RequestException: + if log_errors: + log.error('Entered Ftrack URL is not accesible!') + return False + + if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): + if log_errors: + log.error('Entered Ftrack URL is not accesible!') + return False + + log.debug('Ftrack server {} is accessible.'.format(url)) + + return url + + +def check_mongo_url(host, port, log_error=False): + sock = None + try: + sock = socket.create_connection( + (host, port), + timeout=1 + ) + return True + except socket.error as err: + if log_error: + print("Can't connect to MongoDB at {}:{} because: {}".format( + host, port, err + )) + return False + finally: + if sock is not None: + sock.close() + + +def validate_credentials(url, user, api): + first_validation = True + if not user: + log.error('Ftrack Username is not set! Exiting.') + first_validation = False + if not api: + log.error('Ftrack API key is not set! Exiting.') + first_validation = False + if not first_validation: + return False + + try: + session = ftrack_api.Session( + server_url=url, + api_user=user, + api_key=api + ) + session.close() + except Exception as e: + log.error( + 'Can\'t log into Ftrack with used credentials:' + ' Ftrack server: "{}" // Username: {} // API key: {}'.format( + url, user, api + )) + return False + + log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + user, api + )) + return True + + +def process_event_paths(event_paths): + log.debug('Processing event paths: {}.'.format(str(event_paths))) + return_paths = [] + not_found = [] + if not event_paths: + return return_paths, not_found + + if isinstance(event_paths, str): + event_paths = event_paths.split(os.pathsep) + + for path in event_paths: + if os.path.exists(path): + return_paths.append(path) + else: + not_found.append(path) + + return os.pathsep.join(return_paths), not_found + + +def main_loop(ftrack_url, username, api_key, event_paths): + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + # Get mongo hostname and port for testing mongo connection + mongo_url = os.environ["AVALON_MONGO"].strip('/ ') + result = urlparse(mongo_url) + url_items = result.netloc.split("@") + mongo_url = url_items[0] + if len(url_items) == 2: + mongo_url = url_items[1] + + mongo_url = "://".join([result.scheme, mongo_url]) + result = urlparse(mongo_url) + + mongo_hostname = result.hostname + mongo_port = result.port + + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + # Threads data + storer_name = "StorerThread" + storer_port = 10005 + storer_path = "{}/sub_event_storer.py".format(file_path) + storer_thread = None + + processor_name = "ProcessorThread" + processor_port = 10006 + processor_path = "{}/sub_event_processor.py".format(file_path) + processor_thread = None + + backup_name = "BackupThread" + backup_port = 10007 + backup_path = "{}/sub_event_backup.py".format(file_path) + backup_thread = None + + ftrack_accessible = False + mongo_accessible = False + + # Main loop + while True: + # Check if accessible Ftrack and Mongo url + if not ftrack_accessible or not mongo_accessible: + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible: + time.sleep(1) + continue + + # Run backup thread which does not requeire mongo to work + if not mongo_accessible: + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + if backup_thread is None: + backup_thread = socket_thread.SocketThread( + backup_name, backup_port, backup_path + ) + backup_thread.start() + + else: + if backup_thread is not None: + if backup_thread.finished: + backup_thread.join() + backup_thread = None + else: + backup_thread.process_to_die() + + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False + + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False + + time.sleep(1) + + +def main(argv): + ''' + There are 4 values neccessary for event server: + 1.) Ftrack url - "studio.ftrackapp.com" + 2.) Username - "my.username" + 3.) API key - "apikey-long11223344-6665588-5565" + 4.) Path/s to events - "X:/path/to/folder/with/events" + + All these values can be entered with arguments or environment variables. + - arguments: + "-ftrackurl {url}" + "-ftrackuser {username}" + "-ftrackapikey {api key}" + "-ftrackeventpaths {path to events}" + - environment variables: + FTRACK_SERVER + FTRACK_API_USER + FTRACK_API_KEY + FTRACK_EVENTS_PATH + + Credentials (Username & API key): + - Credentials can be stored for auto load on next start + - To *Store/Update* these values add argument "-storecred" + - They will be stored to appsdir file when login is successful + - To *Update/Override* values with enviromnet variables is also needed to: + - *don't enter argument for that value* + - add argument "-noloadcred" (currently stored credentials won't be loaded) + + Order of getting values: + 1.) Arguments are always used when entered. + - entered values through args have most priority! (in each case) + 2.) Credentials are tried to load from appsdir file. + - skipped when credentials were entered through args or credentials + are not stored yet + - can be skipped with "-noloadcred" argument + 3.) Environment variables are last source of values. + - will try to get not yet set values from environments + + Best practice: + - set environment variables FTRACK_SERVER & FTRACK_EVENTS_PATH + - launch event_server_cli with args: + ~/event_server_cli.py -ftrackuser "{username}" -ftrackapikey "{API key}" -storecred + - next time launch event_server_cli.py only with set environment variables + FTRACK_SERVER & FTRACK_EVENTS_PATH + ''' + parser = argparse.ArgumentParser(description='Ftrack event server') + parser.add_argument( + "-ftrackurl", type=str, metavar='FTRACKURL', + help=( + "URL to ftrack server where events should handle" + " (default from environment: $FTRACK_SERVER)" + ) + ) + parser.add_argument( + "-ftrackuser", type=str, + help=( + "Username should be the username of the user in ftrack" + " to record operations against." + " (default from environment: $FTRACK_API_USER)" + ) + ) + parser.add_argument( + "-ftrackapikey", type=str, + help=( + "Should be the API key to use for authentication" + " (default from environment: $FTRACK_API_KEY)" + ) + ) + parser.add_argument( + "-ftrackeventpaths", nargs='+', + help=( + "List of paths where events are stored." + " (default from environment: $FTRACK_EVENTS_PATH)" + ) + ) + parser.add_argument( + '-storecred', + help=( + "Entered credentials will be also stored" + " to apps dir for future usage" + ), + action="store_true" + ) + parser.add_argument( + '-noloadcred', + help="Load creadentials from apps dir", + action="store_true" + ) + + ftrack_url = os.environ.get('FTRACK_SERVER') + username = os.environ.get('FTRACK_API_USER') + api_key = os.environ.get('FTRACK_API_KEY') + event_paths = os.environ.get('FTRACK_EVENTS_PATH') + + kwargs, args = parser.parse_known_args(argv) + + if kwargs.ftrackurl: + ftrack_url = kwargs.ftrackurl + + if kwargs.ftrackeventpaths: + event_paths = kwargs.ftrackeventpaths + + if not kwargs.noloadcred: + cred = credentials._get_credentials(True) + username = cred.get('username') + api_key = cred.get('apiKey') + + if kwargs.ftrackuser: + username = kwargs.ftrackuser + + if kwargs.ftrackapikey: + api_key = kwargs.ftrackapikey + + # Check url regex and accessibility + ftrack_url = check_ftrack_url(ftrack_url) + if not ftrack_url: + return 1 + + # Validate entered credentials + if not validate_credentials(ftrack_url, username, api_key): + return 1 + + # Process events path + event_paths, not_found = process_event_paths(event_paths) + if not_found: + log.warning( + 'These paths were not found: {}'.format(str(not_found)) + ) + if not event_paths: + if not_found: + log.error('Any of entered paths is valid or can be accesible.') + else: + log.error('Paths to events are not set. Exiting.') + return 1 + + if kwargs.storecred: + credentials._save_credentials(username, api_key, True) + + main_loop(ftrack_url, username, api_key, event_paths) + + +if (__name__ == ('__main__')): + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + sys.exit(main(sys.argv)) From 28f2d1431856b395efcdc43d3d520cedbac1bc9e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 14:01:58 +0200 Subject: [PATCH 08/65] pype logging is not used in event_server_cli for cases when mongo is not accessible and removed backup solution --- .../parallel_event_server/event_server_cli.py | 131 ++++++++---------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index b1955a9f91..3814cd0549 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -10,15 +10,12 @@ import requests from pype.vendor import ftrack_api from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer -from pypeapp import Logger import socket_thread -log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") - def check_ftrack_url(url, log_errors=True): if not url: - log.error('Ftrack URL is not set!') + print('ERROR: Ftrack URL is not set!') return None url = url.strip('/ ') @@ -32,15 +29,15 @@ def check_ftrack_url(url, log_errors=True): result = requests.get(url, allow_redirects=False) except requests.exceptions.RequestException: if log_errors: - log.error('Entered Ftrack URL is not accesible!') + print('ERROR: Entered Ftrack URL is not accesible!') return False if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): if log_errors: - log.error('Entered Ftrack URL is not accesible!') + print('ERROR: Entered Ftrack URL is not accesible!') return False - log.debug('Ftrack server {} is accessible.'.format(url)) + print('DEBUG: Ftrack server {} is accessible.'.format(url)) return url @@ -67,10 +64,10 @@ def check_mongo_url(host, port, log_error=False): def validate_credentials(url, user, api): first_validation = True if not user: - log.error('Ftrack Username is not set! Exiting.') + print('ERROR: Ftrack Username is not set! Exiting.') first_validation = False if not api: - log.error('Ftrack API key is not set! Exiting.') + print('ERROR: Ftrack API key is not set! Exiting.') first_validation = False if not first_validation: return False @@ -83,21 +80,21 @@ def validate_credentials(url, user, api): ) session.close() except Exception as e: - log.error( - 'Can\'t log into Ftrack with used credentials:' + print( + 'ERROR: Can\'t log into Ftrack with used credentials:' ' Ftrack server: "{}" // Username: {} // API key: {}'.format( url, user, api )) return False - log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( user, api )) return True def process_event_paths(event_paths): - log.debug('Processing event paths: {}.'.format(str(event_paths))) + print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) return_paths = [] not_found = [] if not event_paths: @@ -150,80 +147,64 @@ def main_loop(ftrack_url, username, api_key, event_paths): processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None - backup_name = "BackupThread" - backup_port = 10007 - backup_path = "{}/sub_event_backup.py".format(file_path) - backup_thread = None - ftrack_accessible = False mongo_accessible = False + printed_ftrack_error = False + printed_mongo_error = False + # Main loop while True: # Check if accessible Ftrack and Mongo url - if not ftrack_accessible or not mongo_accessible: - if not ftrack_accessible: - ftrack_accessible = check_ftrack_url(ftrack_url) + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) - if not mongo_accessible: - mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) # Run threads only if Ftrack is accessible - if not ftrack_accessible: + if not ftrack_accessible or not mongo_accessible: + if not mongo_accessible and not printed_mongo_error: + print("Can't access Mongo {}".format(mongo_url)) + + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {}".format(ftrack_url)) + + printed_ftrack_error = True + printed_mongo_error = True + time.sleep(1) continue + printed_ftrack_error = False + printed_mongo_error = False + # Run backup thread which does not requeire mongo to work - if not mongo_accessible: - if storer_thread is not None: - storer_thread.stop() - storer_thread.join() - storer_thread = None + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() - if processor_thread is not None: - processor_thread.stop() - processor_thread.join() - processor_thread = None + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False - if backup_thread is None: - backup_thread = socket_thread.SocketThread( - backup_name, backup_port, backup_path - ) - backup_thread.start() + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() - else: - if backup_thread is not None: - if backup_thread.finished: - backup_thread.join() - backup_thread = None - else: - backup_thread.process_to_die() - - if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif not storer_thread.isAlive(): - storer_thread.join() - storer_thread = None - ftrack_accessible = False - mongo_accessible = False - - if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): - processor_thread.join() - processor_thread = None - ftrack_accessible = False - mongo_accessible = False + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False time.sleep(1) @@ -353,14 +334,14 @@ def main(argv): # Process events path event_paths, not_found = process_event_paths(event_paths) if not_found: - log.warning( - 'These paths were not found: {}'.format(str(not_found)) + print( + 'WARNING: These paths were not found: {}'.format(str(not_found)) ) if not event_paths: if not_found: - log.error('Any of entered paths is valid or can be accesible.') + print('ERROR: Any of entered paths is valid or can be accesible.') else: - log.error('Paths to events are not set. Exiting.') + print('ERROR: Paths to events are not set. Exiting.') return 1 if kwargs.storecred: From bcf8bf17a521fb8b0643fbc786317bec192bfefb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 14:02:49 +0200 Subject: [PATCH 09/65] session instance validation also checks for process session and raises exception if does not match ftrack_api session --- pype/ftrack/lib/ftrack_base_handler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 9821bfd81f..05c84eea84 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -32,18 +32,18 @@ class BaseHandler(object): def __init__(self, session, plugins_presets={}): '''Expects a ftrack_api.Session instance''' self.log = Logger().get_logger(self.__class__.__name__) - if not isinstance(session, ftrack_api.session.Session): - self.log.warning(( + if not( + isinstance(session, ftrack_api.session.Session) or + isinstance(session, ProcessSession) + ): + raise Exception(( "Session object entered with args is instance of \"{}\"" - " but expected instance is \"{}\"." + " but expected instances are \"{}\" and \"{}\"" ).format( str(type(session)), - str(ftrack_api.session.Session.__qualname__) + str(ftrack_api.session.Session), + str(ProcessSession) )) - self.register = self.register_without_session - self._session = None - - return self._session = session From b7c29540612443d77f435a00e9d628a089c1e5d0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:26:31 +0200 Subject: [PATCH 10/65] changed default ports --- .../ftrack_server/parallel_event_server/event_server_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index 3814cd0549..15afb6672d 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -138,12 +138,12 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Threads data storer_name = "StorerThread" - storer_port = 10005 + storer_port = 10001 storer_path = "{}/sub_event_storer.py".format(file_path) storer_thread = None processor_name = "ProcessorThread" - processor_port = 10006 + processor_port = 10011 processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None From d1bfa2412e417b455ba716e42e180d964aec7339 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:33:01 +0200 Subject: [PATCH 11/65] pymongo connection error cause that subprocess will end --- .../session_processor.py | 28 +++++++++++-------- .../parallel_event_server/sub_event_storer.py | 13 ++++++--- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py index 55bdf0fba9..d51da28938 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py @@ -31,11 +31,14 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): super(ProcessEventHub, self).__init__(*args, **kwargs) def prepare_dbcon(self): - self.dbcon.install() - if not self.is_table_created: - self.dbcon.create_table(self.table_name, capped=False) - self.dbcon.active_table = self.table_name - self.is_table_created = True + try: + self.dbcon.install() + if not self.is_table_created: + self.dbcon.create_table(self.table_name, capped=False) + self.dbcon.active_table = self.table_name + self.is_table_created = True + except pymongo.errors.AutoReconnect: + sys.exit(0) def wait(self, duration=None): '''Wait for events and handle as they arrive. @@ -46,7 +49,6 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): ''' started = time.time() - self.prepare_dbcon() while True: try: @@ -55,11 +57,14 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): if not self.load_events(): time.sleep(0.5) else: - self._handle(event) - self.dbcon.update_one( - {"id": event["id"]}, - {"$set": {"pype_data.is_processed": True}} - ) + try: + self._handle(event) + self.dbcon.update_one( + {"id": event["id"]}, + {"$set": {"pype_data.is_processed": True}} + ) + except pymongo.errors.AutoReconnect: + sys.exit(0) # Additional special processing of events. if event['topic'] == 'ftrack.meta.disconnected': break @@ -78,7 +83,6 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): } try: event = ftrack_api.event.base.Event(**new_event_data) - print(event) except Exception: self.logger.exception(L( 'Failed to convert payload into event: {0}', diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py index 295504f389..bf2dd0eefa 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -23,9 +23,12 @@ table_name = "ftrack_events" ignore_topics = [] def install_db(): - dbcon.install() - dbcon.create_table(table_name, capped=False) - dbcon.active_table = table_name + try: + dbcon.install() + dbcon.create_table(table_name, capped=False) + dbcon.active_table = table_name + except pymongo.errors.AutoReconnect: + sys.exit(0) def launch(event): if event.get("topic") in ignore_topics: @@ -47,6 +50,9 @@ def launch(event): except pymongo.errors.DuplicateKeyError: log.debug("Event: {} already exists".format(event_id)) + except pymongo.errors.AutoReconnect: + sys.exit(0) + except Exception as exc: log.error( "Event: {} failed to store".format(event_id), @@ -76,7 +82,6 @@ def main(args): session = StorerSession(auto_connect_event_hub=True, sock=sock) register(session) server = FtrackServer("event") - log.info(os.environ["FTRACK_EVENTS_PATH"]) log.debug("Launched Ftrack Event storer") server.run_server(session, load_files=False) From 4921783d1a25976275dc0ebf22333d0114d2b7e7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:36:54 +0200 Subject: [PATCH 12/65] terminate signal is registered too to be able terminate subprocesses --- .../ftrack_server/parallel_event_server/socket_thread.py | 6 +++--- .../parallel_event_server/sub_event_processor.py | 1 + .../ftrack_server/parallel_event_server/sub_event_storer.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py index deede2ed03..1ec4eca57f 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -68,7 +68,7 @@ class SocketThread(threading.Thread): if (time.time() - time_socket) > self.MAX_TIMEOUT: self.log.error("Connection timeout passed. Terminating.") self._is_running = False - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() break continue @@ -89,7 +89,7 @@ class SocketThread(threading.Thread): "Connection timeout passed. Terminating." ) self._is_running = False - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() break continue @@ -109,6 +109,6 @@ class SocketThread(threading.Thread): # Clean up the connection connection.close() if self.subproc.poll() is None: - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() self.finished = True diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py index bd3118a893..a795775a94 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -42,5 +42,6 @@ if __name__ == "__main__": sys.exit(0) signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) main(sys.argv) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py index bf2dd0eefa..864b4b7c1d 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -98,5 +98,6 @@ if __name__ == "__main__": sys.exit(0) signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) main(sys.argv) From f1c873fedc7ec05bb4d67436d534e0348e6fd376 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:38:32 +0200 Subject: [PATCH 13/65] minor fixes, occupied socket ports are skipped and logger is renamed --- .../parallel_event_server/event_server_cli.py | 10 ++++++++++ .../parallel_event_server/socket_thread.py | 13 +++++++++---- .../parallel_event_server/sub_event_processor.py | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index 15afb6672d..55670178e2 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -170,6 +170,16 @@ def main_loop(ftrack_url, username, api_key, event_paths): if not ftrack_accessible and not printed_ftrack_error: print("Can't access Ftrack {}".format(ftrack_url)) + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + printed_ftrack_error = True printed_mongo_error = True diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py index 1ec4eca57f..e787004afc 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -24,7 +24,6 @@ class SocketThread(threading.Thread): def stop(self): self._is_running = False - super().stop() def process_to_die(self): if not self.connection: @@ -40,9 +39,15 @@ class SocketThread(threading.Thread): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock = sock - # Bind the socket to the port - server_address = ("localhost", self.port) - sock.bind(server_address) + # Bind the socket to the port - skip already used ports + while True: + try: + server_address = ("localhost", self.port) + sock.bind(server_address) + break + except OSError: + self.port += 1 + self.log.debug( "Running Socked thread on {}:{}".format(*server_address) ) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py index a795775a94..c94223deae 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -9,7 +9,7 @@ from pype.ftrack.ftrack_server import FtrackServer from session_processor import ProcessSession from pypeapp import Logger -log = Logger().get_logger("Event storer") +log = Logger().get_logger("Event processor") def main(args): From 4290af23e7845e981aa80898a90d21d2bfa49969 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:57:53 +0200 Subject: [PATCH 14/65] moved event server files in hierarchy --- pype/ftrack/ftrack_server/__init__.py | 11 +- pype/ftrack/ftrack_server/event_server_cli.py | 193 +++++++-- .../parallel_event_server/__init__.py | 0 .../parallel_event_server/event_server_cli.py | 371 ------------------ .../session_processor.py | 0 .../session_storer.py | 0 .../socket_thread.py | 0 .../sub_event_processor.py | 6 +- .../sub_event_storer.py | 7 +- pype/ftrack/lib/ftrack_base_handler.py | 5 +- 10 files changed, 176 insertions(+), 417 deletions(-) delete mode 100644 pype/ftrack/ftrack_server/parallel_event_server/__init__.py delete mode 100644 pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py rename pype/ftrack/ftrack_server/{parallel_event_server => }/session_processor.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/session_storer.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/socket_thread.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/sub_event_processor.py (86%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/sub_event_storer.py (90%) diff --git a/pype/ftrack/ftrack_server/__init__.py b/pype/ftrack/ftrack_server/__init__.py index 4a64ab8848..f8f876bbb6 100644 --- a/pype/ftrack/ftrack_server/__init__.py +++ b/pype/ftrack/ftrack_server/__init__.py @@ -1,7 +1,8 @@ from .ftrack_server import FtrackServer -from . import event_server_cli -__all__ = [ - 'event_server_cli', - 'FtrackServer' -] +import event_server_cli +import session_processor +import session_storer +import socket_thread +import sub_event_processor +import sub_event_storer diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index e06a626468..83ecd0d47a 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -1,18 +1,21 @@ import os import sys +import signal +import socket import argparse +import time +from urllib.parse import urlparse + import requests from pype.vendor import ftrack_api -from pype.ftrack import credentials +from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer -from pypeapp import Logger - -log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") +import socket_thread -def check_url(url): +def check_ftrack_url(url, log_errors=True): if not url: - log.error('Ftrack URL is not set!') + print('ERROR: Ftrack URL is not set!') return None url = url.strip('/ ') @@ -25,24 +28,46 @@ def check_url(url): try: result = requests.get(url, allow_redirects=False) except requests.exceptions.RequestException: - log.error('Entered Ftrack URL is not accesible!') - return None + if log_errors: + print('ERROR: Entered Ftrack URL is not accesible!') + return False if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): - log.error('Entered Ftrack URL is not accesible!') - return None + if log_errors: + print('ERROR: Entered Ftrack URL is not accesible!') + return False - log.debug('Ftrack server {} is accessible.'.format(url)) + print('DEBUG: Ftrack server {} is accessible.'.format(url)) return url + +def check_mongo_url(host, port, log_error=False): + sock = None + try: + sock = socket.create_connection( + (host, port), + timeout=1 + ) + return True + except socket.error as err: + if log_error: + print("Can't connect to MongoDB at {}:{} because: {}".format( + host, port, err + )) + return False + finally: + if sock is not None: + sock.close() + + def validate_credentials(url, user, api): first_validation = True if not user: - log.error('Ftrack Username is not set! Exiting.') + print('ERROR: Ftrack Username is not set! Exiting.') first_validation = False if not api: - log.error('Ftrack API key is not set! Exiting.') + print('ERROR: Ftrack API key is not set! Exiting.') first_validation = False if not first_validation: return False @@ -55,21 +80,21 @@ def validate_credentials(url, user, api): ) session.close() except Exception as e: - log.error( - 'Can\'t log into Ftrack with used credentials:' + print( + 'ERROR: Can\'t log into Ftrack with used credentials:' ' Ftrack server: "{}" // Username: {} // API key: {}'.format( url, user, api )) return False - log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( user, api )) return True def process_event_paths(event_paths): - log.debug('Processing event paths: {}.'.format(str(event_paths))) + print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) return_paths = [] not_found = [] if not event_paths: @@ -87,14 +112,112 @@ def process_event_paths(event_paths): return os.pathsep.join(return_paths), not_found -def run_event_server(ftrack_url, username, api_key, event_paths): - os.environ['FTRACK_SERVER'] = ftrack_url - os.environ['FTRACK_API_USER'] = username - os.environ['FTRACK_API_KEY'] = api_key - os.environ['FTRACK_EVENTS_PATH'] = event_paths +def main_loop(ftrack_url, username, api_key, event_paths): + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + # Get mongo hostname and port for testing mongo connection + mongo_url = os.environ["AVALON_MONGO"].strip('/ ') + result = urlparse(mongo_url) + url_items = result.netloc.split("@") + mongo_url = url_items[0] + if len(url_items) == 2: + mongo_url = url_items[1] + + mongo_url = "://".join([result.scheme, mongo_url]) + result = urlparse(mongo_url) + + mongo_hostname = result.hostname + mongo_port = result.port + + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + # Threads data + storer_name = "StorerThread" + storer_port = 10001 + storer_path = "{}/sub_event_storer.py".format(file_path) + storer_thread = None + + processor_name = "ProcessorThread" + processor_port = 10011 + processor_path = "{}/sub_event_processor.py".format(file_path) + processor_thread = None + + ftrack_accessible = False + mongo_accessible = False + + printed_ftrack_error = False + printed_mongo_error = False + + # Main loop + while True: + # Check if accessible Ftrack and Mongo url + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible or not mongo_accessible: + if not mongo_accessible and not printed_mongo_error: + print("Can't access Mongo {}".format(mongo_url)) + + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {}".format(ftrack_url)) + + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + printed_ftrack_error = True + printed_mongo_error = True + + time.sleep(1) + continue + + printed_ftrack_error = False + printed_mongo_error = False + + # Run backup thread which does not requeire mongo to work + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False + + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False + + time.sleep(1) - server = FtrackServer('event') - server.run_server() def main(argv): ''' @@ -210,7 +333,7 @@ def main(argv): api_key = kwargs.ftrackapikey # Check url regex and accessibility - ftrack_url = check_url(ftrack_url) + ftrack_url = check_ftrack_url(ftrack_url) if not ftrack_url: return 1 @@ -221,21 +344,29 @@ def main(argv): # Process events path event_paths, not_found = process_event_paths(event_paths) if not_found: - log.warning( - 'These paths were not found: {}'.format(str(not_found)) + print( + 'WARNING: These paths were not found: {}'.format(str(not_found)) ) if not event_paths: if not_found: - log.error('Any of entered paths is valid or can be accesible.') + print('ERROR: Any of entered paths is valid or can be accesible.') else: - log.error('Paths to events are not set. Exiting.') + print('ERROR: Paths to events are not set. Exiting.') return 1 if kwargs.storecred: credentials._save_credentials(username, api_key, True) - run_event_server(ftrack_url, username, api_key, event_paths) + main_loop(ftrack_url, username, api_key, event_paths) -if (__name__ == ('__main__')): +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/__init__.py b/pype/ftrack/ftrack_server/parallel_event_server/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py deleted file mode 100644 index 55670178e2..0000000000 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ /dev/null @@ -1,371 +0,0 @@ -import os -import sys -import signal -import socket -import argparse -import time -from urllib.parse import urlparse - -import requests -from pype.vendor import ftrack_api -from pype.ftrack.lib import credentials -from pype.ftrack.ftrack_server import FtrackServer -import socket_thread - - -def check_ftrack_url(url, log_errors=True): - if not url: - print('ERROR: Ftrack URL is not set!') - return None - - url = url.strip('/ ') - - if 'http' not in url: - if url.endswith('ftrackapp.com'): - url = 'https://' + url - else: - url = 'https://{0}.ftrackapp.com'.format(url) - try: - result = requests.get(url, allow_redirects=False) - except requests.exceptions.RequestException: - if log_errors: - print('ERROR: Entered Ftrack URL is not accesible!') - return False - - if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): - if log_errors: - print('ERROR: Entered Ftrack URL is not accesible!') - return False - - print('DEBUG: Ftrack server {} is accessible.'.format(url)) - - return url - - -def check_mongo_url(host, port, log_error=False): - sock = None - try: - sock = socket.create_connection( - (host, port), - timeout=1 - ) - return True - except socket.error as err: - if log_error: - print("Can't connect to MongoDB at {}:{} because: {}".format( - host, port, err - )) - return False - finally: - if sock is not None: - sock.close() - - -def validate_credentials(url, user, api): - first_validation = True - if not user: - print('ERROR: Ftrack Username is not set! Exiting.') - first_validation = False - if not api: - print('ERROR: Ftrack API key is not set! Exiting.') - first_validation = False - if not first_validation: - return False - - try: - session = ftrack_api.Session( - server_url=url, - api_user=user, - api_key=api - ) - session.close() - except Exception as e: - print( - 'ERROR: Can\'t log into Ftrack with used credentials:' - ' Ftrack server: "{}" // Username: {} // API key: {}'.format( - url, user, api - )) - return False - - print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( - user, api - )) - return True - - -def process_event_paths(event_paths): - print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) - return_paths = [] - not_found = [] - if not event_paths: - return return_paths, not_found - - if isinstance(event_paths, str): - event_paths = event_paths.split(os.pathsep) - - for path in event_paths: - if os.path.exists(path): - return_paths.append(path) - else: - not_found.append(path) - - return os.pathsep.join(return_paths), not_found - - -def main_loop(ftrack_url, username, api_key, event_paths): - # Set Ftrack environments - os.environ["FTRACK_SERVER"] = ftrack_url - os.environ["FTRACK_API_USER"] = username - os.environ["FTRACK_API_KEY"] = api_key - os.environ["FTRACK_EVENTS_PATH"] = event_paths - - # Get mongo hostname and port for testing mongo connection - mongo_url = os.environ["AVALON_MONGO"].strip('/ ') - result = urlparse(mongo_url) - url_items = result.netloc.split("@") - mongo_url = url_items[0] - if len(url_items) == 2: - mongo_url = url_items[1] - - mongo_url = "://".join([result.scheme, mongo_url]) - result = urlparse(mongo_url) - - mongo_hostname = result.hostname - mongo_port = result.port - - # Current file - file_path = os.path.dirname(os.path.realpath(__file__)) - - # Threads data - storer_name = "StorerThread" - storer_port = 10001 - storer_path = "{}/sub_event_storer.py".format(file_path) - storer_thread = None - - processor_name = "ProcessorThread" - processor_port = 10011 - processor_path = "{}/sub_event_processor.py".format(file_path) - processor_thread = None - - ftrack_accessible = False - mongo_accessible = False - - printed_ftrack_error = False - printed_mongo_error = False - - # Main loop - while True: - # Check if accessible Ftrack and Mongo url - if not ftrack_accessible: - ftrack_accessible = check_ftrack_url(ftrack_url) - - if not mongo_accessible: - mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) - - # Run threads only if Ftrack is accessible - if not ftrack_accessible or not mongo_accessible: - if not mongo_accessible and not printed_mongo_error: - print("Can't access Mongo {}".format(mongo_url)) - - if not ftrack_accessible and not printed_ftrack_error: - print("Can't access Ftrack {}".format(ftrack_url)) - - if storer_thread is not None: - storer_thread.stop() - storer_thread.join() - storer_thread = None - - if processor_thread is not None: - processor_thread.stop() - processor_thread.join() - processor_thread = None - - printed_ftrack_error = True - printed_mongo_error = True - - time.sleep(1) - continue - - printed_ftrack_error = False - printed_mongo_error = False - - # Run backup thread which does not requeire mongo to work - if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif not storer_thread.isAlive(): - storer_thread.join() - storer_thread = None - ftrack_accessible = False - mongo_accessible = False - - if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): - processor_thread.join() - processor_thread = None - ftrack_accessible = False - mongo_accessible = False - - time.sleep(1) - - -def main(argv): - ''' - There are 4 values neccessary for event server: - 1.) Ftrack url - "studio.ftrackapp.com" - 2.) Username - "my.username" - 3.) API key - "apikey-long11223344-6665588-5565" - 4.) Path/s to events - "X:/path/to/folder/with/events" - - All these values can be entered with arguments or environment variables. - - arguments: - "-ftrackurl {url}" - "-ftrackuser {username}" - "-ftrackapikey {api key}" - "-ftrackeventpaths {path to events}" - - environment variables: - FTRACK_SERVER - FTRACK_API_USER - FTRACK_API_KEY - FTRACK_EVENTS_PATH - - Credentials (Username & API key): - - Credentials can be stored for auto load on next start - - To *Store/Update* these values add argument "-storecred" - - They will be stored to appsdir file when login is successful - - To *Update/Override* values with enviromnet variables is also needed to: - - *don't enter argument for that value* - - add argument "-noloadcred" (currently stored credentials won't be loaded) - - Order of getting values: - 1.) Arguments are always used when entered. - - entered values through args have most priority! (in each case) - 2.) Credentials are tried to load from appsdir file. - - skipped when credentials were entered through args or credentials - are not stored yet - - can be skipped with "-noloadcred" argument - 3.) Environment variables are last source of values. - - will try to get not yet set values from environments - - Best practice: - - set environment variables FTRACK_SERVER & FTRACK_EVENTS_PATH - - launch event_server_cli with args: - ~/event_server_cli.py -ftrackuser "{username}" -ftrackapikey "{API key}" -storecred - - next time launch event_server_cli.py only with set environment variables - FTRACK_SERVER & FTRACK_EVENTS_PATH - ''' - parser = argparse.ArgumentParser(description='Ftrack event server') - parser.add_argument( - "-ftrackurl", type=str, metavar='FTRACKURL', - help=( - "URL to ftrack server where events should handle" - " (default from environment: $FTRACK_SERVER)" - ) - ) - parser.add_argument( - "-ftrackuser", type=str, - help=( - "Username should be the username of the user in ftrack" - " to record operations against." - " (default from environment: $FTRACK_API_USER)" - ) - ) - parser.add_argument( - "-ftrackapikey", type=str, - help=( - "Should be the API key to use for authentication" - " (default from environment: $FTRACK_API_KEY)" - ) - ) - parser.add_argument( - "-ftrackeventpaths", nargs='+', - help=( - "List of paths where events are stored." - " (default from environment: $FTRACK_EVENTS_PATH)" - ) - ) - parser.add_argument( - '-storecred', - help=( - "Entered credentials will be also stored" - " to apps dir for future usage" - ), - action="store_true" - ) - parser.add_argument( - '-noloadcred', - help="Load creadentials from apps dir", - action="store_true" - ) - - ftrack_url = os.environ.get('FTRACK_SERVER') - username = os.environ.get('FTRACK_API_USER') - api_key = os.environ.get('FTRACK_API_KEY') - event_paths = os.environ.get('FTRACK_EVENTS_PATH') - - kwargs, args = parser.parse_known_args(argv) - - if kwargs.ftrackurl: - ftrack_url = kwargs.ftrackurl - - if kwargs.ftrackeventpaths: - event_paths = kwargs.ftrackeventpaths - - if not kwargs.noloadcred: - cred = credentials._get_credentials(True) - username = cred.get('username') - api_key = cred.get('apiKey') - - if kwargs.ftrackuser: - username = kwargs.ftrackuser - - if kwargs.ftrackapikey: - api_key = kwargs.ftrackapikey - - # Check url regex and accessibility - ftrack_url = check_ftrack_url(ftrack_url) - if not ftrack_url: - return 1 - - # Validate entered credentials - if not validate_credentials(ftrack_url, username, api_key): - return 1 - - # Process events path - event_paths, not_found = process_event_paths(event_paths) - if not_found: - print( - 'WARNING: These paths were not found: {}'.format(str(not_found)) - ) - if not event_paths: - if not_found: - print('ERROR: Any of entered paths is valid or can be accesible.') - else: - print('ERROR: Paths to events are not set. Exiting.') - return 1 - - if kwargs.storecred: - credentials._save_credentials(username, api_key, True) - - main_loop(ftrack_url, username, api_key, event_paths) - - -if (__name__ == ('__main__')): - # Register interupt signal - def signal_handler(sig, frame): - print("You pressed Ctrl+C. Process ended.") - sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - - sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/session_processor.py rename to pype/ftrack/ftrack_server/session_processor.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py b/pype/ftrack/ftrack_server/session_storer.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/session_storer.py rename to pype/ftrack/ftrack_server/session_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py rename to pype/ftrack/ftrack_server/socket_thread.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py similarity index 86% rename from pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py rename to pype/ftrack/ftrack_server/sub_event_processor.py index c94223deae..b53c294981 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -5,8 +5,8 @@ import signal import socket import pymongo -from pype.ftrack.ftrack_server import FtrackServer -from session_processor import ProcessSession +from ftrack_server import FtrackServer +import session_processor from pypeapp import Logger log = Logger().get_logger("Event processor") @@ -24,7 +24,7 @@ def main(args): sock.sendall(b"CreatedProcess") try: - session = ProcessSession(auto_connect_event_hub=True, sock=sock) + session = session_processor.ProcessSession(auto_connect_event_hub=True, sock=sock) server = FtrackServer('event') log.debug("Launched Ftrack Event processor") server.run_server(session) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py similarity index 90% rename from pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py rename to pype/ftrack/ftrack_server/sub_event_storer.py index 864b4b7c1d..918e1e1bea 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -5,9 +5,9 @@ import signal import socket import pymongo -from pype.ftrack.ftrack_server import FtrackServer +from ftrack_server import FtrackServer from pype.ftrack.lib.custom_db_connector import DbConnector -from pype.ftrack.ftrack_server.parallel_event_server.session_storer import StorerSession +from session_storer import StorerSession from pypeapp import Logger log = Logger().get_logger("Event storer") @@ -47,9 +47,6 @@ def launch(event): dbcon.update({"id": event_id}, event_data, upsert=True) log.debug("Event: {} stored".format(event_id)) - except pymongo.errors.DuplicateKeyError: - log.debug("Event: {} already exists".format(event_id)) - except pymongo.errors.AutoReconnect: sys.exit(0) diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 05c84eea84..13e1cae9a9 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -3,6 +3,7 @@ import time from pypeapp import Logger from pype.vendor import ftrack_api from pype.vendor.ftrack_api import session as fa_session +from pype.ftrack.ftrack_server import session_processor class MissingPermision(Exception): @@ -34,7 +35,7 @@ class BaseHandler(object): self.log = Logger().get_logger(self.__class__.__name__) if not( isinstance(session, ftrack_api.session.Session) or - isinstance(session, ProcessSession) + isinstance(session, session_processor.ProcessSession) ): raise Exception(( "Session object entered with args is instance of \"{}\"" @@ -42,7 +43,7 @@ class BaseHandler(object): ).format( str(type(session)), str(ftrack_api.session.Session), - str(ProcessSession) + str(session_processor.ProcessSession) )) self._session = session From 19f810ff57e7a55a70ed7cba9afc10924a6e9c48 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:01:51 +0200 Subject: [PATCH 15/65] added check active table to custom db connector --- pype/ftrack/lib/custom_db_connector.py | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 505ac96610..f4cc00444b 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -20,6 +20,9 @@ import requests import pymongo from pymongo.client_session import ClientSession +class NotActiveTable(Exception): + pass + def auto_reconnect(func): """Handling auto reconnect in 3 retry times""" @functools.wraps(func) @@ -37,12 +40,23 @@ def auto_reconnect(func): return decorated +def check_active_table(func): + """Handling auto reconnect in 3 retry times""" + @functools.wraps(func) + def decorated(obj, *args, **kwargs): + if not obj.active_table: + raise NotActiveTable("Active table is not set. (This is bug)") + return func(obj, *args, **kwargs) + + return decorated + + class DbConnector: log = logging.getLogger(__name__) timeout = 1000 - def __init__(self, mongo_url, database_name, table_name): + def __init__(self, mongo_url, database_name, table_name=None): self._mongo_client = None self._sentry_client = None self._sentry_logging_handler = None @@ -115,6 +129,7 @@ class DbConnector: def collections(self): return self._database.collection_names() + @check_active_table @auto_reconnect def insert_one(self, item, session=None): assert isinstance(item, dict), "item must be of type " @@ -123,6 +138,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def insert_many(self, items, ordered=True, session=None): # check if all items are valid @@ -136,6 +152,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find(self, filter, projection=None, sort=None, session=None): return self._database[self.active_table].find( @@ -145,6 +162,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find_one(self, filter, projection=None, sort=None, session=None): assert isinstance(filter, dict), "filter must be " @@ -156,6 +174,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def replace_one(self, filter, replacement, session=None): return self._database[self.active_table].replace_one( @@ -163,6 +182,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def update_one(self, filter, update, session=None): return self._database[self.active_table].update_one( @@ -170,6 +190,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def update_many(self, filter, update, session=None): return self._database[self.active_table].update_many( @@ -177,12 +198,14 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def distinct(self, *args, **kwargs): return self._database[self.active_table].distinct( *args, **kwargs ) + @check_active_table @auto_reconnect def drop_collection(self, name_or_collection, session=None): return self._database[self.active_table].drop( @@ -190,6 +213,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def delete_one(filter, collation=None, session=None): return self._database[self.active_table].delete_one( @@ -198,6 +222,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def delete_many(filter, collation=None, session=None): return self._database[self.active_table].delete_many( From 1ca674f33f18e7821aa47f039f0b808b50293b99 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:05:07 +0200 Subject: [PATCH 16/65] added atexit to custom db connector to run uninstall on exit --- pype/ftrack/lib/custom_db_connector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index f4cc00444b..222a565ded 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -13,6 +13,7 @@ import logging import tempfile import functools import contextlib +import atexit import requests @@ -71,7 +72,7 @@ class DbConnector: """Establish a persistent connection to the database""" if self._is_installed: return - + atexit.register(self.uninstall) logging.basicConfig() self._mongo_client = pymongo.MongoClient( @@ -113,6 +114,7 @@ class DbConnector: self._mongo_client = None self._database = None self._is_installed = False + atexit.unregister(self.uninstall) def tables(self): """List available tables From 45370876e33b1b3c188b2c95932989e2ccad458e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:29:05 +0200 Subject: [PATCH 17/65] replaced args with options(kwargs) in custom db connector --- pype/ftrack/lib/custom_db_connector.py | 95 ++++++++++++-------------- 1 file changed, 43 insertions(+), 52 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 222a565ded..11fd197555 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -68,6 +68,12 @@ class DbConnector: self.active_table = table_name + def __getattribute__(self, attr): + try: + return super().__getattribute__(attr) + except AttributeError: + return self._database[self.active_table].__getattribute__(attr) + def install(self): """Establish a persistent connection to the database""" if self._is_installed: @@ -116,6 +122,15 @@ class DbConnector: self._is_installed = False atexit.unregister(self.uninstall) + def create_table(self, name, **options): + if self.exist_table(name): + return + + return self._database.create_collection(name, **options) + + def exist_table(self, table_name): + return table_name in self.tables() + def tables(self): """List available tables Returns: @@ -133,102 +148,78 @@ class DbConnector: @check_active_table @auto_reconnect - def insert_one(self, item, session=None): + def insert_one(self, item, **options): assert isinstance(item, dict), "item must be of type " - return self._database[self.active_table].insert_one( - item, - session=session - ) + return self._database[self.active_table].insert_one(item, **options) @check_active_table @auto_reconnect - def insert_many(self, items, ordered=True, session=None): + def insert_many(self, items, ordered=True, **options): # check if all items are valid assert isinstance(items, list), "`items` must be of type " for item in items: assert isinstance(item, dict), "`item` must be of type " - return self._database[self.active_table].insert_many( - items, - ordered=ordered, - session=session - ) + options["ordered"] = ordered + return self._database[self.active_table].insert_many(items, **options) @check_active_table @auto_reconnect - def find(self, filter, projection=None, sort=None, session=None): - return self._database[self.active_table].find( - filter=filter, - projection=projection, - sort=sort, - session=session - ) + def find(self, filter, projection=None, sort=None, **options): + options["projection"] = projection + options["sort"] = sort + return self._database[self.active_table].find(filter, **options) @check_active_table @auto_reconnect - def find_one(self, filter, projection=None, sort=None, session=None): + def find_one(self, filter, projection=None, sort=None, **options): assert isinstance(filter, dict), "filter must be " - return self._database[self.active_table].find_one( - filter=filter, - projection=projection, - sort=sort, - session=session - ) + options["projection"] = projection + options["sort"] = sort + return self._database[self.active_table].find_one(filter, **options) @check_active_table @auto_reconnect - def replace_one(self, filter, replacement, session=None): + def replace_one(self, filter, replacement, **options): return self._database[self.active_table].replace_one( - filter, replacement, - session=session + filter, replacement, **options ) @check_active_table @auto_reconnect - def update_one(self, filter, update, session=None): + def update_one(self, filter, update, **options): return self._database[self.active_table].update_one( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect - def update_many(self, filter, update, session=None): + def update_many(self, filter, update, **options): return self._database[self.active_table].update_many( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect def distinct(self, *args, **kwargs): - return self._database[self.active_table].distinct( - *args, **kwargs - ) + return self._database[self.active_table].distinct(*args, **kwargs) @check_active_table @auto_reconnect - def drop_collection(self, name_or_collection, session=None): + def drop_collection(self, name_or_collection, **options): return self._database[self.active_table].drop( - name_or_collection, - session=session + name_or_collection, **options ) @check_active_table @auto_reconnect - def delete_one(filter, collation=None, session=None): - return self._database[self.active_table].delete_one( - filter, - collation=collation, - session=session - ) + def delete_one(self, filter, collation=None, **options): + options["collation"] = collation + return self._database[self.active_table].delete_one(filter, **options) @check_active_table @auto_reconnect - def delete_many(filter, collation=None, session=None): - return self._database[self.active_table].delete_many( - filter, - collation=collation, - session=session - ) + def delete_many(self, filter, collation=None, **options): + options["collation"] = collation + return self._database[self.active_table].delete_many(filter, **options) From ac290711b906379c5c6839b28f13de37baee7870 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:47:45 +0200 Subject: [PATCH 18/65] sort events by date there were stored --- pype/ftrack/ftrack_server/session_processor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index d51da28938..fd797cfb3a 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -6,6 +6,7 @@ import threading import time import requests import queue +import pymongo import ftrack_api import ftrack_api.session @@ -74,7 +75,12 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): break def load_events(self): - not_processed_events = self.dbcon.find({"pype_data.is_processed": False}) + not_processed_events = self.dbcon.find( + {"pype_data.is_processed": False} + ).sort( + [("pype_data.stored", pymongo.ASCENDING)] + ) + found = False for event_data in not_processed_events: new_event_data = { From aad641f233e76791626bb38979f30c5b65d6ef62 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:48:13 +0200 Subject: [PATCH 19/65] import fixes --- pype/ftrack/ftrack_server/__init__.py | 7 ------- pype/ftrack/ftrack_server/socket_thread.py | 7 ------- pype/ftrack/ftrack_server/sub_event_processor.py | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pype/ftrack/ftrack_server/__init__.py b/pype/ftrack/ftrack_server/__init__.py index f8f876bbb6..0861a1bc08 100644 --- a/pype/ftrack/ftrack_server/__init__.py +++ b/pype/ftrack/ftrack_server/__init__.py @@ -1,8 +1 @@ from .ftrack_server import FtrackServer - -import event_server_cli -import session_processor -import session_storer -import socket_thread -import sub_event_processor -import sub_event_storer diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index e787004afc..8d700d7255 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -25,13 +25,6 @@ class SocketThread(threading.Thread): def stop(self): self._is_running = False - def process_to_die(self): - if not self.connection: - self.stop() - return - - self.connection.sendall(b"ptd") - def run(self): self._is_running = True time_socket = time.time() diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index b53c294981..a8f2dc5041 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -6,7 +6,7 @@ import socket import pymongo from ftrack_server import FtrackServer -import session_processor +from pype.ftrack.ftrack_server.session_processor import ProcessSession from pypeapp import Logger log = Logger().get_logger("Event processor") @@ -24,7 +24,7 @@ def main(args): sock.sendall(b"CreatedProcess") try: - session = session_processor.ProcessSession(auto_connect_event_hub=True, sock=sock) + session = ProcessSession(auto_connect_event_hub=True, sock=sock) server = FtrackServer('event') log.debug("Launched Ftrack Event processor") server.run_server(session) From ee8f09cf4635bda3c1a7d1ffafe34e0fa332fb71 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:52:07 +0200 Subject: [PATCH 20/65] added error logs --- pype/ftrack/ftrack_server/session_processor.py | 9 +++++++++ pype/ftrack/ftrack_server/sub_event_processor.py | 2 +- pype/ftrack/ftrack_server/sub_event_storer.py | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index fd797cfb3a..dbad705261 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -17,6 +17,9 @@ import ftrack_api.event from ftrack_api.logging import LazyLogMessage as L from pype.ftrack.lib.custom_db_connector import DbConnector +from pypeapp import Logger + +log = Logger().get_logger("Session processor") class ProcessEventHub(ftrack_api.event.hub.EventHub): @@ -39,6 +42,9 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): self.dbcon.active_table = self.table_name self.is_table_created = True except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) def wait(self, duration=None): @@ -65,6 +71,9 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): {"$set": {"pype_data.is_processed": True}} ) except pymongo.errors.AutoReconnect: + log.error(( + "Mongo server \"{}\" is not responding, exiting." + ).format(os.environ["AVALON_MONGO"])) sys.exit(0) # Additional special processing of events. if event['topic'] == 'ftrack.meta.disconnected': diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index a8f2dc5041..eefcfad6ab 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -44,4 +44,4 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - main(sys.argv) + return main(sys.argv) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index 918e1e1bea..dca9ae55e2 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -28,6 +28,9 @@ def install_db(): dbcon.create_table(table_name, capped=False) dbcon.active_table = table_name except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) def launch(event): @@ -48,6 +51,9 @@ def launch(event): log.debug("Event: {} stored".format(event_id)) except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) except Exception as exc: From 565c43971c1108065c1978c5f101c29b7076a876 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 17:33:32 +0200 Subject: [PATCH 21/65] added sigkill signal --- pype/ftrack/ftrack_server/event_server_cli.py | 2 ++ pype/ftrack/ftrack_server/sub_event_processor.py | 4 +++- pype/ftrack/ftrack_server/sub_event_storer.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 83ecd0d47a..2e9d95467e 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -368,5 +368,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index eefcfad6ab..20e72beb7c 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -43,5 +43,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) - return main(sys.argv) + main(sys.argv) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index dca9ae55e2..dc6fba293b 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -102,5 +102,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) main(sys.argv) From bfa61e2e986ae621547df2e6e2d7e8e5af85c4ee Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 17:34:11 +0200 Subject: [PATCH 22/65] kill threads with subprocesses on exit --- pype/ftrack/ftrack_server/event_server_cli.py | 13 +++++++++++++ pype/ftrack/ftrack_server/sub_event_processor.py | 2 +- pype/ftrack/ftrack_server/sub_event_storer.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 2e9d95467e..81357da2fa 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -3,6 +3,7 @@ import sys import signal import socket import argparse +import atexit import time from urllib.parse import urlparse @@ -153,6 +154,18 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_ftrack_error = False printed_mongo_error = False + def on_exit(): + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + atexit.register(on_exit) # Main loop while True: # Check if accessible Ftrack and Mongo url diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index 20e72beb7c..46d2b861fa 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -46,4 +46,4 @@ if __name__ == "__main__": if hasattr(signal, "SIGKILL"): signal.signal(signal.SIGKILL, signal_handler) - main(sys.argv) + sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index dc6fba293b..da67bc38eb 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -105,4 +105,4 @@ if __name__ == "__main__": if hasattr(signal, "SIGKILL"): signal.signal(signal.SIGKILL, signal_handler) - main(sys.argv) + sys.exit(main(sys.argv)) From 11f4451af1382d6be5c6110bf9ea42e97a4c7b54 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:02:53 +0200 Subject: [PATCH 23/65] added basic docstrings --- pype/ftrack/ftrack_server/event_server_cli.py | 12 ++++++++++++ pype/ftrack/ftrack_server/session_processor.py | 14 ++++++-------- pype/ftrack/ftrack_server/session_storer.py | 2 +- pype/ftrack/ftrack_server/socket_thread.py | 1 + 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 81357da2fa..815af50c5a 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -15,6 +15,7 @@ import socket_thread def check_ftrack_url(url, log_errors=True): + """Checks if Ftrack server is responding""" if not url: print('ERROR: Ftrack URL is not set!') return None @@ -44,6 +45,7 @@ def check_ftrack_url(url, log_errors=True): def check_mongo_url(host, port, log_error=False): + """Checks if mongo server is responding""" sock = None try: sock = socket.create_connection( @@ -114,6 +116,15 @@ def process_event_paths(event_paths): def main_loop(ftrack_url, username, api_key, event_paths): + """ This is main loop of event handling. + + Loop is handling threads which handles subprocesses of event storer and + processor. When one of threads is stopped it is tested to connect to + ftrack and mongo server. Threads are not started when ftrack or mongo + server is not accessible. When threads are started it is checked for socket + signals as heartbeat. Heartbeat must become at least once per 30sec + otherwise thread will be killed. + """ # Set Ftrack environments os.environ["FTRACK_SERVER"] = ftrack_url os.environ["FTRACK_API_USER"] = username @@ -154,6 +165,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_ftrack_error = False printed_mongo_error = False + # stop threads on exit def on_exit(): if processor_thread is not None: processor_thread.stop() diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index dbad705261..2dfa94dc94 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -48,13 +48,11 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): sys.exit(0) def wait(self, duration=None): - '''Wait for events and handle as they arrive. + """Overriden wait - If *duration* is specified, then only process events until duration is - reached. *duration* is in seconds though float values can be used for - smaller values. - - ''' + Event are loaded from Mongo DB when queue is empty. Handled event is + set as processed in Mongo DB. + """ started = time.time() self.prepare_dbcon() while True: @@ -84,6 +82,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): break def load_events(self): + """Load not processed events sorted by stored date""" not_processed_events = self.dbcon.find( {"pype_data.is_processed": False} ).sort( @@ -110,8 +109,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): return found def _handle_packet(self, code, packet_identifier, path, data): - '''Handle packet received from server.''' - # if self.is_waiting: + """Override `_handle_packet` which skip events and extend heartbeat""" code_name = self._code_name_mapping[code] if code_name == "event": return diff --git a/pype/ftrack/ftrack_server/session_storer.py b/pype/ftrack/ftrack_server/session_storer.py index 01b1228d55..b3201c9e4d 100644 --- a/pype/ftrack/ftrack_server/session_storer.py +++ b/pype/ftrack/ftrack_server/session_storer.py @@ -20,7 +20,7 @@ class StorerEventHub(ftrack_api.event.hub.EventHub): super(StorerEventHub, self).__init__(*args, **kwargs) def _handle_packet(self, code, packet_identifier, path, data): - '''Handle packet received from server.''' + """Override `_handle_packet` which extend heartbeat""" if self._code_name_mapping[code] == "heartbeat": # Reply with heartbeat. self.sock.sendall(b"storer") diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index 8d700d7255..ba6e0ec89b 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -9,6 +9,7 @@ from pypeapp import Logger class SocketThread(threading.Thread): MAX_TIMEOUT = 30 + """Thread that checks suprocess of storer of processor of events""" def __init__(self, name, port, filepath): super(SocketThread, self).__init__() self.log = Logger().get_logger("SocketThread", "Event Thread") From 7c59df5ec573ea47f1e0e4fca8011a7670de4c99 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:03:29 +0200 Subject: [PATCH 24/65] max timeout of heartbeat increased to 35 --- pype/ftrack/ftrack_server/socket_thread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index ba6e0ec89b..7b7d9cd852 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -8,8 +8,8 @@ from pypeapp import Logger class SocketThread(threading.Thread): - MAX_TIMEOUT = 30 """Thread that checks suprocess of storer of processor of events""" + MAX_TIMEOUT = 35 def __init__(self, name, port, filepath): super(SocketThread, self).__init__() self.log = Logger().get_logger("SocketThread", "Event Thread") From 3a5525edf68f2b4062e9bb680a36b6d435b84b62 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:45:41 +0200 Subject: [PATCH 25/65] processed events older than 3 days are removed from mongo db --- pype/ftrack/ftrack_server/session_processor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index 2dfa94dc94..686e89dc79 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -1,6 +1,7 @@ import logging import os import atexit +import datetime import tempfile import threading import time @@ -83,6 +84,12 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): def load_events(self): """Load not processed events sorted by stored date""" + ago_date = datetime.datetime.now() - datetime.timedelta(days=3) + result = self.dbcon.delete_many({ + "pype_data.stored": {"$lte": ago_date}, + "pype_data.is_processed": True + }) + not_processed_events = self.dbcon.find( {"pype_data.is_processed": False} ).sort( From 83367047ab0f92d33f3a3149d6d1644b6b7a3337 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 25 Oct 2019 13:51:11 +0200 Subject: [PATCH 26/65] added global validator for ftrack attributes --- .../validate_custom_ftrack_attributes.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 pype/plugins/global/publish/validate_custom_ftrack_attributes.py diff --git a/pype/plugins/global/publish/validate_custom_ftrack_attributes.py b/pype/plugins/global/publish/validate_custom_ftrack_attributes.py new file mode 100644 index 0000000000..0d0eb2a7f0 --- /dev/null +++ b/pype/plugins/global/publish/validate_custom_ftrack_attributes.py @@ -0,0 +1,190 @@ +import pyblish.api +import pype.api + + +class ValidateFtrackAttributes(pyblish.api.InstancePlugin): + """ + This will validate attributes in ftrack against data in scene. + + Attributes to be validated are specified in: + + `$PYPE_CONFIG/presets//ftrack_attributes.json` + + This is array (list) of checks in format: + [ + [, , ] + ] + + Where is name of ftrack attribute, is one of: + + "is", is_not", "greater_than", "less_than", "contains", "not_contains", + "starts_with", "ends_with" + + is python code that is evaluated by validator. This allows + you to fetch whatever value in scene you want, for example in Maya: + + [ + "fps", "is", + "from maya import mel; out = mel.eval('currentTimeUnitToFPS()')" + ] + + will test if ftrack fps attribute on current Task parent is same as fps + info we get from maya. Store the value you need to compare in + variable `out` in your expression. + """ + + label = "Validate Custom Ftrack Attributes" + order = pype.api.ValidateContentsOrder + families = ["ftrack"] + optional = True + + def process(self, instance): + context = instance.context + task = context.data.get('ftrackTask', False) + if not task: + self._raise(AttributeError, + "Missing FTrack Task entity in context") + + host = pyblish.api.current_host() + to_check = context.data["presets"][host].get("ftrack_attributes") + if not to_check: + self.log.warning("ftrack_attributes preset not found") + return + + self.log.info("getting attributes from ftrack ...") + # get parent of task + custom_attributes = {} + try: + parent = task["parent"] + custom_attributes = parent["custom_attributes"].items() + except KeyError: + self._raise(KeyError, "missing `parent` or `attributes`") + + custom_attributes = dict(custom_attributes) + + # get list of hierarchical attributes from ftrack + session = context.data["ftrackSession"] + + custom_hier_attributes = self._get_custom_hier_attrs(session) + custom_attributes = {} + _nonhier = {} + custom_hier_attributes = {k: None for k in custom_hier_attributes} + + for key, value in dict(parent["custom_attributes"]).items(): + if key in custom_hier_attributes: + custom_hier_attributes[key] = value + else: + _nonhier[key] = value + + custom_hier_values = self._get_hierarchical_values( + custom_hier_attributes, parent) + + custom_hier_values.update(_nonhier) + + errors = [] + attribs = custom_hier_values + for check in to_check: + ev = {} + # WARNING(Ondrej Samohel): This is really not secure as we are + # basically executing user code. But there's no other way to make + # it flexible enough for users to get stuff from + exec(str(check[2]), {}, ev) + if not ev.get("out"): + errors.append("{} code doesn't return 'out': '{}'".format( + check[0], check[2])) + continue + if check[0] in attribs: + if check[1] == "is": + if attribs[check[0]] != ev["out"]: + errors.append("{}: {} is not {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "is_not": + if attribs[check[0]] == ev["out"]: + errors.append("{}: {} is {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "less_than": + if attribs[check[0]] < ev["out"]: + errors.append("{}: {} is greater {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "greater_than": + if attribs[check[0]] < ev["out"]: + errors.append("{}: {} is less {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "contains": + if attribs[check[0]] in ev["out"]: + errors.append("{}: {} does not contain {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "not_contains": + if attribs[check[0]] not in ev["out"]: + errors.append("{}: {} contains {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "starts_with": + if attribs[check[0]].startswith(ev["out"]): + errors.append("{}: {} does not starts with {}".format( + check[0], attribs[check[0]], ev["out"])) + elif check[1] == "ends_with": + if attribs[check[0]].endswith(ev["out"]): + errors.append("{}: {} does not end with {}".format( + check[0], attribs[check[0]], ev["out"])) + + if errors: + self.log.error('There are invalid values for attributes:') + for e in errors: + self.log.error(e) + raise ValueError("ftrack attributes doesn't match") + + def _get_custom_hier_attrs(self, session): + hier_custom_attributes = [] + cust_attrs_query = ( + "select id, entity_type, object_type_id, is_hierarchical" + " from CustomAttributeConfiguration" + ) + all_avalon_attr = session.query(cust_attrs_query).all() + for cust_attr in all_avalon_attr: + if cust_attr["is_hierarchical"]: + hier_custom_attributes.append(cust_attr["key"]) + + return hier_custom_attributes + + def _get_hierarchical_values(self, keys_dict, entity): + # check values already set + _set_keys = [] + for key, value in keys_dict.items(): + if value is not None: + _set_keys.append(key) + + # pop set values from keys_dict + set_keys = {} + for key in _set_keys: + set_keys[key] = keys_dict.pop(key) + + # find if entity has set values and pop them out + keys_to_pop = [] + for key in keys_dict.keys(): + _val = entity["custom_attributes"][key] + if _val: + keys_to_pop.append(key) + set_keys[key] = _val + + for key in keys_to_pop: + keys_dict.pop(key) + + # if there are not keys to find value return found + if not keys_dict: + return set_keys + + # end recursion if entity is project + if entity.entity_type.lower() == "project": + for key, value in keys_dict.items(): + set_keys[key] = value + + else: + result = self._get_hierarchical_values(keys_dict, entity["parent"]) + for key, value in result.items(): + set_keys[key] = value + + return set_keys + + def _raise(self, exc, msg): + self.log.error(msg) + raise exc(msg) From e7c9a61b2e4e3e368f17bd92466c268e1fa96cab Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:21:44 +0200 Subject: [PATCH 27/65] added lib to ftrack_server with functios able to handle mongo url parts --- pype/ftrack/__init__.py | 2 +- pype/ftrack/ftrack_server/lib.py | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 pype/ftrack/ftrack_server/lib.py diff --git a/pype/ftrack/__init__.py b/pype/ftrack/__init__.py index bf18979e91..45ca8384b5 100644 --- a/pype/ftrack/__init__.py +++ b/pype/ftrack/__init__.py @@ -1,2 +1,2 @@ from .lib import * -from .ftrack_server import * +from .ftrack_server import FtrackServer diff --git a/pype/ftrack/ftrack_server/lib.py b/pype/ftrack/ftrack_server/lib.py new file mode 100644 index 0000000000..12159693fe --- /dev/null +++ b/pype/ftrack/ftrack_server/lib.py @@ -0,0 +1,68 @@ +import os +try: + from urllib.parse import urlparse, parse_qs +except ImportError: + from urlparse import urlparse, parse_qs + + +def ftrack_events_mongo_settings(): + host = None + port = None + username = None + password = None + collection = None + database = None + auth_db = "" + + if os.environ.get('FTRACK_EVENTS_MONGO_URL'): + result = urlparse(os.environ['FTRACK_EVENTS_MONGO_URL']) + + host = result.hostname + try: + port = result.port + except ValueError: + raise RuntimeError("invalid port specified") + username = result.username + password = result.password + try: + database = result.path.lstrip("/").split("/")[0] + collection = result.path.lstrip("/").split("/")[1] + except IndexError: + if not database: + raise RuntimeError("missing database name for logging") + try: + auth_db = parse_qs(result.query)['authSource'][0] + except KeyError: + # no auth db provided, mongo will use the one we are connecting to + pass + else: + host = os.environ.get('FTRACK_EVENTS_MONGO_HOST') + port = int(os.environ.get('FTRACK_EVENTS_MONGO_PORT', "0")) + database = os.environ.get('FTRACK_EVENTS_MONGO_DB') + username = os.environ.get('FTRACK_EVENTS_MONGO_USER') + password = os.environ.get('FTRACK_EVENTS_MONGO_PASSWORD') + collection = os.environ.get('FTRACK_EVENTS_MONGO_COL') + auth_db = os.environ.get('FTRACK_EVENTS_MONGO_AUTH_DB', 'avalon') + + return host, port, database, username, password, collection, auth_db + + +def get_ftrack_event_mongo_info(): + host, port, database, username, password, collection, auth_db = ftrack_events_mongo_settings() + user_pass = "" + if username and password: + user_pass = "{}:{}@".format(username, password) + + socket_path = "{}:{}".format(host, port) + + dab = "" + if database: + dab = "/{}".format(database) + + auth = "" + if auth_db: + auth = "?authSource={}".format(auth_db) + + url = "mongodb://{}{}{}{}".format(user_pass, socket_path, dab, auth) + + return url, database, collection From 24175a258766959de53dcc6fc2ffbee0f964847d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:22:28 +0200 Subject: [PATCH 28/65] atexit register has args now --- pype/ftrack/ftrack_server/event_server_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 815af50c5a..6796399ad8 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -166,7 +166,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_mongo_error = False # stop threads on exit - def on_exit(): + def on_exit(processor_thread, storer_thread): if processor_thread is not None: processor_thread.stop() processor_thread.join() @@ -177,7 +177,9 @@ def main_loop(ftrack_url, username, api_key, event_paths): storer_thread.join() storer_thread = None - atexit.register(on_exit) + atexit.register( + on_exit, processor_thread=processor_thread, storer_thread=storer_thread + ) # Main loop while True: # Check if accessible Ftrack and Mongo url From 4313cc5667bbb79195fdea32c7b94624c27eb961 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:23:52 +0200 Subject: [PATCH 29/65] mongo permissions error is sent with socket message --- .../ftrack/ftrack_server/session_processor.py | 27 ++++++++++++------- pype/ftrack/ftrack_server/sub_event_storer.py | 20 ++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index 686e89dc79..2f5818aab4 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -18,36 +18,45 @@ import ftrack_api.event from ftrack_api.logging import LazyLogMessage as L from pype.ftrack.lib.custom_db_connector import DbConnector +from pype.ftrack.ftrack_server.lib import get_ftrack_event_mongo_info from pypeapp import Logger log = Logger().get_logger("Session processor") class ProcessEventHub(ftrack_api.event.hub.EventHub): - dbcon = DbConnector( - mongo_url=os.environ["AVALON_MONGO"], - database_name="pype" - ) - table_name = "ftrack_events" + url, database, table_name = get_ftrack_event_mongo_info() + is_table_created = False def __init__(self, *args, **kwargs): + self.dbcon = DbConnector( + mongo_url=self.url, + database_name=self.database, + table_name=self.table_name + ) self.sock = kwargs.pop("sock") super(ProcessEventHub, self).__init__(*args, **kwargs) def prepare_dbcon(self): try: self.dbcon.install() - if not self.is_table_created: - self.dbcon.create_table(self.table_name, capped=False) - self.dbcon.active_table = self.table_name - self.is_table_created = True + dbcon._database.collection_names() except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( os.environ["AVALON_MONGO"] )) sys.exit(0) + except pymongo.errors.OperationFailure: + log.error(( + "Error with Mongo access, probably permissions." + "Check if exist database with name \"{}\"" + " and collection \"{}\" inside." + ).format(self.database, self.table_name)) + self.sock.sendall(b"MongoError") + sys.exit(0) + def wait(self, duration=None): """Overriden wait diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index da67bc38eb..6e30fb99e2 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -6,18 +6,20 @@ import socket import pymongo from ftrack_server import FtrackServer +from pype.ftrack.ftrack_server.lib import get_ftrack_event_mongo_info from pype.ftrack.lib.custom_db_connector import DbConnector from session_storer import StorerSession from pypeapp import Logger log = Logger().get_logger("Event storer") +url, database, table_name = get_ftrack_event_mongo_info() dbcon = DbConnector( - mongo_url=os.environ["AVALON_MONGO"], - database_name="pype" + mongo_url=url, + database_name=database, + table_name=table_name ) -table_name = "ftrack_events" # ignore_topics = ["ftrack.meta.connected"] ignore_topics = [] @@ -25,14 +27,14 @@ ignore_topics = [] def install_db(): try: dbcon.install() - dbcon.create_table(table_name, capped=False) - dbcon.active_table = table_name + dbcon._database.collection_names() except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( os.environ["AVALON_MONGO"] )) sys.exit(0) + def launch(event): if event.get("topic") in ignore_topics: return @@ -88,6 +90,14 @@ def main(args): log.debug("Launched Ftrack Event storer") server.run_server(session, load_files=False) + except pymongo.errors.OperationFailure: + log.error(( + "Error with Mongo access, probably permissions." + "Check if exist database with name \"{}\"" + " and collection \"{}\" inside." + ).format(database, table_name)) + sock.sendall(b"MongoError") + finally: log.debug("First closing socket") sock.close() From e0f7752205faef93ca376d0cf8f1b3966c7c91d0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:24:12 +0200 Subject: [PATCH 30/65] sub event processor print traceback --- pype/ftrack/ftrack_server/sub_event_processor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index 46d2b861fa..9444fe3ff0 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -29,6 +29,10 @@ def main(args): log.debug("Launched Ftrack Event processor") server.run_server(session) + except Exception as exc: + import traceback + traceback.print_tb(exc.__traceback__) + finally: log.debug("First closing socket") sock.close() From 5d6f91d6141ded073b5109ede3bad47b9143e349 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:24:36 +0200 Subject: [PATCH 31/65] socket thread stores info if mongo error has happened during subprocess --- pype/ftrack/ftrack_server/socket_thread.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index 7b7d9cd852..d0a2868743 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -1,4 +1,5 @@ import os +import sys import time import signal import socket @@ -23,6 +24,8 @@ class SocketThread(threading.Thread): self._is_running = False self.finished = False + self.mongo_error = False + def stop(self): self._is_running = False @@ -97,6 +100,8 @@ class SocketThread(threading.Thread): break if data: + if data == b"MongoError": + self.mongo_error = True connection.sendall(data) except Exception as exc: @@ -110,4 +115,9 @@ class SocketThread(threading.Thread): if self.subproc.poll() is None: self.subproc.terminate() + lines = self.subproc.stdout.readlines() + if lines: + print("*** Socked Thread stdout ***") + for line in lines: + os.write(1, line) self.finished = True From 11f0c41e904b5c0b3fb28280c4801ed436848759 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:25:44 +0200 Subject: [PATCH 32/65] crash all server if mongo error happened --- pype/ftrack/ftrack_server/event_server_cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 6796399ad8..a016f1aaf4 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -225,6 +225,10 @@ def main_loop(ftrack_url, username, api_key, event_paths): # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): + if storer_thread.mongo_error: + raise Exception( + "Exiting because have issue with acces to MongoDB" + ) storer_thread.join() storer_thread = None ftrack_accessible = False @@ -237,7 +241,11 @@ def main_loop(ftrack_url, username, api_key, event_paths): processor_thread.start() # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): + elif not processor_thread.isAlive(): + if storer_thread.mongo_error: + raise Exception( + "Exiting because have issue with acces to MongoDB" + ) processor_thread.join() processor_thread = None ftrack_accessible = False From 38a6c84fe55ebb17a188998a431125a63b4b12ca Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:26:38 +0200 Subject: [PATCH 33/65] its used lib to get mongo port and host in event server cli --- pype/ftrack/ftrack_server/event_server_cli.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index a016f1aaf4..0698f5728f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -11,6 +11,7 @@ import requests from pype.vendor import ftrack_api from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer +from pype.ftrack.ftrack_server.lib import ftrack_events_mongo_settings import socket_thread @@ -132,18 +133,9 @@ def main_loop(ftrack_url, username, api_key, event_paths): os.environ["FTRACK_EVENTS_PATH"] = event_paths # Get mongo hostname and port for testing mongo connection - mongo_url = os.environ["AVALON_MONGO"].strip('/ ') - result = urlparse(mongo_url) - url_items = result.netloc.split("@") - mongo_url = url_items[0] - if len(url_items) == 2: - mongo_url = url_items[1] - - mongo_url = "://".join([result.scheme, mongo_url]) - result = urlparse(mongo_url) - - mongo_hostname = result.hostname - mongo_port = result.port + mongo_list = ftrack_events_mongo_settings() + mongo_hostname = mongo_list[0] + mongo_port = mongo_list[1] # Current file file_path = os.path.dirname(os.path.realpath(__file__)) From 10d695603765fe9aecf5213b189c0ede700d9de2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:27:31 +0200 Subject: [PATCH 34/65] event server will wait some time if subprocess crashed many times in row --- pype/ftrack/ftrack_server/event_server_cli.py | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 0698f5728f..768a4425bb 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -1,6 +1,7 @@ import os import sys import signal +import datetime import socket import argparse import atexit @@ -140,16 +141,24 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Current file file_path = os.path.dirname(os.path.realpath(__file__)) + min_fail_seconds = 5 + max_fail_count = 3 + wait_time_after_max_fail = 10 + # Threads data storer_name = "StorerThread" storer_port = 10001 storer_path = "{}/sub_event_storer.py".format(file_path) storer_thread = None + storer_last_failed = datetime.datetime.now() + storer_failed_count = 0 processor_name = "ProcessorThread" processor_port = 10011 processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None + processor_last_failed = datetime.datetime.now() + processor_failed_count = 0 ftrack_accessible = False mongo_accessible = False @@ -210,10 +219,20 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Run backup thread which does not requeire mongo to work if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() + if storer_failed_count < max_fail_count: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + elif storer_failed_count == max_fail_count: + print(( + "Storer failed {}times I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + storer_failed_count += 1 + elif (( + datetime.datetime.now() - storer_last_failed + ).seconds > wait_time_after_max_fail): + storer_failed_count = 0 # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): @@ -226,11 +245,32 @@ def main_loop(ftrack_url, username, api_key, event_paths): ftrack_accessible = False mongo_accessible = False + _storer_last_failed = datetime.datetime.now() + delta_time = (_storer_last_failed - storer_last_failed).seconds + if delta_time < min_fail_seconds: + storer_failed_count += 1 + else: + storer_failed_count = 0 + storer_last_failed = _storer_last_failed + if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() + if processor_failed_count < max_fail_count: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + elif processor_failed_count == max_fail_count: + print(( + "Processor failed {}times in row" + " I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + processor_failed_count += 1 + + elif (( + datetime.datetime.now() - processor_last_failed + ).seconds > wait_time_after_max_fail): + processor_failed_count = 0 # If thread failed test Ftrack and Mongo connection elif not processor_thread.isAlive(): @@ -243,6 +283,17 @@ def main_loop(ftrack_url, username, api_key, event_paths): ftrack_accessible = False mongo_accessible = False + _processor_last_failed = datetime.datetime.now() + delta_time = ( + _processor_last_failed - processor_last_failed + ).seconds + + if delta_time < min_fail_seconds: + processor_failed_count += 1 + else: + processor_failed_count = 0 + processor_last_failed = _processor_last_failed + time.sleep(1) From 64e86e39881392ae80883588387ec3ac4bb18df9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 29 Oct 2019 15:41:45 +0000 Subject: [PATCH 35/65] fixed Maya set fps --- pype/maya/lib.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pype/maya/lib.py b/pype/maya/lib.py index 55a138428b..b9387b65fd 100644 --- a/pype/maya/lib.py +++ b/pype/maya/lib.py @@ -1774,6 +1774,11 @@ def set_scene_fps(fps, update=True): '48000': '48000fps'} # pull from mapping + # this should convert float string to float and int to int + # so 25.0 is converted to 25, but 23.98 will be still float. + decimals = int(str(fps-int(fps))[2:]) + if decimals == 0: + fps = int(fps) unit = fps_mapping.get(str(fps), None) if unit is None: raise ValueError("Unsupported FPS value: `%s`" % fps) From 75ce8f1196d280aab1980ed6c97f52092b8c1eda Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 17:58:29 +0100 Subject: [PATCH 36/65] added subprocess file for oldway event server --- pype/ftrack/ftrack_server/sub_old_way.py | 100 +++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 pype/ftrack/ftrack_server/sub_old_way.py diff --git a/pype/ftrack/ftrack_server/sub_old_way.py b/pype/ftrack/ftrack_server/sub_old_way.py new file mode 100644 index 0000000000..92e7c0cf8c --- /dev/null +++ b/pype/ftrack/ftrack_server/sub_old_way.py @@ -0,0 +1,100 @@ +import os +import sys +import time +import datetime +import signal +import threading + +from ftrack_server import FtrackServer +from pype.vendor import ftrack_api +from pype.vendor.ftrack_api.event.hub import EventHub +from pypeapp import Logger + +log = Logger().get_logger("Event Server Old") + + +class TimerChecker(threading.Thread): + max_time_out = 35 + + def __init__(self, server, session): + self.server = server + self.session = session + self.is_running = False + self.failed = False + super().__init__() + + def stop(self): + self.is_running = False + + def run(self): + start = datetime.datetime.now() + self.is_running = True + connected = False + + while True: + if not self.is_running: + break + + if not self.session.event_hub.connected: + if not connected: + if (datetime.datetime.now() - start).seconds > self.max_time_out: + log.error(( + "Exiting event server. Session was not connected" + " to ftrack server in {} seconds." + ).format(self.max_time_out)) + self.failed = True + break + else: + log.error( + "Exiting event server. Event Hub is not connected." + ) + self.server.stop_session() + self.failed = True + break + else: + if not connected: + connected = True + + time.sleep(1) + + +def main(args): + check_thread = None + try: + server = FtrackServer('event') + session = ftrack_api.Session(auto_connect_event_hub=True) + + check_thread = TimerChecker(server, session) + check_thread.start() + + log.debug("Launching Ftrack Event Old Way Server") + server.run_server(session) + + except Exception as exc: + import traceback + traceback.print_tb(exc.__traceback__) + + finally: + log_info = True + if check_thread is not None: + check_thread.stop() + check_thread.join() + if check_thread.failed: + log_info = False + if log_info: + log.info("Exiting Event server subprocess") + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) + + sys.exit(main(sys.argv)) From 5b70ddf32f9d0c836eb99337fcf6fcac42b4ac9d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 17:59:36 +0100 Subject: [PATCH 37/65] added `oldway` argument to event server cli --- pype/ftrack/ftrack_server/event_server_cli.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 768a4425bb..ade80117af 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -127,11 +127,6 @@ def main_loop(ftrack_url, username, api_key, event_paths): signals as heartbeat. Heartbeat must become at least once per 30sec otherwise thread will be killed. """ - # Set Ftrack environments - os.environ["FTRACK_SERVER"] = ftrack_url - os.environ["FTRACK_API_USER"] = username - os.environ["FTRACK_API_KEY"] = api_key - os.environ["FTRACK_EVENTS_PATH"] = event_paths # Get mongo hostname and port for testing mongo connection mongo_list = ftrack_events_mongo_settings() @@ -385,7 +380,11 @@ def main(argv): help="Load creadentials from apps dir", action="store_true" ) - + parser.add_argument( + '-oldway', + help="Load creadentials from apps dir", + action="store_true" + ) ftrack_url = os.environ.get('FTRACK_SERVER') username = os.environ.get('FTRACK_API_USER') api_key = os.environ.get('FTRACK_API_KEY') @@ -410,6 +409,7 @@ def main(argv): if kwargs.ftrackapikey: api_key = kwargs.ftrackapikey + oldway = kwargs.oldway # Check url regex and accessibility ftrack_url = check_ftrack_url(ftrack_url) if not ftrack_url: @@ -435,7 +435,16 @@ def main(argv): if kwargs.storecred: credentials._save_credentials(username, api_key, True) - main_loop(ftrack_url, username, api_key, event_paths) + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + if oldway: + return old_way_server(ftrack_url) + + return main_loop(ftrack_url) if __name__ == "__main__": From 92ccd406c57b13dca335fb73e53b34cf00048ea2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 18:00:20 +0100 Subject: [PATCH 38/65] created exception for mongo permission error --- pype/ftrack/ftrack_server/event_server_cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index ade80117af..b0dfb48a1f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -16,6 +16,14 @@ from pype.ftrack.ftrack_server.lib import ftrack_events_mongo_settings import socket_thread +class MongoPermissionsError(Exception): + """Is used when is created multiple objects of same RestApi class.""" + def __init__(self, message=None): + if not message: + message = "Exiting because have issue with acces to MongoDB" + super().__init__(message) + + def check_ftrack_url(url, log_errors=True): """Checks if Ftrack server is responding""" if not url: @@ -162,6 +170,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_mongo_error = False # stop threads on exit + # TODO check if works and args have thread objects! def on_exit(processor_thread, storer_thread): if processor_thread is not None: processor_thread.stop() @@ -232,9 +241,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): if storer_thread.mongo_error: - raise Exception( - "Exiting because have issue with acces to MongoDB" - ) + raise MongoPermissionsError() storer_thread.join() storer_thread = None ftrack_accessible = False From 6f93743c8df32c81c1dad71dfdc747882434c55a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 18:01:02 +0100 Subject: [PATCH 39/65] added old_way_server which can handle same way as was before and restart event server when ftrack connection fails --- pype/ftrack/ftrack_server/event_server_cli.py | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index b0dfb48a1f..2106da3b5f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -2,6 +2,7 @@ import os import sys import signal import datetime +import subprocess import socket import argparse import atexit @@ -125,7 +126,77 @@ def process_event_paths(event_paths): return os.pathsep.join(return_paths), not_found -def main_loop(ftrack_url, username, api_key, event_paths): +def old_way_server(ftrack_url): + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + min_fail_seconds = 5 + max_fail_count = 3 + wait_time_after_max_fail = 10 + + subproc = None + subproc_path = "{}/sub_old_way.py".format(file_path) + subproc_last_failed = datetime.datetime.now() + subproc_failed_count = 0 + + ftrack_accessible = False + printed_ftrack_error = False + + while True: + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {} <{}>".format( + ftrack_url, str(datetime.datetime.now()) + )) + if subproc is not None: + if subproc.poll() is None: + subproc.terminate() + + subproc = None + + printed_ftrack_error = True + + time.sleep(1) + continue + + printed_ftrack_error = False + + if subproc is None: + if subproc_failed_count < max_fail_count: + subproc = subprocess.Popen( + ["python", subproc_path], + stdout=subprocess.PIPE + ) + elif subproc_failed_count == max_fail_count: + print(( + "Storer failed {}times I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + subproc_failed_count += 1 + elif (( + datetime.datetime.now() - subproc_last_failed + ).seconds > wait_time_after_max_fail): + subproc_failed_count = 0 + + # If thread failed test Ftrack and Mongo connection + elif subproc.poll() is not None: + subproc = None + ftrack_accessible = False + + _subproc_last_failed = datetime.datetime.now() + delta_time = (_subproc_last_failed - subproc_last_failed).seconds + if delta_time < min_fail_seconds: + subproc_failed_count += 1 + else: + subproc_failed_count = 0 + subproc_last_failed = _subproc_last_failed + + time.sleep(1) + + +def main_loop(ftrack_url): """ This is main loop of event handling. Loop is handling threads which handles subprocesses of event storer and From ff44a00354acfc2644ba9dc4857c5a8c2a0f2da0 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 29 Oct 2019 20:57:41 +0100 Subject: [PATCH 40/65] hotfix: ignore shapes when referencing and preserve references when importing --- pype/plugins/maya/load/actions.py | 1 + pype/plugins/maya/load/load_reference.py | 16 +++++++++++----- pype/plugins/maya/load/load_rig.py | 10 ++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pype/plugins/maya/load/actions.py b/pype/plugins/maya/load/actions.py index 60316aaf9b..9f6a5c4d34 100644 --- a/pype/plugins/maya/load/actions.py +++ b/pype/plugins/maya/load/actions.py @@ -116,6 +116,7 @@ class ImportMayaLoader(api.Loader): with maya.maintained_selection(): cmds.file(self.fname, i=True, + preserveReferences=True, namespace=namespace, returnNewNodes=True, groupReference=True, diff --git a/pype/plugins/maya/load/load_reference.py b/pype/plugins/maya/load/load_reference.py index f855cb55f9..a754c3be98 100644 --- a/pype/plugins/maya/load/load_reference.py +++ b/pype/plugins/maya/load/load_reference.py @@ -4,7 +4,8 @@ import os from pypeapp import config import pymel.core as pm reload(config) - +import pype.maya.plugin +reload(pype.maya.plugin) class ReferenceLoader(pype.maya.plugin.ReferenceLoader): """Load the model""" @@ -42,11 +43,17 @@ class ReferenceLoader(pype.maya.plugin.ReferenceLoader): namespace = cmds.referenceQuery(nodes[0], namespace=True) + shapes = cmds.ls(nodes, shapes=True, long=True) + print(shapes) + + newNodes = (list(set(nodes) - set(shapes))) + print(newNodes) + groupNode = pm.PyNode(groupName) roots = set() print(nodes) - for node in nodes: + for node in newNodes: try: roots.add(pm.PyNode(node).getAllParents()[-2]) except: @@ -59,7 +66,6 @@ class ReferenceLoader(pype.maya.plugin.ReferenceLoader): root.setParent(groupNode) cmds.setAttr(groupName + ".displayHandle", 1) - groupNode presets = config.get_presets(project=os.environ['AVALON_PROJECT']) colors = presets['plugins']['maya']['load']['colors'] @@ -68,7 +74,7 @@ class ReferenceLoader(pype.maya.plugin.ReferenceLoader): groupNode.useOutlinerColor.set(1) groupNode.outlinerColor.set(c[0], c[1], c[2]) - self[:] = nodes + self[:] = newNodes cmds.setAttr(groupName + ".displayHandle", 1) # get bounding box @@ -88,7 +94,7 @@ class ReferenceLoader(pype.maya.plugin.ReferenceLoader): cmds.setAttr(groupName + ".selectHandleY", cy) cmds.setAttr(groupName + ".selectHandleZ", cz) - return nodes + return newNodes def switch(self, container, representation): self.update(container, representation) diff --git a/pype/plugins/maya/load/load_rig.py b/pype/plugins/maya/load/load_rig.py index 9358d941db..fc6e666ac6 100644 --- a/pype/plugins/maya/load/load_rig.py +++ b/pype/plugins/maya/load/load_rig.py @@ -47,12 +47,18 @@ class RigLoader(pype.maya.plugin.ReferenceLoader): cmds.setAttr(groupName + ".outlinerColor", c[0], c[1], c[2]) + shapes = cmds.ls(nodes, shapes=True, long=True) + print(shapes) + + newNodes = (list(set(nodes) - set(shapes))) + print(newNodes) + # Store for post-process - self[:] = nodes + self[:] = newNodes if data.get("post_process", True): self._post_process(name, namespace, context, data) - return nodes + return newNodes def _post_process(self, name, namespace, context, data): From 744a606d69001251b386442ca573e5b9ac39bcdc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 30 Oct 2019 10:44:19 +0100 Subject: [PATCH 41/65] added try except of Attribute error to catch not existing thread error --- pype/services/idle_manager/idle_manager.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pype/services/idle_manager/idle_manager.py b/pype/services/idle_manager/idle_manager.py index 57b827a37e..0f7c7bb7ac 100644 --- a/pype/services/idle_manager/idle_manager.py +++ b/pype/services/idle_manager/idle_manager.py @@ -70,12 +70,22 @@ class IdleManager(QtCore.QThread): if self.qaction and self.failed_icon: self.qaction.setIcon(self.failed_icon) - thread_mouse.signal_stop.emit() - thread_mouse.terminate() - thread_mouse.wait() - thread_keyboard.signal_stop.emit() - thread_keyboard.terminate() - thread_keyboard.wait() + + # Threads don't have their attrs when Qt application already finished + try: + thread_mouse.signal_stop.emit() + thread_mouse.terminate() + thread_mouse.wait() + except AttributeError: + pass + + try: + thread_keyboard.signal_stop.emit() + thread_keyboard.terminate() + thread_keyboard.wait() + except AttributeError: + pass + self._is_running = False self.log.info('IdleManager has stopped') From 628514948909013ca2976749dc76c6a2c68cc07e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 30 Oct 2019 10:44:47 +0100 Subject: [PATCH 42/65] enhanced signal dictionary creating --- pype/services/idle_manager/idle_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pype/services/idle_manager/idle_manager.py b/pype/services/idle_manager/idle_manager.py index 0f7c7bb7ac..fd07eeec14 100644 --- a/pype/services/idle_manager/idle_manager.py +++ b/pype/services/idle_manager/idle_manager.py @@ -1,4 +1,5 @@ import time +import collections from Qt import QtCore, QtGui, QtWidgets from pype.vendor.pynput import mouse, keyboard from pypeapp import Logger @@ -9,7 +10,7 @@ class IdleManager(QtCore.QThread): Idle time resets on keyboard/mouse input. Is able to emit signals at specific time idle. """ - time_signals = {} + time_signals = collections.defaultdict(list) idle_time = 0 signal_reset_timer = QtCore.Signal() @@ -35,8 +36,6 @@ class IdleManager(QtCore.QThread): :param signal: signal that will be emitted (without objects) :type signal: QtCore.Signal """ - if emit_time not in self.time_signals: - self.time_signals[emit_time] = [] self.time_signals[emit_time].append(signal) @property From 0de66c1c457ed515e3ba408c698fa525a3111ff7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 30 Oct 2019 10:48:25 +0100 Subject: [PATCH 43/65] fixed logging of job killer action --- pype/ftrack/actions/action_job_killer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/actions/action_job_killer.py b/pype/ftrack/actions/action_job_killer.py index 717f87e879..93d877eaf7 100644 --- a/pype/ftrack/actions/action_job_killer.py +++ b/pype/ftrack/actions/action_job_killer.py @@ -101,13 +101,14 @@ class JobKiller(BaseAction): # Update all the queried jobs, setting the status to failed. for job in jobs: try: + origin_status = job["status"] job['status'] = 'failed' session.commit() self.log.debug(( 'Changing Job ({}) status: {} -> failed' - ).format(job['id'], job['status'])) + ).format(job['id'], origin_status)) except Exception: - self.log.warning.debug(( + self.log.warning(( 'Changing Job ({}) has failed' ).format(job['id'])) From dbe79c7741c35d165e8bd2b7d777ee6a9b0a0b99 Mon Sep 17 00:00:00 2001 From: Jana Mizikova Date: Wed, 30 Oct 2019 19:06:18 +0100 Subject: [PATCH 44/65] add frames to published representations --- pype/plugins/global/publish/integrate_new.py | 5 +++++ pype/plugins/maya/publish/extract_yeti_cache.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index ad7be306ca..6ace625743 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -305,6 +305,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dst_tail = dst_collection.format("{tail}") index_frame_start = None + if repre.get("frameStart"): frame_start_padding = len(str( repre.get("frameEnd"))) @@ -391,6 +392,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "representation": repre['ext'] } } + + if frame_start_padding: + representation['context']['frame'] = repre.get("frameStart") + self.log.debug("__ representation: {}".format(representation)) destination_list.append(dst) self.log.debug("__ destination_list: {}".format(destination_list)) diff --git a/pype/plugins/maya/publish/extract_yeti_cache.py b/pype/plugins/maya/publish/extract_yeti_cache.py index 8cb94021ed..7d85f396ae 100644 --- a/pype/plugins/maya/publish/extract_yeti_cache.py +++ b/pype/plugins/maya/publish/extract_yeti_cache.py @@ -73,7 +73,9 @@ class ExtractYetiCache(pype.api.Extractor): 'ext': 'fur', 'files': cache_files[0] if len(cache_files) == 1 else cache_files, 'stagingDir': dirname, - 'anatomy_template': 'publish' + 'anatomy_template': 'publish', + 'frameStart': int(start_frame), + 'frameEnd': int(end_frame) } ) From 3c4083fb999e96c61c280094e5966eb97f46cf36 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 31 Oct 2019 17:07:54 +0100 Subject: [PATCH 45/65] hotfix: yeti frames in integrator --- pype/plugins/global/publish/integrate_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 6ace625743..4c771cf0da 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -393,7 +393,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): } } - if frame_start_padding: + if repre.get("frameStart"): representation['context']['frame'] = repre.get("frameStart") self.log.debug("__ representation: {}".format(representation)) From ad41ffafe5d72ae2391b3d765cc0f90f27e88064 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 31 Oct 2019 18:15:30 +0100 Subject: [PATCH 46/65] hotfix: frames in representation --- pype/plugins/global/publish/integrate_new.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 4c771cf0da..a82ebdd5a6 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -268,7 +268,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): template = os.path.normpath( anatomy.templates[template_name]["path"]) - if isinstance(files, list): + sequence_repre = isinstance(files, list) + + if sequence_repre: src_collections, remainder = clique.assemble(files) self.log.debug( "src_tail_collections: {}".format(str(src_collections))) @@ -393,7 +395,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): } } - if repre.get("frameStart"): + if sequence_repre and repre.get("frameStart"): representation['context']['frame'] = repre.get("frameStart") self.log.debug("__ representation: {}".format(representation)) From c3cbab14b9638f416b9c17859fc9cf47d23abb47 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 31 Oct 2019 18:42:15 +0100 Subject: [PATCH 47/65] missing self --- pype/ftrack/ftrack_server/session_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index 2f5818aab4..86a9775dce 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -41,7 +41,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): def prepare_dbcon(self): try: self.dbcon.install() - dbcon._database.collection_names() + self.dbcon._database.collection_names() except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( os.environ["AVALON_MONGO"] From 384843a8b4bbe267ac8d013a2782b19b2c94cffd Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 31 Oct 2019 18:52:35 +0100 Subject: [PATCH 48/65] change `old way` to `legacy --- pype/ftrack/ftrack_server/event_server_cli.py | 12 ++++++------ .../{sub_old_way.py => sub_legacy_server.py} | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) rename pype/ftrack/ftrack_server/{sub_old_way.py => sub_legacy_server.py} (96%) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 2106da3b5f..ec7cac9d6a 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -126,7 +126,7 @@ def process_event_paths(event_paths): return os.pathsep.join(return_paths), not_found -def old_way_server(ftrack_url): +def legacy_server(ftrack_url): # Current file file_path = os.path.dirname(os.path.realpath(__file__)) @@ -135,7 +135,7 @@ def old_way_server(ftrack_url): wait_time_after_max_fail = 10 subproc = None - subproc_path = "{}/sub_old_way.py".format(file_path) + subproc_path = "{}/sub_legacy_server.py".format(file_path) subproc_last_failed = datetime.datetime.now() subproc_failed_count = 0 @@ -459,7 +459,7 @@ def main(argv): action="store_true" ) parser.add_argument( - '-oldway', + '-legacy', help="Load creadentials from apps dir", action="store_true" ) @@ -487,7 +487,7 @@ def main(argv): if kwargs.ftrackapikey: api_key = kwargs.ftrackapikey - oldway = kwargs.oldway + legacy = kwargs.legacy # Check url regex and accessibility ftrack_url = check_ftrack_url(ftrack_url) if not ftrack_url: @@ -519,8 +519,8 @@ def main(argv): os.environ["FTRACK_API_KEY"] = api_key os.environ["FTRACK_EVENTS_PATH"] = event_paths - if oldway: - return old_way_server(ftrack_url) + if legacy: + return legacy_server(ftrack_url) return main_loop(ftrack_url) diff --git a/pype/ftrack/ftrack_server/sub_old_way.py b/pype/ftrack/ftrack_server/sub_legacy_server.py similarity index 96% rename from pype/ftrack/ftrack_server/sub_old_way.py rename to pype/ftrack/ftrack_server/sub_legacy_server.py index 92e7c0cf8c..c162b1abe1 100644 --- a/pype/ftrack/ftrack_server/sub_old_way.py +++ b/pype/ftrack/ftrack_server/sub_legacy_server.py @@ -10,7 +10,7 @@ from pype.vendor import ftrack_api from pype.vendor.ftrack_api.event.hub import EventHub from pypeapp import Logger -log = Logger().get_logger("Event Server Old") +log = Logger().get_logger("Event Server Legacy") class TimerChecker(threading.Thread): @@ -67,7 +67,7 @@ def main(args): check_thread = TimerChecker(server, session) check_thread.start() - log.debug("Launching Ftrack Event Old Way Server") + log.debug("Launching Ftrack Event Legacy Server") server.run_server(session) except Exception as exc: From f217899ea131c5569f46d851be53ead036e98f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Fri, 1 Nov 2019 09:09:16 +0000 Subject: [PATCH 49/65] fixed rv action not to load when preset is missing --- pype/ftrack/actions/action_rv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/actions/action_rv.py b/pype/ftrack/actions/action_rv.py index 69c6624b71..e32997e5a9 100644 --- a/pype/ftrack/actions/action_rv.py +++ b/pype/ftrack/actions/action_rv.py @@ -15,6 +15,7 @@ log = Logger().get_logger(__name__) class RVAction(BaseAction): """ Launch RV action """ + ignore_me = "rv" not in config.get_presets() identifier = "rv.launch.action" label = "rv" description = "rv Launcher" @@ -42,8 +43,9 @@ class RVAction(BaseAction): ) else: # if not, fallback to config file location - self.config_data = config.get_presets()['rv']['config'] - self.set_rv_path() + if "rv" in config.get_presets(): + self.config_data = config.get_presets()['rv']['config'] + self.set_rv_path() if self.rv_path is None: return From d2c8810470bcf2d22fe2c51e80f52b45c24c741d Mon Sep 17 00:00:00 2001 From: Jana Mizikova Date: Fri, 1 Nov 2019 11:25:48 +0100 Subject: [PATCH 50/65] fix(plugins): ditching `silo` from assumig destination --- pype/plugins/global/publish/integrate_assumed_destination.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_assumed_destination.py b/pype/plugins/global/publish/integrate_assumed_destination.py index 3bbd4cf33b..a26529fc2c 100644 --- a/pype/plugins/global/publish/integrate_assumed_destination.py +++ b/pype/plugins/global/publish/integrate_assumed_destination.py @@ -66,7 +66,7 @@ class IntegrateAssumedDestination(pyblish.api.InstancePlugin): """Create a filepath based on the current data available Example template: - {root}/{project}/{silo}/{asset}/publish/{subset}/v{version:0>3}/ + {root}/{project}/{asset}/publish/{subset}/v{version:0>3}/ {subset}.{representation} Args: instance: the instance to publish @@ -95,7 +95,6 @@ class IntegrateAssumedDestination(pyblish.api.InstancePlugin): assert asset, ("No asset found by the name '{}' " "in project '{}'".format(asset_name, project_name)) - silo = asset['silo'] subset = io.find_one({"type": "subset", "name": subset_name, @@ -126,7 +125,6 @@ class IntegrateAssumedDestination(pyblish.api.InstancePlugin): template_data = {"root": api.Session["AVALON_PROJECTS"], "project": {"name": project_name, "code": project['data']['code']}, - "silo": silo, "family": instance.data['family'], "asset": asset_name, "subset": subset_name, From 2d65ab83f11d86d82a913a6047586dd24e3d2ebf Mon Sep 17 00:00:00 2001 From: Jana Mizikova Date: Fri, 1 Nov 2019 11:27:46 +0100 Subject: [PATCH 51/65] fix(plugins): changing the way ftrack is querying entity_type this will remove the server entity duplicity error on mysql --- .../ftrack/publish/integrate_hierarchy_ftrack.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py b/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py index 25c641c168..73a4d8af97 100644 --- a/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py +++ b/pype/plugins/ftrack/publish/integrate_hierarchy_ftrack.py @@ -44,7 +44,15 @@ class IntegrateHierarchyToFtrack(pyblish.api.ContextPlugin): input_data = context.data["hierarchyContext"] - self.import_to_ftrack(input_data) + # self.import_to_ftrack(input_data) + + try: + self.import_to_ftrack(input_data) + except Exception as exc: + import sys + import traceback + self.log.info(traceback.format_exc(sys.exc_info())) + raise Exception("failed") def import_to_ftrack(self, input_data, parent=None): for entity_name in input_data: @@ -66,9 +74,9 @@ class IntegrateHierarchyToFtrack(pyblish.api.ContextPlugin): # try to find if entity already exists else: - query = '{} where name is "{}" and parent_id is "{}"'.format( - entity_type, entity_name, parent['id'] - ) + query = 'TypedContext where name is "{0}" and project.full_name is "{1}"'.format( + entity_name, self.ft_project["full_name"] + ) try: entity = self.session.query(query).one() except Exception: From c009f661e6f389fac532a9a794a95ff3f79a92f1 Mon Sep 17 00:00:00 2001 From: Jana Mizikova Date: Fri, 1 Nov 2019 17:06:35 +0100 Subject: [PATCH 52/65] fix(nks): thumbnails, build workfile with preview mov - thumbnail for clip is taken from middle of duration --- pype/nuke/lib.py | 4 +++- pype/plugins/nukestudio/publish/collect_plates.py | 6 ++++-- pype/plugins/nukestudio/publish/collect_reviews.py | 11 ++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index f182088457..cd809bf640 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -927,7 +927,7 @@ class BuildWorkfile(WorkfileSettings): def process(self, regex_filter=None, version=None, - representations=["exr", "dpx", "lutJson"]): + representations=["exr", "dpx", "lutJson", "mov", "preview"]): """ A short description. @@ -984,6 +984,8 @@ class BuildWorkfile(WorkfileSettings): version=version, representations=representations) + log.info("__ subsets: `{}`".format(subsets)) + nodes_backdrop = list() for name, subset in subsets.items(): diff --git a/pype/plugins/nukestudio/publish/collect_plates.py b/pype/plugins/nukestudio/publish/collect_plates.py index 2ebbfde551..f9eb126772 100644 --- a/pype/plugins/nukestudio/publish/collect_plates.py +++ b/pype/plugins/nukestudio/publish/collect_plates.py @@ -196,12 +196,14 @@ class CollectPlatesData(api.InstancePlugin): thumb_file = head + ".png" thumb_path = os.path.join(staging_dir, thumb_file) + thumb_frame = instance.data["sourceIn"] + ((instance.data["sourceOut"] - instance.data["sourceIn"])/2) - thumbnail = item.thumbnail(instance.data["sourceIn"]).save( + thumbnail = item.thumbnail(thumb_frame).save( thumb_path, format='png' ) - self.log.debug("__ thumbnail: {}".format(thumbnail)) + self.log.debug("__ sourceIn: `{}`".format(instance.data["sourceIn"])) + self.log.debug("__ thumbnail: `{}`, frame: `{}`".format(thumbnail, thumb_frame)) thumb_representation = { 'files': thumb_file, diff --git a/pype/plugins/nukestudio/publish/collect_reviews.py b/pype/plugins/nukestudio/publish/collect_reviews.py index 9fab0f0741..f9032b2ca4 100644 --- a/pype/plugins/nukestudio/publish/collect_reviews.py +++ b/pype/plugins/nukestudio/publish/collect_reviews.py @@ -106,7 +106,6 @@ class CollectReviews(api.InstancePlugin): def create_thumbnail(self, instance): item = instance.data["item"] - source_in = instance.data["sourceIn"] source_path = instance.data["sourcePath"] source_file = os.path.basename(source_path) @@ -119,11 +118,17 @@ class CollectReviews(api.InstancePlugin): thumb_file = head + ".png" thumb_path = os.path.join(staging_dir, thumb_file) self.log.debug("__ thumb_path: {}".format(thumb_path)) - self.log.debug("__ source_in: {}".format(source_in)) - thumbnail = item.thumbnail(source_in).save( + + thumb_frame = instance.data["sourceIn"] + ((instance.data["sourceOut"] - instance.data["sourceIn"])/2) + + thumbnail = item.thumbnail(thumb_frame).save( thumb_path, format='png' ) + + self.log.debug("__ sourceIn: `{}`".format(instance.data["sourceIn"])) + self.log.debug("__ thumbnail: `{}`, frame: `{}`".format(thumbnail, thumb_frame)) + self.log.debug("__ thumbnail: {}".format(thumbnail)) thumb_representation = { From 66544273fd08fc22aedc4267b5716d0dbb8abd23 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 1 Nov 2019 19:06:00 +0100 Subject: [PATCH 53/65] connect shapes to loaded Yeti Rig --- pype/plugins/maya/load/load_yeti_rig.py | 52 +++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/pype/plugins/maya/load/load_yeti_rig.py b/pype/plugins/maya/load/load_yeti_rig.py index eb75ff6bdc..a3e03e8a6c 100644 --- a/pype/plugins/maya/load/load_yeti_rig.py +++ b/pype/plugins/maya/load/load_yeti_rig.py @@ -1,9 +1,17 @@ -import pype.maya.plugin import os +from collections import defaultdict + from pypeapp import config +import pype.maya.plugin +from pype.maya import lib class YetiRigLoader(pype.maya.plugin.ReferenceLoader): + """ + This loader will load Yeti rig. You can select something in scene and if it + has same ID as mesh published with rig, their shapes will be linked + together. + """ families = ["yetiRig"] representations = ["ma"] @@ -18,6 +26,32 @@ class YetiRigLoader(pype.maya.plugin.ReferenceLoader): import maya.cmds as cmds from avalon import maya + # get roots of selected hierarchies + selected_roots = [] + for sel in cmds.ls(sl=True, long=True): + selected_roots.append(sel.split("|")[1]) + + # get all objects under those roots + selected_hierarchy = [] + for root in selected_roots: + selected_hierarchy.append(cmds.listRelatives( + root, + allDescendents=True) or []) + + # flatten the list and filter only shapes + shapes_flat = [] + for root in selected_hierarchy: + shapes = cmds.ls(root, long=True, type="mesh") or [] + for shape in shapes: + shapes_flat.append(shape) + + # create dictionary of cbId and shape nodes + scene_lookup = defaultdict(list) + for node in shapes_flat: + cb_id = lib.get_id(node) + scene_lookup[cb_id] = node + + # load rig with maya.maintained_selection(): nodes = cmds.file(self.fname, namespace=namespace, @@ -26,6 +60,20 @@ class YetiRigLoader(pype.maya.plugin.ReferenceLoader): groupReference=True, groupName="{}:{}".format(namespace, name)) + # for every shape node we've just loaded find matching shape by its + # cbId in selection. If found outMesh of scene shape will connect to + # inMesh of loaded shape. + for destination_node in nodes: + source_node = scene_lookup[lib.get_id(destination_node)] + if source_node: + self.log.info("found: {}".format(source_node)) + self.log.info( + "creating connection to {}".format(destination_node)) + + cmds.connectAttr("{}.outMesh".format(source_node), + "{}.inMesh".format(destination_node), + force=True) + groupName = "{}:{}".format(namespace, name) presets = config.get_presets(project=os.environ['AVALON_PROJECT']) @@ -38,6 +86,4 @@ class YetiRigLoader(pype.maya.plugin.ReferenceLoader): c[0], c[1], c[2]) self[:] = nodes - self.log.info("Yeti Rig Connection Manager will be available soon") - return nodes From fdccabeeb99c24f16768d51c34764aea0c24350e Mon Sep 17 00:00:00 2001 From: Toke Jepsen Date: Sun, 3 Nov 2019 18:45:30 +0000 Subject: [PATCH 54/65] Support Deadline priority in Nuke. --- pype/nuke/lib.py | 8 ++++++++ pype/plugins/global/publish/submit_publish_job.py | 3 ++- pype/plugins/nuke/publish/collect_writes.py | 7 ++++++- pype/plugins/nuke/publish/submit_nuke_deadline.py | 1 + .../nuke/publish/validate_write_deadline_tab.py | 15 +++++++++++++-- 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index f182088457..d8a1352a55 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -389,6 +389,14 @@ def add_deadline_tab(node): knob.setValue(1) node.addKnob(knob) + knob = nuke.Int_Knob("deadlinePriority", "Priority") + knob.setValue(50) + node.addKnob(knob) + + +def get_deadline_knob_names(): + return ["Deadline", "deadlineChunkSize", "deadlinePriority"] + def create_backdrop(label="", color=None, layer=0, nodes=None): diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 0130eaf0c4..1072f95b8e 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -174,7 +174,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "JobDependency0": job["_id"], "UserName": job["Props"]["User"], "Comment": instance.context.data.get("comment", ""), - "InitialStatus": state + "InitialStatus": state, + "Priority": job["Props"]["Pri"] }, "PluginInfo": { "Version": "3.6", diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index 29ae6cb929..adfc5c69f0 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -106,6 +106,10 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): if "deadlineChunkSize" in group_node.knobs(): deadlineChunkSize = group_node["deadlineChunkSize"].value() + deadlinePriority = 50 + if "deadlinePriority" in group_node.knobs(): + deadlinePriority = group_node["deadlinePriority"].value() + instance.data.update({ "versionData": version_data, "path": path, @@ -117,7 +121,8 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): "frameEnd": last_frame, "outputType": output_type, "colorspace": node["colorspace"].value(), - "deadlineChunkSize": deadlineChunkSize + "deadlineChunkSize": deadlineChunkSize, + "deadlinePriority": deadlinePriority }) self.log.debug("instance.data: {}".format(instance.data)) diff --git a/pype/plugins/nuke/publish/submit_nuke_deadline.py b/pype/plugins/nuke/publish/submit_nuke_deadline.py index ef971f3a37..4044026b5e 100644 --- a/pype/plugins/nuke/publish/submit_nuke_deadline.py +++ b/pype/plugins/nuke/publish/submit_nuke_deadline.py @@ -85,6 +85,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): end=int(instance.data["frameEnd"]) ), "ChunkSize": instance.data["deadlineChunkSize"], + "Priority": instance.data["deadlinePriority"], "Comment": comment, diff --git a/pype/plugins/nuke/publish/validate_write_deadline_tab.py b/pype/plugins/nuke/publish/validate_write_deadline_tab.py index 0c222a164a..4e63e14a7f 100644 --- a/pype/plugins/nuke/publish/validate_write_deadline_tab.py +++ b/pype/plugins/nuke/publish/validate_write_deadline_tab.py @@ -22,6 +22,13 @@ class RepairNukeWriteDeadlineTab(pyblish.api.Action): for instance in instances: group_node = [x for x in instance if x.Class() == "Group"][0] + + # Remove exising knobs. + knob_names = pype.nuke.lib.get_deadline_knob_names() + for name, knob in group_node.knobs().iteritems(): + if name in knob_names: + group_node.removeKnob(knob) + pype.nuke.lib.add_deadline_tab(group_node) @@ -38,5 +45,9 @@ class ValidateNukeWriteDeadlineTab(pyblish.api.InstancePlugin): def process(self, instance): group_node = [x for x in instance if x.Class() == "Group"][0] - msg = "Deadline tab missing on \"{}\"".format(group_node.name()) - assert "Deadline" in group_node.knobs(), msg + knob_names = pype.nuke.lib.get_deadline_knob_names() + missing_knobs = [] + for name in knob_names: + if name not in group_node.knobs().keys(): + missing_knobs.append(name) + assert not missing_knobs, "Missing knobs: {}".format(missing_knobs) From d698a5773387178bceb18418a581bfed389f9a2e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Nov 2019 15:59:35 +0100 Subject: [PATCH 55/65] fix(nuke): adding custom ocio path support --- pype/nuke/lib.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index d8a1352a55..7b0735234a 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -551,17 +551,34 @@ class WorkfileSettings(object): assert isinstance(root_dict, dict), log.error( "set_root_colorspace(): argument should be dictionary") + log.debug(">> root_dict: {}".format(root_dict)) + # first set OCIO if self._root_node["colorManagement"].value() \ not in str(root_dict["colorManagement"]): self._root_node["colorManagement"].setValue( str(root_dict["colorManagement"])) + log.debug("nuke.root()['{0}'] changed to: {1}".format( + "colorManagement", root_dict["colorManagement"])) + root_dict.pop("colorManagement") # second set ocio version if self._root_node["OCIO_config"].value() \ not in str(root_dict["OCIO_config"]): self._root_node["OCIO_config"].setValue( str(root_dict["OCIO_config"])) + log.debug("nuke.root()['{0}'] changed to: {1}".format( + "OCIO_config", root_dict["OCIO_config"])) + root_dict.pop("OCIO_config") + + # third set ocio custom path + if root_dict.get("customOCIOConfigPath"): + self._root_node["customOCIOConfigPath"].setValue( + str(root_dict["customOCIOConfigPath"]).format(**os.environ) + ) + log.debug("nuke.root()['{}'] changed to: {}".format( + "customOCIOConfigPath", root_dict["customOCIOConfigPath"])) + root_dict.pop("customOCIOConfigPath") # then set the rest for knob, value in root_dict.items(): From 95655d005693f37cd0590803751c54c830f557a0 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 4 Nov 2019 16:04:54 +0100 Subject: [PATCH 56/65] added texture extraction to yeti rig extractor --- pype/plugins/maya/publish/extract_yeti_rig.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pype/plugins/maya/publish/extract_yeti_rig.py b/pype/plugins/maya/publish/extract_yeti_rig.py index b575c07cf4..892bc0bea6 100644 --- a/pype/plugins/maya/publish/extract_yeti_rig.py +++ b/pype/plugins/maya/publish/extract_yeti_rig.py @@ -126,6 +126,18 @@ class ExtractYetiRig(pype.api.Extractor): with open(settings_path, "w") as fp: json.dump(settings, fp, ensure_ascii=False) + # add textures to transfers + if 'transfers' not in instance.data: + instance.data['transfers'] = [] + + for resource in instance.data.get('resources', []): + for file in resource['files']: + src = file + dst = os.path.join(image_search_path, os.path.basename(file)) + instance.data['transfers'].append([src, dst]) + + self.log.info("adding transfer {} -> {}". format(src, dst)) + # Ensure the imageSearchPath is being remapped to the publish folder attr_value = {"%s.imageSearchPath" % n: str(image_search_path) for n in yeti_nodes} From b170fe1a6f2314c149663df563f8305e5998fff8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Nov 2019 19:23:29 +0100 Subject: [PATCH 57/65] fix(global): data.path without hashes - removing hashes from data.path --- pype/plugins/global/publish/integrate_new.py | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index a82ebdd5a6..4ef2616cda 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -314,8 +314,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): index_frame_start = int(repre.get("frameStart")) dst_padding_exp = src_padding_exp + dst_start_frame = None for i in src_collection.indexes: src_padding = src_padding_exp % i + + # for adding first frame into db + if not dst_start_frame: + dst_start_frame = src_padding + src_file_name = "{0}{1}{2}".format( src_head, src_padding, src_tail) @@ -326,19 +332,22 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dst_padding = dst_padding_exp % index_frame_start index_frame_start += 1 - dst = "{0}{1}{2}".format(dst_head, dst_padding, dst_tail).replace("..", ".") + dst = "{0}{1}{2}".format( + dst_head, + dst_padding, + dst_tail).replace("..", ".") + self.log.debug("destination: `{}`".format(dst)) src = os.path.join(stagingdir, src_file_name) + self.log.debug("source: {}".format(src)) instance.data["transfers"].append([src, dst]) - repre['published_path'] = "{0}{1}{2}".format(dst_head, - dst_padding_exp, - dst_tail) - # for imagesequence version data - hashes = '#' * len(dst_padding) - dst = os.path.normpath("{0}{1}{2}".format( - dst_head, hashes, dst_tail)) + dst = "{0}{1}{2}".format( + dst_head, + dst_start_frame, + dst_tail).replace("..", ".") + repre['published_path'] = dst else: # Single file From 66d0b1b25326a00926dc961728af1f054d2a9e0a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Nov 2019 19:28:39 +0100 Subject: [PATCH 58/65] feat(nuke): improving nuke features - loader reads fpath and convert to hashes in path - adding `review` knob to write - removing extract_frames.py to _unused plugins --- pype/nuke/lib.py | 4 ++++ .../extract_frames.py | 0 pype/plugins/nuke/load/load_sequence.py | 6 ++++++ pype/plugins/nuke/publish/collect_review.py | 21 ++++++++++++++----- pype/plugins/nuke/publish/collect_writes.py | 7 +++---- .../nuke/publish/extract_render_local.py | 1 + 6 files changed, 30 insertions(+), 9 deletions(-) rename pype/plugins/nuke/{publish => _publish_unused}/extract_frames.py (100%) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index 7b0735234a..00a264c669 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -379,6 +379,10 @@ def add_rendering_knobs(node): knob = nuke.Boolean_Knob("render_farm", "Render on Farm") knob.setValue(False) node.addKnob(knob) + if "review" not in node.knobs(): + knob = nuke.Boolean_Knob("review", "Review") + knob.setValue(True) + node.addKnob(knob) return node diff --git a/pype/plugins/nuke/publish/extract_frames.py b/pype/plugins/nuke/_publish_unused/extract_frames.py similarity index 100% rename from pype/plugins/nuke/publish/extract_frames.py rename to pype/plugins/nuke/_publish_unused/extract_frames.py diff --git a/pype/plugins/nuke/load/load_sequence.py b/pype/plugins/nuke/load/load_sequence.py index 2946857e09..e1c75584d7 100644 --- a/pype/plugins/nuke/load/load_sequence.py +++ b/pype/plugins/nuke/load/load_sequence.py @@ -110,6 +110,7 @@ class LoadSequence(api.Loader): last += self.handle_end file = self.fname.replace("\\", "/") + log.info("file: {}\n".format(self.fname)) repr_cont = context["representation"]["context"] @@ -118,6 +119,11 @@ class LoadSequence(api.Loader): repr_cont["subset"], repr_cont["representation"]) + if "#" not in file: + frame = repr_cont.get("frame") + padding = len(frame) + file = file.replace(frame, "#"*padding) + # Create the Loader with the filename path set with viewer_update_and_undo_stop(): # TODO: it might be universal read to img/geo/camera diff --git a/pype/plugins/nuke/publish/collect_review.py b/pype/plugins/nuke/publish/collect_review.py index 0ab5424434..7e7cbedd6c 100644 --- a/pype/plugins/nuke/publish/collect_review.py +++ b/pype/plugins/nuke/publish/collect_review.py @@ -1,5 +1,5 @@ import pyblish.api - +import nuke class CollectReview(pyblish.api.InstancePlugin): """Collect review instance from rendered frames @@ -9,9 +9,20 @@ class CollectReview(pyblish.api.InstancePlugin): family = "review" label = "Collect Review" hosts = ["nuke"] - families = ["render", "render.local"] + families = ["render", "render.local", "render.farm"] def process(self, instance): - if instance.data["families"]: - instance.data["families"].append("review") - self.log.info("Review collected: `{}`".format(instance)) + + node = instance[0] + + if "review" not in node.knobs(): + knob = nuke.Boolean_Knob("review", "Review") + knob.setValue(True) + node.addKnob(knob) + + if not node["review"].value(): + return + + instance.data["families"].append("review") + instance.data['families'].append('ftrack') + self.log.info("Review collected: `{}`".format(instance)) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index adfc5c69f0..fa92192563 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -65,7 +65,6 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): ) if 'render' in instance.data['families']: - instance.data['families'].append('ftrack') if "representations" not in instance.data: instance.data["representations"] = list() @@ -78,15 +77,15 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): try: collected_frames = os.listdir(output_dir) + if collected_frames: + representation['frameStart'] = "%0{}d".format( + len(str(last_frame))) % first_frame representation['files'] = collected_frames instance.data["representations"].append(representation) except Exception: instance.data["representations"].append(representation) self.log.debug("couldn't collect frames: {}".format(label)) - if 'render.local' in instance.data['families']: - instance.data['families'].append('ftrack') - # Add version data to instance version_data = { "handles": handle_start, diff --git a/pype/plugins/nuke/publish/extract_render_local.py b/pype/plugins/nuke/publish/extract_render_local.py index 29de347288..825db67e9d 100644 --- a/pype/plugins/nuke/publish/extract_render_local.py +++ b/pype/plugins/nuke/publish/extract_render_local.py @@ -58,6 +58,7 @@ class NukeRenderLocal(pype.api.Extractor): repre = { 'name': ext, 'ext': ext, + 'frameStart': "%0{}d".format(len(str(last_frame))) % first_frame, 'files': collected_frames, "stagingDir": out_dir, "anatomy_template": "render" From a188d37d624009ad018c29101c76734848afbee6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Nov 2019 19:37:33 +0100 Subject: [PATCH 59/65] fix(gloabal): validator didn't work properly --- .../global/publish/validate_custom_ftrack_attributes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/validate_custom_ftrack_attributes.py b/pype/plugins/global/publish/validate_custom_ftrack_attributes.py index 0d0eb2a7f0..2386b359e4 100644 --- a/pype/plugins/global/publish/validate_custom_ftrack_attributes.py +++ b/pype/plugins/global/publish/validate_custom_ftrack_attributes.py @@ -46,7 +46,8 @@ class ValidateFtrackAttributes(pyblish.api.InstancePlugin): "Missing FTrack Task entity in context") host = pyblish.api.current_host() - to_check = context.data["presets"][host].get("ftrack_attributes") + to_check = context.data["presets"].get( + host, {}).get("ftrack_attributes") if not to_check: self.log.warning("ftrack_attributes preset not found") return From 1df729c206c93b8b5752585fb98ab1bc18647824 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Nov 2019 20:31:21 +0100 Subject: [PATCH 60/65] fix(nuke): version data frameStart/end excluded handles --- pype/plugins/nuke/publish/collect_writes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index fa92192563..28470c94de 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -91,8 +91,8 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): "handles": handle_start, "handleStart": handle_start, "handleEnd": handle_end, - "frameStart": first_frame, - "frameEnd": last_frame, + "frameStart": first_frame + handle_start, + "frameEnd": last_frame - handle_end, "version": int(version), "colorspace": node["colorspace"].value(), "families": [instance.data["family"]], From 26454a752b506a5fffd698cee612164c8ac67feb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Nov 2019 10:23:49 +0100 Subject: [PATCH 61/65] fixed not crashing event thumbnail updates if thumbnail is not happening on task --- pype/ftrack/events/event_thumbnail_updates.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pype/ftrack/events/event_thumbnail_updates.py b/pype/ftrack/events/event_thumbnail_updates.py index ae6f8adb5e..65c81f6545 100644 --- a/pype/ftrack/events/event_thumbnail_updates.py +++ b/pype/ftrack/events/event_thumbnail_updates.py @@ -20,7 +20,8 @@ class ThumbnailEvents(BaseEvent): if parent.get('thumbnail') and not task.get('thumbnail'): task['thumbnail'] = parent['thumbnail'] self.log.info('>>> Updated thumbnail on [ %s/%s ]'.format( - parent['name'], task['name'])) + parent['name'], task['name'] + )) # Update task thumbnail from published version # if (entity['entityType'] == 'assetversion' and @@ -32,18 +33,24 @@ class ThumbnailEvents(BaseEvent): version = session.get('AssetVersion', entity['entityId']) thumbnail = version.get('thumbnail') - task = version['task'] - if thumbnail: - task['thumbnail'] = thumbnail - task['parent']['thumbnail'] = thumbnail - self.log.info('>>> Updating thumbnail for task and shot\ - [ {} ]'.format(task['name'])) + parent = version['asset']['parent'] + task = version['task'] + parent['thumbnail_id'] = version['thumbnail_id'] + if parent.entity_type.lower() == "project": + name = parent["full_name"] + else: + name = parent["name"] + msg = '>>> Updating thumbnail for shot [ {} ]'.format(name) + + if task: + task['thumbnail_id'] = asset_version['thumbnail_id'] + msg += " and task [ {} ]".format(task["name"])) + + self.log.info(msg) session.commit() - pass - def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' From b3dbf0f228131b3ffaae6208123feee7cf2c62bf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Nov 2019 10:29:02 +0100 Subject: [PATCH 62/65] variable name fix --- pype/ftrack/events/event_thumbnail_updates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/events/event_thumbnail_updates.py b/pype/ftrack/events/event_thumbnail_updates.py index 65c81f6545..620d1a5b72 100644 --- a/pype/ftrack/events/event_thumbnail_updates.py +++ b/pype/ftrack/events/event_thumbnail_updates.py @@ -44,7 +44,7 @@ class ThumbnailEvents(BaseEvent): msg = '>>> Updating thumbnail for shot [ {} ]'.format(name) if task: - task['thumbnail_id'] = asset_version['thumbnail_id'] + task['thumbnail_id'] = version['thumbnail_id'] msg += " and task [ {} ]".format(task["name"])) self.log.info(msg) From b22f7536caa25754d0f82aeb080287ff8f9d5bfa Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 5 Nov 2019 18:27:09 +0100 Subject: [PATCH 63/65] moved pymel.core imports to process parts of actions to not slow down discover --- pype/plugins/maya/load/load_ass.py | 2 +- pype/plugins/maya/load/load_gpucache.py | 1 - pype/plugins/maya/load/load_image_plane.py | 4 ++-- pype/plugins/maya/load/load_reference.py | 5 ++--- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pype/plugins/maya/load/load_ass.py b/pype/plugins/maya/load/load_ass.py index 979d4b5767..2960e4403e 100644 --- a/pype/plugins/maya/load/load_ass.py +++ b/pype/plugins/maya/load/load_ass.py @@ -1,7 +1,6 @@ from avalon import api import pype.maya.plugin import os -import pymel.core as pm from pypeapp import config @@ -70,6 +69,7 @@ class AssProxyLoader(pype.maya.plugin.ReferenceLoader): import os from maya import cmds + import pymel.core as pm node = container["objectName"] diff --git a/pype/plugins/maya/load/load_gpucache.py b/pype/plugins/maya/load/load_gpucache.py index b98ca8b7f4..9e7938777e 100644 --- a/pype/plugins/maya/load/load_gpucache.py +++ b/pype/plugins/maya/load/load_gpucache.py @@ -2,7 +2,6 @@ from avalon import api import pype.maya.plugin import os from pypeapp import config -import pymel.core as pm reload(config) diff --git a/pype/plugins/maya/load/load_image_plane.py b/pype/plugins/maya/load/load_image_plane.py index e2d94ac82e..e95ea6cd8f 100644 --- a/pype/plugins/maya/load/load_image_plane.py +++ b/pype/plugins/maya/load/load_image_plane.py @@ -1,5 +1,3 @@ -import pymel.core as pc - from avalon import api from Qt import QtWidgets @@ -14,6 +12,8 @@ class ImagePlaneLoader(api.Loader): color = "orange" def load(self, context, name, namespace, data): + import pymel.core as pc + new_nodes = [] image_plane_depth = 1000 diff --git a/pype/plugins/maya/load/load_reference.py b/pype/plugins/maya/load/load_reference.py index a754c3be98..55db019cf4 100644 --- a/pype/plugins/maya/load/load_reference.py +++ b/pype/plugins/maya/load/load_reference.py @@ -1,8 +1,6 @@ - import pype.maya.plugin import os from pypeapp import config -import pymel.core as pm reload(config) import pype.maya.plugin reload(pype.maya.plugin) @@ -20,9 +18,10 @@ class ReferenceLoader(pype.maya.plugin.ReferenceLoader): color = "orange" def process_reference(self, context, name, namespace, data): - import maya.cmds as cmds from avalon import maya + import pymel.core as pm + try: family = context["representation"]["context"]["family"] From f35f4c1e93c4b7e64c13334036ccf3fdce1ed683 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Nov 2019 01:14:09 +0100 Subject: [PATCH 64/65] feat(nuke): templates.py to presets.py - rename key `preset` to `families` - `families`: looping family as preset - omit tags for later clearing --- pype/nuke/lib.py | 4 +++- pype/nuke/{templates.py => presets.py} | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) rename pype/nuke/{templates.py => presets.py} (75%) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index 00a264c669..2021e6bdd4 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -8,7 +8,9 @@ import avalon.nuke import pype.api as pype import nuke -from .templates import ( + + +from .presets import ( get_colorspace_preset, get_node_dataflow_preset, get_node_colorspace_preset diff --git a/pype/nuke/templates.py b/pype/nuke/presets.py similarity index 75% rename from pype/nuke/templates.py rename to pype/nuke/presets.py index 6434d73f1d..a3f62764c8 100644 --- a/pype/nuke/templates.py +++ b/pype/nuke/presets.py @@ -25,16 +25,22 @@ def get_node_dataflow_preset(**kwarg): log.info(kwarg) host = kwarg.get("host", "nuke") cls = kwarg.get("class", None) - preset = kwarg.get("preset", None) - assert any([host, cls]), log.error("nuke.templates.get_node_dataflow_preset(): \ - Missing mandatory kwargs `host`, `cls`") + families = kwarg.get("families", []) + preset = kwarg.get("preset", None) # omit < 2.0.0v + + assert any([host, cls]), log.error( + "`{}`: Missing mandatory kwargs `host`, `cls`".format(__file__)) nuke_dataflow = get_dataflow_preset().get(str(host), None) nuke_dataflow_nodes = nuke_dataflow.get('nodes', None) nuke_dataflow_node = nuke_dataflow_nodes.get(str(cls), None) - if preset: + if preset: # omit < 2.0.0v nuke_dataflow_node = nuke_dataflow_node.get(str(preset), None) + # omit < 2.0.0v + + for family in families: + nuke_dataflow_node = nuke_dataflow_node.get(str(family), None) log.info("Dataflow: {}".format(nuke_dataflow_node)) return nuke_dataflow_node @@ -47,8 +53,8 @@ def get_node_colorspace_preset(**kwarg): host = kwarg.get("host", "nuke") cls = kwarg.get("class", None) preset = kwarg.get("preset", None) - assert any([host, cls]), log.error("nuke.templates.get_node_colorspace_preset(): \ - Missing mandatory kwargs `host`, `cls`") + assert any([host, cls]), log.error( + "`{}`: Missing mandatory kwargs `host`, `cls`".format(__file__)) nuke_colorspace = get_colorspace_preset().get(str(host), None) nuke_colorspace_node = nuke_colorspace.get(str(cls), None) From 2d6e11c19a54f83fea83f814635523eb6fd12ba6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 6 Nov 2019 01:17:42 +0100 Subject: [PATCH 65/65] feat(nuke): node avalon knob prefix as list for backward compatibility - `avalon:` or `ak:` - omit tags for later clearing in pype.nuke.presets --- pype/nuke/lib.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index 2021e6bdd4..8ef8ccf26f 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -57,7 +57,8 @@ def checkInventoryVersions(): if container: node = container["_node"] - avalon_knob_data = avalon.nuke.get_avalon_knob_data(node) + avalon_knob_data = avalon.nuke.get_avalon_knob_data( + node, ['avalon:', 'ak:']) # get representation from io representation = io.find_one({ @@ -103,7 +104,8 @@ def writes_version_sync(): for each in nuke.allNodes(): if each.Class() == 'Write': - avalon_knob_data = avalon.nuke.get_avalon_knob_data(each) + avalon_knob_data = avalon.nuke.get_avalon_knob_data( + each, ['avalon:', 'ak:']) try: if avalon_knob_data['families'] not in ["render"]: @@ -136,7 +138,8 @@ def get_render_path(node): ''' Generate Render path from presets regarding avalon knob data ''' data = dict() - data['avalon'] = avalon.nuke.get_avalon_knob_data(node) + data['avalon'] = avalon.nuke.get_avalon_knob_data( + node, ['avalon:', 'ak:']) data_preset = { "class": data['avalon']['family'], @@ -829,10 +832,12 @@ def get_write_node_template_attr(node): ''' # get avalon data from node data = dict() - data['avalon'] = avalon.nuke.get_avalon_knob_data(node) + data['avalon'] = avalon.nuke.get_avalon_knob_data( + node, ['avalon:', 'ak:']) data_preset = { "class": data['avalon']['family'], - "preset": data['avalon']['families'] + "families": data['avalon']['families'], + "preset": data['avalon']['families'] # omit < 2.0.0v } # get template data