From 11e2382dfd539ae36c4ff3bbff896d69588a99ff Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:00:04 +0200 Subject: [PATCH 01/37] session check is happening during initialization not before register of each plugin --- pype/ftrack/actions/action_attributes_remapper.py | 3 --- pype/ftrack/actions/action_client_review_sort.py | 5 +---- pype/ftrack/actions/action_component_open.py | 6 ------ pype/ftrack/actions/action_create_cust_attrs.py | 6 ------ pype/ftrack/actions/action_create_folders.py | 3 --- .../actions/action_create_project_structure.py | 3 --- pype/ftrack/actions/action_cust_attr_doctor.py | 5 +---- pype/ftrack/actions/action_delete_asset.py | 8 +------- pype/ftrack/actions/action_delete_asset_byname.py | 6 ------ pype/ftrack/actions/action_djvview.py | 2 -- pype/ftrack/actions/action_job_killer.py | 6 ------ pype/ftrack/actions/action_multiple_notes.py | 3 --- pype/ftrack/actions/action_prepare_project.py | 3 --- pype/ftrack/actions/action_rv.py | 2 -- pype/ftrack/actions/action_start_timer.py | 5 +---- .../actions/action_sync_hier_attrs_local.py | 3 --- .../ftrack/actions/action_sync_to_avalon_local.py | 7 ------- pype/ftrack/actions/action_test.py | 3 --- .../actions/action_thumbnail_to_childern.py | 2 -- pype/ftrack/actions/action_thumbnail_to_parent.py | 2 -- pype/ftrack/actions/action_where_run_ask.py | 3 --- pype/ftrack/actions/action_where_run_show.py | 3 --- pype/ftrack/events/action_sync_hier_attrs.py | 5 +---- pype/ftrack/events/action_sync_to_avalon.py | 3 --- .../ftrack/events/event_del_avalon_id_from_new.py | 2 -- pype/ftrack/events/event_next_task_update.py | 2 -- pype/ftrack/events/event_radio_buttons.py | 2 -- pype/ftrack/events/event_sync_hier_attr.py | 2 -- pype/ftrack/events/event_sync_to_avalon.py | 4 ---- pype/ftrack/events/event_test.py | 4 +--- pype/ftrack/events/event_thumbnail_updates.py | 2 -- pype/ftrack/events/event_user_assigment.py | 2 -- .../events/event_version_to_task_statuses.py | 2 -- pype/ftrack/lib/ftrack_base_handler.py | 15 ++++++++++++++- 34 files changed, 20 insertions(+), 114 deletions(-) diff --git a/pype/ftrack/actions/action_attributes_remapper.py b/pype/ftrack/actions/action_attributes_remapper.py index 759b5765e5..a0393ece40 100644 --- a/pype/ftrack/actions/action_attributes_remapper.py +++ b/pype/ftrack/actions/action_attributes_remapper.py @@ -281,7 +281,4 @@ class AttributesRemapper(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - AttributesRemapper(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_client_review_sort.py b/pype/ftrack/actions/action_client_review_sort.py index 6a659ce5e3..91926c2874 100644 --- a/pype/ftrack/actions/action_client_review_sort.py +++ b/pype/ftrack/actions/action_client_review_sort.py @@ -55,11 +55,8 @@ class ClientReviewSort(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - action_handler = ClientReviewSort(session, plugins_presets) - action_handler.register() + ClientReviewSort(session, plugins_presets).register() def main(arguments=None): diff --git a/pype/ftrack/actions/action_component_open.py b/pype/ftrack/actions/action_component_open.py index 33f4d38890..98d773dba6 100644 --- a/pype/ftrack/actions/action_component_open.py +++ b/pype/ftrack/actions/action_component_open.py @@ -68,12 +68,6 @@ class ComponentOpen(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - ComponentOpen(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_cust_attrs.py b/pype/ftrack/actions/action_create_cust_attrs.py index 47a6bb5d5f..ac6dcb0fd7 100644 --- a/pype/ftrack/actions/action_create_cust_attrs.py +++ b/pype/ftrack/actions/action_create_cust_attrs.py @@ -572,12 +572,6 @@ class CustomAttributes(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - CustomAttributes(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_folders.py b/pype/ftrack/actions/action_create_folders.py index 269316e052..44e9741bab 100644 --- a/pype/ftrack/actions/action_create_folders.py +++ b/pype/ftrack/actions/action_create_folders.py @@ -327,9 +327,6 @@ class PartialDict(dict): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CreateFolders(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_create_project_structure.py b/pype/ftrack/actions/action_create_project_structure.py index 74d458b5f8..c39c717b11 100644 --- a/pype/ftrack/actions/action_create_project_structure.py +++ b/pype/ftrack/actions/action_create_project_structure.py @@ -198,9 +198,6 @@ class CreateProjectFolders(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CreateProjectFolders(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_cust_attr_doctor.py b/pype/ftrack/actions/action_cust_attr_doctor.py index af5fe2dc4a..0469b3a1e6 100644 --- a/pype/ftrack/actions/action_cust_attr_doctor.py +++ b/pype/ftrack/actions/action_cust_attr_doctor.py @@ -9,7 +9,7 @@ from pype.ftrack import BaseAction class CustomAttributeDoctor(BaseAction): - + ignore_me = True #: Action identifier. identifier = 'custom.attributes.doctor' @@ -294,9 +294,6 @@ class CustomAttributeDoctor(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - CustomAttributeDoctor(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_delete_asset.py b/pype/ftrack/actions/action_delete_asset.py index 1b1e7fc905..106c81758a 100644 --- a/pype/ftrack/actions/action_delete_asset.py +++ b/pype/ftrack/actions/action_delete_asset.py @@ -85,7 +85,7 @@ class DeleteAsset(BaseAction): 'type': 'asset', 'name': entity['name'] }) - + if av_entity is None: return { 'success': False, @@ -314,12 +314,6 @@ class DeleteAsset(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - DeleteAsset(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_delete_asset_byname.py b/pype/ftrack/actions/action_delete_asset_byname.py index 2431b2311e..4a3807f8f0 100644 --- a/pype/ftrack/actions/action_delete_asset_byname.py +++ b/pype/ftrack/actions/action_delete_asset_byname.py @@ -135,12 +135,6 @@ class AssetsRemover(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - AssetsRemover(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_djvview.py b/pype/ftrack/actions/action_djvview.py index 58914fbc1e..9da12dd67c 100644 --- a/pype/ftrack/actions/action_djvview.py +++ b/pype/ftrack/actions/action_djvview.py @@ -220,8 +220,6 @@ class DJVViewAction(BaseAction): def register(session, plugins_presets={}): """Register hooks.""" - if not isinstance(session, ftrack_api.session.Session): - return DJVViewAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_job_killer.py b/pype/ftrack/actions/action_job_killer.py index 717f87e879..64fb99133d 100644 --- a/pype/ftrack/actions/action_job_killer.py +++ b/pype/ftrack/actions/action_job_killer.py @@ -121,12 +121,6 @@ class JobKiller(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - JobKiller(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_multiple_notes.py b/pype/ftrack/actions/action_multiple_notes.py index 6e28b7bed6..bd51cb2984 100644 --- a/pype/ftrack/actions/action_multiple_notes.py +++ b/pype/ftrack/actions/action_multiple_notes.py @@ -115,9 +115,6 @@ class MultipleNotes(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - MultipleNotes(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_prepare_project.py b/pype/ftrack/actions/action_prepare_project.py index e914fa74f0..d645748f6f 100644 --- a/pype/ftrack/actions/action_prepare_project.py +++ b/pype/ftrack/actions/action_prepare_project.py @@ -372,7 +372,4 @@ class PrepareProject(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - PrepareProject(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_rv.py b/pype/ftrack/actions/action_rv.py index 6b6591355f..69c6624b71 100644 --- a/pype/ftrack/actions/action_rv.py +++ b/pype/ftrack/actions/action_rv.py @@ -328,8 +328,6 @@ class RVAction(BaseAction): def register(session, plugins_presets={}): """Register hooks.""" - if not isinstance(session, ftrack_api.session.Session): - return RVAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_start_timer.py b/pype/ftrack/actions/action_start_timer.py index 36752a1edc..292789e9f3 100644 --- a/pype/ftrack/actions/action_start_timer.py +++ b/pype/ftrack/actions/action_start_timer.py @@ -26,7 +26,7 @@ class StartTimer(BaseAction): user.start_timer(entity, force=True) self.session.commit() - + self.log.info( "Starting Ftrack timer for task: {}".format(entity['name']) ) @@ -37,7 +37,4 @@ class StartTimer(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - StartTimer(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_sync_hier_attrs_local.py b/pype/ftrack/actions/action_sync_hier_attrs_local.py index 05a70461a1..289abd0122 100644 --- a/pype/ftrack/actions/action_sync_hier_attrs_local.py +++ b/pype/ftrack/actions/action_sync_hier_attrs_local.py @@ -309,9 +309,6 @@ class SyncHierarchicalAttrs(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_sync_to_avalon_local.py b/pype/ftrack/actions/action_sync_to_avalon_local.py index ddf8ed6571..61050f9883 100644 --- a/pype/ftrack/actions/action_sync_to_avalon_local.py +++ b/pype/ftrack/actions/action_sync_to_avalon_local.py @@ -263,11 +263,4 @@ class SyncToAvalon(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - - # Validate that session is an instance of ftrack_api.Session. If not, - # assume that register is being called from an old or incompatible API and - # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - SyncToAvalon(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_test.py b/pype/ftrack/actions/action_test.py index a2bc8bf892..58f7210e3b 100644 --- a/pype/ftrack/actions/action_test.py +++ b/pype/ftrack/actions/action_test.py @@ -43,9 +43,6 @@ class TestAction(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - TestAction(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_thumbnail_to_childern.py b/pype/ftrack/actions/action_thumbnail_to_childern.py index 101b678512..7d189cf652 100644 --- a/pype/ftrack/actions/action_thumbnail_to_childern.py +++ b/pype/ftrack/actions/action_thumbnail_to_childern.py @@ -68,8 +68,6 @@ class ThumbToChildren(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbToChildren(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_thumbnail_to_parent.py b/pype/ftrack/actions/action_thumbnail_to_parent.py index c382d9303c..efafca4a96 100644 --- a/pype/ftrack/actions/action_thumbnail_to_parent.py +++ b/pype/ftrack/actions/action_thumbnail_to_parent.py @@ -90,8 +90,6 @@ class ThumbToParent(BaseAction): def register(session, plugins_presets={}): '''Register action. Called when used as an event plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbToParent(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_where_run_ask.py b/pype/ftrack/actions/action_where_run_ask.py index 0351c09909..795c2664cc 100644 --- a/pype/ftrack/actions/action_where_run_ask.py +++ b/pype/ftrack/actions/action_where_run_ask.py @@ -40,7 +40,4 @@ class ActionAskWhereIRun(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - ActionAskWhereIRun(session, plugins_presets).register() diff --git a/pype/ftrack/actions/action_where_run_show.py b/pype/ftrack/actions/action_where_run_show.py index 7fea23e3b7..48618f0251 100644 --- a/pype/ftrack/actions/action_where_run_show.py +++ b/pype/ftrack/actions/action_where_run_show.py @@ -80,7 +80,4 @@ class ActionShowWhereIRun(BaseAction): def register(session, plugins_presets={}): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - ActionShowWhereIRun(session, plugins_presets).register() diff --git a/pype/ftrack/events/action_sync_hier_attrs.py b/pype/ftrack/events/action_sync_hier_attrs.py index c9d968ee5d..23ac319261 100644 --- a/pype/ftrack/events/action_sync_hier_attrs.py +++ b/pype/ftrack/events/action_sync_hier_attrs.py @@ -220,7 +220,7 @@ class SyncHierarchicalAttrs(BaseAction): if job['status'] in ('queued', 'running'): job['status'] = 'failed' session.commit() - + if self.interface_messages: self.show_interface_from_dict( messages=self.interface_messages, @@ -341,9 +341,6 @@ class SyncHierarchicalAttrs(BaseAction): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return - SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/events/action_sync_to_avalon.py b/pype/ftrack/events/action_sync_to_avalon.py index 51a4ae9475..7b5f94f216 100644 --- a/pype/ftrack/events/action_sync_to_avalon.py +++ b/pype/ftrack/events/action_sync_to_avalon.py @@ -296,9 +296,6 @@ def register(session, plugins_presets): # Validate that session is an instance of ftrack_api.Session. If not, # assume that register is being called from an old or incompatible API and # return without doing anything. - if not isinstance(session, ftrack_api.session.Session): - return - SyncToAvalon(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_del_avalon_id_from_new.py b/pype/ftrack/events/event_del_avalon_id_from_new.py index 6f6320f51b..e5c4b1be45 100644 --- a/pype/ftrack/events/event_del_avalon_id_from_new.py +++ b/pype/ftrack/events/event_del_avalon_id_from_new.py @@ -53,7 +53,5 @@ class DelAvalonIdFromNew(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return DelAvalonIdFromNew(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_next_task_update.py b/pype/ftrack/events/event_next_task_update.py index 18a7abf328..51ccb2f057 100644 --- a/pype/ftrack/events/event_next_task_update.py +++ b/pype/ftrack/events/event_next_task_update.py @@ -88,7 +88,5 @@ class NextTaskUpdate(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return NextTaskUpdate(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_radio_buttons.py b/pype/ftrack/events/event_radio_buttons.py index 9c6f2d490a..917c7a49e6 100644 --- a/pype/ftrack/events/event_radio_buttons.py +++ b/pype/ftrack/events/event_radio_buttons.py @@ -36,7 +36,5 @@ class Radio_buttons(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return Radio_buttons(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_sync_hier_attr.py b/pype/ftrack/events/event_sync_hier_attr.py index 5ddc2394af..8031ec9e55 100644 --- a/pype/ftrack/events/event_sync_hier_attr.py +++ b/pype/ftrack/events/event_sync_hier_attr.py @@ -209,7 +209,5 @@ class SyncHierarchicalAttrs(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return SyncHierarchicalAttrs(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_sync_to_avalon.py b/pype/ftrack/events/event_sync_to_avalon.py index 4e7b208726..a25866be65 100644 --- a/pype/ftrack/events/event_sync_to_avalon.py +++ b/pype/ftrack/events/event_sync_to_avalon.py @@ -122,8 +122,4 @@ class Sync_to_Avalon(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - - if not isinstance(session, ftrack_api.session.Session): - return - Sync_to_Avalon(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_test.py b/pype/ftrack/events/event_test.py index 94d99dbf67..a909aa5510 100644 --- a/pype/ftrack/events/event_test.py +++ b/pype/ftrack/events/event_test.py @@ -8,7 +8,7 @@ from pype.ftrack import BaseEvent class Test_Event(BaseEvent): ignore_me = True - + priority = 10000 def launch(self, session, event): @@ -22,7 +22,5 @@ class Test_Event(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return Test_Event(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_thumbnail_updates.py b/pype/ftrack/events/event_thumbnail_updates.py index 51bb15a4c7..ae6f8adb5e 100644 --- a/pype/ftrack/events/event_thumbnail_updates.py +++ b/pype/ftrack/events/event_thumbnail_updates.py @@ -47,7 +47,5 @@ class ThumbnailEvents(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return ThumbnailEvents(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_user_assigment.py b/pype/ftrack/events/event_user_assigment.py index fe8b331629..61699d736c 100644 --- a/pype/ftrack/events/event_user_assigment.py +++ b/pype/ftrack/events/event_user_assigment.py @@ -233,7 +233,5 @@ def register(session, plugins_presets): """ Register plugin. Called when used as an plugin. """ - if not isinstance(session, ftrack_api.session.Session): - return UserAssigmentEvent(session, plugins_presets).register() diff --git a/pype/ftrack/events/event_version_to_task_statuses.py b/pype/ftrack/events/event_version_to_task_statuses.py index 85a31383d5..66a55c0cf7 100644 --- a/pype/ftrack/events/event_version_to_task_statuses.py +++ b/pype/ftrack/events/event_version_to_task_statuses.py @@ -71,7 +71,5 @@ class VersionToTaskStatus(BaseEvent): def register(session, plugins_presets): '''Register plugin. Called when used as an plugin.''' - if not isinstance(session, ftrack_api.session.Session): - return VersionToTaskStatus(session, plugins_presets).register() diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 9eda74f0f3..9821bfd81f 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -31,8 +31,21 @@ class BaseHandler(object): def __init__(self, session, plugins_presets={}): '''Expects a ftrack_api.Session instance''' - self._session = session self.log = Logger().get_logger(self.__class__.__name__) + if not isinstance(session, ftrack_api.session.Session): + self.log.warning(( + "Session object entered with args is instance of \"{}\"" + " but expected instance is \"{}\"." + ).format( + str(type(session)), + str(ftrack_api.session.Session.__qualname__) + )) + self.register = self.register_without_session + self._session = None + + return + + self._session = session # Using decorator self.register = self.register_decorator(self.register) From 43f63fd260768b03ceac194d40083e4b188f3043 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:01:43 +0200 Subject: [PATCH 02/37] print traceback when error during plugin registration happens --- pype/ftrack/ftrack_server/ftrack_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/ftrack_server.py b/pype/ftrack/ftrack_server/ftrack_server.py index e1c13cda32..612e89e60f 100644 --- a/pype/ftrack/ftrack_server/ftrack_server.py +++ b/pype/ftrack/ftrack_server/ftrack_server.py @@ -126,7 +126,7 @@ class FtrackServer: msg = '"{}" - register was not successful ({})'.format( function_dict['name'], str(exc) ) - log.warning(msg) + log.warning(msg, exc_info=True) def run_server(self): self.session = ftrack_api.Session(auto_connect_event_hub=True,) From f0e5f39342fadcfdc626e06141cfdeba1af47d2e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:02:22 +0200 Subject: [PATCH 03/37] run server in ftrack server gives ability to use other session and not to load plugins --- pype/ftrack/ftrack_server/ftrack_server.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/pype/ftrack/ftrack_server/ftrack_server.py b/pype/ftrack/ftrack_server/ftrack_server.py index 612e89e60f..12b046c510 100644 --- a/pype/ftrack/ftrack_server/ftrack_server.py +++ b/pype/ftrack/ftrack_server/ftrack_server.py @@ -128,21 +128,25 @@ class FtrackServer: ) log.warning(msg, exc_info=True) - def run_server(self): - self.session = ftrack_api.Session(auto_connect_event_hub=True,) + def run_server(self, session=None, load_files=True): + if not session: + session = ftrack_api.Session(auto_connect_event_hub=True) - paths_str = os.environ.get(self.env_key) - if paths_str is None: - log.error(( - "Env var \"{}\" is not set, \"{}\" server won\'t launch" - ).format(self.env_key, self.server_type)) - return + self.session = session - paths = paths_str.split(os.pathsep) - self.set_files(paths) + if load_files: + paths_str = os.environ.get(self.env_key) + if paths_str is None: + log.error(( + "Env var \"{}\" is not set, \"{}\" server won\'t launch" + ).format(self.env_key, self.server_type)) + return - log.info(60*"*") - log.info('Registration of actions/events has finished!') + paths = paths_str.split(os.pathsep) + self.set_files(paths) + + log.info(60*"*") + log.info('Registration of actions/events has finished!') # keep event_hub on session running self.session.event_hub.wait() From 8b8427ca871a5a36e462d929d76a7521d206853c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:11:12 +0200 Subject: [PATCH 04/37] added scripts to be able to be run as subprocess, one for storing, second for processing events --- .../sub_event_processor.py | 46 +++++++++ .../parallel_event_server/sub_event_storer.py | 97 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py new file mode 100644 index 0000000000..bd3118a893 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -0,0 +1,46 @@ +import os +import sys +import datetime +import signal +import socket +import pymongo + +from pype.ftrack.ftrack_server import FtrackServer +from session_processor import ProcessSession +from pypeapp import Logger + +log = Logger().get_logger("Event storer") + + +def main(args): + port = int(args[-1]) + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Connect the socket to the port where the server is listening + server_address = ("localhost", port) + log.debug("Processor connected to {} port {}".format(*server_address)) + sock.connect(server_address) + + sock.sendall(b"CreatedProcess") + try: + session = ProcessSession(auto_connect_event_hub=True, sock=sock) + server = FtrackServer('event') + log.debug("Launched Ftrack Event processor") + server.run_server(session) + + finally: + log.debug("First closing socket") + sock.close() + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + main(sys.argv) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py new file mode 100644 index 0000000000..295504f389 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -0,0 +1,97 @@ +import os +import sys +import datetime +import signal +import socket +import pymongo + +from pype.ftrack.ftrack_server import FtrackServer +from pype.ftrack.lib.custom_db_connector import DbConnector +from pype.ftrack.ftrack_server.parallel_event_server.session_storer import StorerSession +from pypeapp import Logger + +log = Logger().get_logger("Event storer") + + +dbcon = DbConnector( + mongo_url=os.environ["AVALON_MONGO"], + database_name="pype" +) +table_name = "ftrack_events" + +# ignore_topics = ["ftrack.meta.connected"] +ignore_topics = [] + +def install_db(): + dbcon.install() + dbcon.create_table(table_name, capped=False) + dbcon.active_table = table_name + +def launch(event): + if event.get("topic") in ignore_topics: + return + + event_data = event._data + event_id = event["id"] + + event_data["pype_data"] = { + "stored": datetime.datetime.utcnow(), + "is_processed": False + } + + try: + # dbcon.insert_one(event_data) + dbcon.update({"id": event_id}, event_data, upsert=True) + log.debug("Event: {} stored".format(event_id)) + + except pymongo.errors.DuplicateKeyError: + log.debug("Event: {} already exists".format(event_id)) + + except Exception as exc: + log.error( + "Event: {} failed to store".format(event_id), + exc_info=True + ) + + +def register(session): + '''Registers the event, subscribing the discover and launch topics.''' + install_db() + session.event_hub.subscribe("topic=*", launch) + + +def main(args): + port = int(args[-1]) + + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + # Connect the socket to the port where the server is listening + server_address = ("localhost", port) + log.debug("Storer connected to {} port {}".format(*server_address)) + sock.connect(server_address) + sock.sendall(b"CreatedStore") + + try: + session = StorerSession(auto_connect_event_hub=True, sock=sock) + register(session) + server = FtrackServer("event") + log.info(os.environ["FTRACK_EVENTS_PATH"]) + log.debug("Launched Ftrack Event storer") + server.run_server(session, load_files=False) + + finally: + log.debug("First closing socket") + sock.close() + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + main(sys.argv) From 1ee8ab0dabeec7b4cfb2098e051bbd2bd583e050 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:11:46 +0200 Subject: [PATCH 05/37] added socket thread which is able to start subprocess with connection to specific port with sockets --- .../parallel_event_server/socket_thread.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py new file mode 100644 index 0000000000..deede2ed03 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -0,0 +1,114 @@ +import os +import time +import signal +import socket +import threading +import subprocess +from pypeapp import Logger + + +class SocketThread(threading.Thread): + MAX_TIMEOUT = 30 + def __init__(self, name, port, filepath): + super(SocketThread, self).__init__() + self.log = Logger().get_logger("SocketThread", "Event Thread") + self.setName(name) + self.name = name + self.port = port + self.filepath = filepath + self.sock = None + self.subproc = None + self.connection = None + self._is_running = False + self.finished = False + + def stop(self): + self._is_running = False + super().stop() + + def process_to_die(self): + if not self.connection: + self.stop() + return + + self.connection.sendall(b"ptd") + + def run(self): + self._is_running = True + time_socket = time.time() + # Create a TCP/IP socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = sock + + # Bind the socket to the port + server_address = ("localhost", self.port) + sock.bind(server_address) + self.log.debug( + "Running Socked thread on {}:{}".format(*server_address) + ) + + self.subproc = subprocess.Popen( + ["python", self.filepath, "-port", str(self.port)], + stdout=subprocess.PIPE + ) + + # Listen for incoming connections + sock.listen(1) + sock.settimeout(1.0) + while True: + if not self._is_running: + break + try: + connection, client_address = sock.accept() + time_socket = time.time() + connection.settimeout(1.0) + self.connection = connection + + except socket.timeout: + if (time.time() - time_socket) > self.MAX_TIMEOUT: + self.log.error("Connection timeout passed. Terminating.") + self._is_running = False + os.kill(self.subproc.pid, signal.SIGINT) + break + continue + + try: + time_con = time.time() + # Receive the data in small chunks and retransmit it + while True: + try: + if not self._is_running: + break + try: + data = connection.recv(16) + time_con = time.time() + + except socket.timeout: + if (time.time() - time_con) > self.MAX_TIMEOUT: + self.log.error( + "Connection timeout passed. Terminating." + ) + self._is_running = False + os.kill(self.subproc.pid, signal.SIGINT) + break + continue + + except ConnectionResetError: + self._is_running = False + break + + if data: + connection.sendall(data) + + except Exception as exc: + self.log.error( + "Event server process failed", exc_info=True + ) + + finally: + # Clean up the connection + connection.close() + if self.subproc.poll() is None: + os.kill(self.subproc.pid, signal.SIGINT) + + self.finished = True From 631dcb56ea4959fd1f592feb6377e284849d612e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:12:15 +0200 Subject: [PATCH 06/37] added modified ftrack sessions, one for storing, second for processing events --- .../session_processor.py | 259 ++++++++++++++++++ .../parallel_event_server/session_storer.py | 257 +++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/session_processor.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/session_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py new file mode 100644 index 0000000000..55bdf0fba9 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py @@ -0,0 +1,259 @@ +import logging +import os +import atexit +import tempfile +import threading +import time +import requests +import queue + +import ftrack_api +import ftrack_api.session +import ftrack_api.cache +import ftrack_api.operation +import ftrack_api._centralized_storage_scenario +import ftrack_api.event +from ftrack_api.logging import LazyLogMessage as L + +from pype.ftrack.lib.custom_db_connector import DbConnector + + +class ProcessEventHub(ftrack_api.event.hub.EventHub): + dbcon = DbConnector( + mongo_url=os.environ["AVALON_MONGO"], + database_name="pype" + ) + table_name = "ftrack_events" + is_table_created = False + + def __init__(self, *args, **kwargs): + self.sock = kwargs.pop("sock") + super(ProcessEventHub, self).__init__(*args, **kwargs) + + def prepare_dbcon(self): + self.dbcon.install() + if not self.is_table_created: + self.dbcon.create_table(self.table_name, capped=False) + self.dbcon.active_table = self.table_name + self.is_table_created = True + + def wait(self, duration=None): + '''Wait for events and handle as they arrive. + + If *duration* is specified, then only process events until duration is + reached. *duration* is in seconds though float values can be used for + smaller values. + + ''' + started = time.time() + + self.prepare_dbcon() + while True: + try: + event = self._event_queue.get(timeout=0.1) + except queue.Empty: + if not self.load_events(): + time.sleep(0.5) + else: + self._handle(event) + self.dbcon.update_one( + {"id": event["id"]}, + {"$set": {"pype_data.is_processed": True}} + ) + # Additional special processing of events. + if event['topic'] == 'ftrack.meta.disconnected': + break + + if duration is not None: + if (time.time() - started) > duration: + break + + def load_events(self): + not_processed_events = self.dbcon.find({"pype_data.is_processed": False}) + found = False + for event_data in not_processed_events: + new_event_data = { + k: v for k, v in event_data.items() + if k not in ["_id", "pype_data"] + } + try: + event = ftrack_api.event.base.Event(**new_event_data) + print(event) + except Exception: + self.logger.exception(L( + 'Failed to convert payload into event: {0}', + event_data + )) + continue + found = True + self._event_queue.put(event) + + return found + + def _handle_packet(self, code, packet_identifier, path, data): + '''Handle packet received from server.''' + # if self.is_waiting: + code_name = self._code_name_mapping[code] + if code_name == "event": + return + if code_name == "heartbeat": + self.sock.sendall(b"processor") + return self._send_packet(self._code_name_mapping["heartbeat"]) + + return super()._handle_packet(code, packet_identifier, path, data) + + +class ProcessSession(ftrack_api.session.Session): + '''An isolated session for interaction with an ftrack server.''' + def __init__( + self, server_url=None, api_key=None, api_user=None, auto_populate=True, + plugin_paths=None, cache=None, cache_key_maker=None, + auto_connect_event_hub=None, schema_cache_path=None, + plugin_arguments=None, sock=None + ): + super(ftrack_api.session.Session, self).__init__() + self.logger = logging.getLogger( + __name__ + '.' + self.__class__.__name__ + ) + self._closed = False + + if server_url is None: + server_url = os.environ.get('FTRACK_SERVER') + + if not server_url: + raise TypeError( + 'Required "server_url" not specified. Pass as argument or set ' + 'in environment variable FTRACK_SERVER.' + ) + + self._server_url = server_url + + if api_key is None: + api_key = os.environ.get( + 'FTRACK_API_KEY', + # Backwards compatibility + os.environ.get('FTRACK_APIKEY') + ) + + if not api_key: + raise TypeError( + 'Required "api_key" not specified. Pass as argument or set in ' + 'environment variable FTRACK_API_KEY.' + ) + + self._api_key = api_key + + if api_user is None: + api_user = os.environ.get('FTRACK_API_USER') + if not api_user: + try: + api_user = getpass.getuser() + except Exception: + pass + + if not api_user: + raise TypeError( + 'Required "api_user" not specified. Pass as argument, set in ' + 'environment variable FTRACK_API_USER or one of the standard ' + 'environment variables used by Python\'s getpass module.' + ) + + self._api_user = api_user + + # Currently pending operations. + self.recorded_operations = ftrack_api.operation.Operations() + self.record_operations = True + + self.cache_key_maker = cache_key_maker + if self.cache_key_maker is None: + self.cache_key_maker = ftrack_api.cache.StringKeyMaker() + + # Enforce always having a memory cache at top level so that the same + # in-memory instance is returned from session. + self.cache = ftrack_api.cache.LayeredCache([ + ftrack_api.cache.MemoryCache() + ]) + + if cache is not None: + if callable(cache): + cache = cache(self) + + if cache is not None: + self.cache.caches.append(cache) + + self._managed_request = None + self._request = requests.Session() + self._request.auth = ftrack_api.session.SessionAuthentication( + self._api_key, self._api_user + ) + + self.auto_populate = auto_populate + + # Fetch server information and in doing so also check credentials. + self._server_information = self._fetch_server_information() + + # Now check compatibility of server based on retrieved information. + self.check_server_compatibility() + + # Construct event hub and load plugins. + self._event_hub = ProcessEventHub( + self._server_url, + self._api_user, + self._api_key, + sock=sock + ) + + self._auto_connect_event_hub_thread = None + if auto_connect_event_hub in (None, True): + # Connect to event hub in background thread so as not to block main + # session usage waiting for event hub connection. + self._auto_connect_event_hub_thread = threading.Thread( + target=self._event_hub.connect + ) + self._auto_connect_event_hub_thread.daemon = True + self._auto_connect_event_hub_thread.start() + + # To help with migration from auto_connect_event_hub default changing + # from True to False. + self._event_hub._deprecation_warning_auto_connect = ( + auto_connect_event_hub is None + ) + + # Register to auto-close session on exit. + atexit.register(self.close) + + self._plugin_paths = plugin_paths + if self._plugin_paths is None: + self._plugin_paths = os.environ.get( + 'FTRACK_EVENT_PLUGIN_PATH', '' + ).split(os.pathsep) + + self._discover_plugins(plugin_arguments=plugin_arguments) + + # TODO: Make schemas read-only and non-mutable (or at least without + # rebuilding types)? + if schema_cache_path is not False: + if schema_cache_path is None: + schema_cache_path = os.environ.get( + 'FTRACK_API_SCHEMA_CACHE_PATH', tempfile.gettempdir() + ) + + schema_cache_path = os.path.join( + schema_cache_path, 'ftrack_api_schema_cache.json' + ) + + self.schemas = self._load_schemas(schema_cache_path) + self.types = self._build_entity_type_classes(self.schemas) + + ftrack_api._centralized_storage_scenario.register(self) + + self._configure_locations() + self.event_hub.publish( + ftrack_api.event.base.Event( + topic='ftrack.api.session.ready', + data=dict( + session=self + ) + ), + synchronous=True + ) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py new file mode 100644 index 0000000000..01b1228d55 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py @@ -0,0 +1,257 @@ +import logging +import os +import atexit +import tempfile +import threading +import requests + +import ftrack_api +import ftrack_api.session +import ftrack_api.cache +import ftrack_api.operation +import ftrack_api._centralized_storage_scenario +import ftrack_api.event +from ftrack_api.logging import LazyLogMessage as L + + +class StorerEventHub(ftrack_api.event.hub.EventHub): + def __init__(self, *args, **kwargs): + self.sock = kwargs.pop("sock") + super(StorerEventHub, self).__init__(*args, **kwargs) + + def _handle_packet(self, code, packet_identifier, path, data): + '''Handle packet received from server.''' + if self._code_name_mapping[code] == "heartbeat": + # Reply with heartbeat. + self.sock.sendall(b"storer") + return self._send_packet(self._code_name_mapping['heartbeat']) + + return super(StorerEventHub, self)._handle_packet( + code, packet_identifier, path, data + ) + + +class StorerSession(ftrack_api.session.Session): + '''An isolated session for interaction with an ftrack server.''' + def __init__( + self, server_url=None, api_key=None, api_user=None, auto_populate=True, + plugin_paths=None, cache=None, cache_key_maker=None, + auto_connect_event_hub=None, schema_cache_path=None, + plugin_arguments=None, sock=None + ): + '''Initialise session. + + *server_url* should be the URL of the ftrack server to connect to + including any port number. If not specified attempt to look up from + :envvar:`FTRACK_SERVER`. + + *api_key* should be the API key to use for authentication whilst + *api_user* should be the username of the user in ftrack to record + operations against. If not specified, *api_key* should be retrieved + from :envvar:`FTRACK_API_KEY` and *api_user* from + :envvar:`FTRACK_API_USER`. + + If *auto_populate* is True (the default), then accessing entity + attributes will cause them to be automatically fetched from the server + if they are not already. This flag can be changed on the session + directly at any time. + + *plugin_paths* should be a list of paths to search for plugins. If not + specified, default to looking up :envvar:`FTRACK_EVENT_PLUGIN_PATH`. + + *cache* should be an instance of a cache that fulfils the + :class:`ftrack_api.cache.Cache` interface and will be used as the cache + for the session. It can also be a callable that will be called with the + session instance as sole argument. The callable should return ``None`` + if a suitable cache could not be configured, but session instantiation + can continue safely. + + .. note:: + + The session will add the specified cache to a pre-configured layered + cache that specifies the top level cache as a + :class:`ftrack_api.cache.MemoryCache`. Therefore, it is unnecessary + to construct a separate memory cache for typical behaviour. Working + around this behaviour or removing the memory cache can lead to + unexpected behaviour. + + *cache_key_maker* should be an instance of a key maker that fulfils the + :class:`ftrack_api.cache.KeyMaker` interface and will be used to + generate keys for objects being stored in the *cache*. If not specified, + a :class:`~ftrack_api.cache.StringKeyMaker` will be used. + + If *auto_connect_event_hub* is True then embedded event hub will be + automatically connected to the event server and allow for publishing and + subscribing to **non-local** events. If False, then only publishing and + subscribing to **local** events will be possible until the hub is + manually connected using :meth:`EventHub.connect + `. + + .. note:: + + The event hub connection is performed in a background thread to + improve session startup time. If a registered plugin requires a + connected event hub then it should check the event hub connection + status explicitly. Subscribing to events does *not* require a + connected event hub. + + Enable schema caching by setting *schema_cache_path* to a folder path. + If not set, :envvar:`FTRACK_API_SCHEMA_CACHE_PATH` will be used to + determine the path to store cache in. If the environment variable is + also not specified then a temporary directory will be used. Set to + `False` to disable schema caching entirely. + + *plugin_arguments* should be an optional mapping (dict) of keyword + arguments to pass to plugin register functions upon discovery. If a + discovered plugin has a signature that is incompatible with the passed + arguments, the discovery mechanism will attempt to reduce the passed + arguments to only those that the plugin accepts. Note that a warning + will be logged in this case. + + ''' + super(ftrack_api.session.Session, self).__init__() + self.logger = logging.getLogger( + __name__ + '.' + self.__class__.__name__ + ) + self._closed = False + + if server_url is None: + server_url = os.environ.get('FTRACK_SERVER') + + if not server_url: + raise TypeError( + 'Required "server_url" not specified. Pass as argument or set ' + 'in environment variable FTRACK_SERVER.' + ) + + self._server_url = server_url + + if api_key is None: + api_key = os.environ.get( + 'FTRACK_API_KEY', + # Backwards compatibility + os.environ.get('FTRACK_APIKEY') + ) + + if not api_key: + raise TypeError( + 'Required "api_key" not specified. Pass as argument or set in ' + 'environment variable FTRACK_API_KEY.' + ) + + self._api_key = api_key + + if api_user is None: + api_user = os.environ.get('FTRACK_API_USER') + if not api_user: + try: + api_user = getpass.getuser() + except Exception: + pass + + if not api_user: + raise TypeError( + 'Required "api_user" not specified. Pass as argument, set in ' + 'environment variable FTRACK_API_USER or one of the standard ' + 'environment variables used by Python\'s getpass module.' + ) + + self._api_user = api_user + + # Currently pending operations. + self.recorded_operations = ftrack_api.operation.Operations() + self.record_operations = True + + self.cache_key_maker = cache_key_maker + if self.cache_key_maker is None: + self.cache_key_maker = ftrack_api.cache.StringKeyMaker() + + # Enforce always having a memory cache at top level so that the same + # in-memory instance is returned from session. + self.cache = ftrack_api.cache.LayeredCache([ + ftrack_api.cache.MemoryCache() + ]) + + if cache is not None: + if callable(cache): + cache = cache(self) + + if cache is not None: + self.cache.caches.append(cache) + + self._managed_request = None + self._request = requests.Session() + self._request.auth = ftrack_api.session.SessionAuthentication( + self._api_key, self._api_user + ) + + self.auto_populate = auto_populate + + # Fetch server information and in doing so also check credentials. + self._server_information = self._fetch_server_information() + + # Now check compatibility of server based on retrieved information. + self.check_server_compatibility() + + # Construct event hub and load plugins. + self._event_hub = StorerEventHub( + self._server_url, + self._api_user, + self._api_key, + sock=sock + ) + + self._auto_connect_event_hub_thread = None + if auto_connect_event_hub in (None, True): + # Connect to event hub in background thread so as not to block main + # session usage waiting for event hub connection. + self._auto_connect_event_hub_thread = threading.Thread( + target=self._event_hub.connect + ) + self._auto_connect_event_hub_thread.daemon = True + self._auto_connect_event_hub_thread.start() + + # To help with migration from auto_connect_event_hub default changing + # from True to False. + self._event_hub._deprecation_warning_auto_connect = ( + auto_connect_event_hub is None + ) + + # Register to auto-close session on exit. + atexit.register(self.close) + + self._plugin_paths = plugin_paths + if self._plugin_paths is None: + self._plugin_paths = os.environ.get( + 'FTRACK_EVENT_PLUGIN_PATH', '' + ).split(os.pathsep) + + self._discover_plugins(plugin_arguments=plugin_arguments) + + # TODO: Make schemas read-only and non-mutable (or at least without + # rebuilding types)? + if schema_cache_path is not False: + if schema_cache_path is None: + schema_cache_path = os.environ.get( + 'FTRACK_API_SCHEMA_CACHE_PATH', tempfile.gettempdir() + ) + + schema_cache_path = os.path.join( + schema_cache_path, 'ftrack_api_schema_cache.json' + ) + + self.schemas = self._load_schemas(schema_cache_path) + self.types = self._build_entity_type_classes(self.schemas) + + ftrack_api._centralized_storage_scenario.register(self) + + self._configure_locations() + self.event_hub.publish( + ftrack_api.event.base.Event( + topic='ftrack.api.session.ready', + data=dict( + session=self + ) + ), + synchronous=True + ) From 5c70f3eb13497dd42c2681097ed0ba99d7b762da Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 12:12:54 +0200 Subject: [PATCH 07/37] modified event_server_cli to be able run subprocesses and handle their liveability --- .../parallel_event_server/__init__.py | 0 .../parallel_event_server/event_server_cli.py | 380 ++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/__init__.py create mode 100644 pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/__init__.py b/pype/ftrack/ftrack_server/parallel_event_server/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py new file mode 100644 index 0000000000..b1955a9f91 --- /dev/null +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -0,0 +1,380 @@ +import os +import sys +import signal +import socket +import argparse +import time +from urllib.parse import urlparse + +import requests +from pype.vendor import ftrack_api +from pype.ftrack.lib import credentials +from pype.ftrack.ftrack_server import FtrackServer +from pypeapp import Logger +import socket_thread + +log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") + + +def check_ftrack_url(url, log_errors=True): + if not url: + log.error('Ftrack URL is not set!') + return None + + url = url.strip('/ ') + + if 'http' not in url: + if url.endswith('ftrackapp.com'): + url = 'https://' + url + else: + url = 'https://{0}.ftrackapp.com'.format(url) + try: + result = requests.get(url, allow_redirects=False) + except requests.exceptions.RequestException: + if log_errors: + log.error('Entered Ftrack URL is not accesible!') + return False + + if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): + if log_errors: + log.error('Entered Ftrack URL is not accesible!') + return False + + log.debug('Ftrack server {} is accessible.'.format(url)) + + return url + + +def check_mongo_url(host, port, log_error=False): + sock = None + try: + sock = socket.create_connection( + (host, port), + timeout=1 + ) + return True + except socket.error as err: + if log_error: + print("Can't connect to MongoDB at {}:{} because: {}".format( + host, port, err + )) + return False + finally: + if sock is not None: + sock.close() + + +def validate_credentials(url, user, api): + first_validation = True + if not user: + log.error('Ftrack Username is not set! Exiting.') + first_validation = False + if not api: + log.error('Ftrack API key is not set! Exiting.') + first_validation = False + if not first_validation: + return False + + try: + session = ftrack_api.Session( + server_url=url, + api_user=user, + api_key=api + ) + session.close() + except Exception as e: + log.error( + 'Can\'t log into Ftrack with used credentials:' + ' Ftrack server: "{}" // Username: {} // API key: {}'.format( + url, user, api + )) + return False + + log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + user, api + )) + return True + + +def process_event_paths(event_paths): + log.debug('Processing event paths: {}.'.format(str(event_paths))) + return_paths = [] + not_found = [] + if not event_paths: + return return_paths, not_found + + if isinstance(event_paths, str): + event_paths = event_paths.split(os.pathsep) + + for path in event_paths: + if os.path.exists(path): + return_paths.append(path) + else: + not_found.append(path) + + return os.pathsep.join(return_paths), not_found + + +def main_loop(ftrack_url, username, api_key, event_paths): + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + # Get mongo hostname and port for testing mongo connection + mongo_url = os.environ["AVALON_MONGO"].strip('/ ') + result = urlparse(mongo_url) + url_items = result.netloc.split("@") + mongo_url = url_items[0] + if len(url_items) == 2: + mongo_url = url_items[1] + + mongo_url = "://".join([result.scheme, mongo_url]) + result = urlparse(mongo_url) + + mongo_hostname = result.hostname + mongo_port = result.port + + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + # Threads data + storer_name = "StorerThread" + storer_port = 10005 + storer_path = "{}/sub_event_storer.py".format(file_path) + storer_thread = None + + processor_name = "ProcessorThread" + processor_port = 10006 + processor_path = "{}/sub_event_processor.py".format(file_path) + processor_thread = None + + backup_name = "BackupThread" + backup_port = 10007 + backup_path = "{}/sub_event_backup.py".format(file_path) + backup_thread = None + + ftrack_accessible = False + mongo_accessible = False + + # Main loop + while True: + # Check if accessible Ftrack and Mongo url + if not ftrack_accessible or not mongo_accessible: + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible: + time.sleep(1) + continue + + # Run backup thread which does not requeire mongo to work + if not mongo_accessible: + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + if backup_thread is None: + backup_thread = socket_thread.SocketThread( + backup_name, backup_port, backup_path + ) + backup_thread.start() + + else: + if backup_thread is not None: + if backup_thread.finished: + backup_thread.join() + backup_thread = None + else: + backup_thread.process_to_die() + + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False + + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False + + time.sleep(1) + + +def main(argv): + ''' + There are 4 values neccessary for event server: + 1.) Ftrack url - "studio.ftrackapp.com" + 2.) Username - "my.username" + 3.) API key - "apikey-long11223344-6665588-5565" + 4.) Path/s to events - "X:/path/to/folder/with/events" + + All these values can be entered with arguments or environment variables. + - arguments: + "-ftrackurl {url}" + "-ftrackuser {username}" + "-ftrackapikey {api key}" + "-ftrackeventpaths {path to events}" + - environment variables: + FTRACK_SERVER + FTRACK_API_USER + FTRACK_API_KEY + FTRACK_EVENTS_PATH + + Credentials (Username & API key): + - Credentials can be stored for auto load on next start + - To *Store/Update* these values add argument "-storecred" + - They will be stored to appsdir file when login is successful + - To *Update/Override* values with enviromnet variables is also needed to: + - *don't enter argument for that value* + - add argument "-noloadcred" (currently stored credentials won't be loaded) + + Order of getting values: + 1.) Arguments are always used when entered. + - entered values through args have most priority! (in each case) + 2.) Credentials are tried to load from appsdir file. + - skipped when credentials were entered through args or credentials + are not stored yet + - can be skipped with "-noloadcred" argument + 3.) Environment variables are last source of values. + - will try to get not yet set values from environments + + Best practice: + - set environment variables FTRACK_SERVER & FTRACK_EVENTS_PATH + - launch event_server_cli with args: + ~/event_server_cli.py -ftrackuser "{username}" -ftrackapikey "{API key}" -storecred + - next time launch event_server_cli.py only with set environment variables + FTRACK_SERVER & FTRACK_EVENTS_PATH + ''' + parser = argparse.ArgumentParser(description='Ftrack event server') + parser.add_argument( + "-ftrackurl", type=str, metavar='FTRACKURL', + help=( + "URL to ftrack server where events should handle" + " (default from environment: $FTRACK_SERVER)" + ) + ) + parser.add_argument( + "-ftrackuser", type=str, + help=( + "Username should be the username of the user in ftrack" + " to record operations against." + " (default from environment: $FTRACK_API_USER)" + ) + ) + parser.add_argument( + "-ftrackapikey", type=str, + help=( + "Should be the API key to use for authentication" + " (default from environment: $FTRACK_API_KEY)" + ) + ) + parser.add_argument( + "-ftrackeventpaths", nargs='+', + help=( + "List of paths where events are stored." + " (default from environment: $FTRACK_EVENTS_PATH)" + ) + ) + parser.add_argument( + '-storecred', + help=( + "Entered credentials will be also stored" + " to apps dir for future usage" + ), + action="store_true" + ) + parser.add_argument( + '-noloadcred', + help="Load creadentials from apps dir", + action="store_true" + ) + + ftrack_url = os.environ.get('FTRACK_SERVER') + username = os.environ.get('FTRACK_API_USER') + api_key = os.environ.get('FTRACK_API_KEY') + event_paths = os.environ.get('FTRACK_EVENTS_PATH') + + kwargs, args = parser.parse_known_args(argv) + + if kwargs.ftrackurl: + ftrack_url = kwargs.ftrackurl + + if kwargs.ftrackeventpaths: + event_paths = kwargs.ftrackeventpaths + + if not kwargs.noloadcred: + cred = credentials._get_credentials(True) + username = cred.get('username') + api_key = cred.get('apiKey') + + if kwargs.ftrackuser: + username = kwargs.ftrackuser + + if kwargs.ftrackapikey: + api_key = kwargs.ftrackapikey + + # Check url regex and accessibility + ftrack_url = check_ftrack_url(ftrack_url) + if not ftrack_url: + return 1 + + # Validate entered credentials + if not validate_credentials(ftrack_url, username, api_key): + return 1 + + # Process events path + event_paths, not_found = process_event_paths(event_paths) + if not_found: + log.warning( + 'These paths were not found: {}'.format(str(not_found)) + ) + if not event_paths: + if not_found: + log.error('Any of entered paths is valid or can be accesible.') + else: + log.error('Paths to events are not set. Exiting.') + return 1 + + if kwargs.storecred: + credentials._save_credentials(username, api_key, True) + + main_loop(ftrack_url, username, api_key, event_paths) + + +if (__name__ == ('__main__')): + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + + sys.exit(main(sys.argv)) From 28f2d1431856b395efcdc43d3d520cedbac1bc9e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 14:01:58 +0200 Subject: [PATCH 08/37] pype logging is not used in event_server_cli for cases when mongo is not accessible and removed backup solution --- .../parallel_event_server/event_server_cli.py | 131 ++++++++---------- 1 file changed, 56 insertions(+), 75 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index b1955a9f91..3814cd0549 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -10,15 +10,12 @@ import requests from pype.vendor import ftrack_api from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer -from pypeapp import Logger import socket_thread -log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") - def check_ftrack_url(url, log_errors=True): if not url: - log.error('Ftrack URL is not set!') + print('ERROR: Ftrack URL is not set!') return None url = url.strip('/ ') @@ -32,15 +29,15 @@ def check_ftrack_url(url, log_errors=True): result = requests.get(url, allow_redirects=False) except requests.exceptions.RequestException: if log_errors: - log.error('Entered Ftrack URL is not accesible!') + print('ERROR: Entered Ftrack URL is not accesible!') return False if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): if log_errors: - log.error('Entered Ftrack URL is not accesible!') + print('ERROR: Entered Ftrack URL is not accesible!') return False - log.debug('Ftrack server {} is accessible.'.format(url)) + print('DEBUG: Ftrack server {} is accessible.'.format(url)) return url @@ -67,10 +64,10 @@ def check_mongo_url(host, port, log_error=False): def validate_credentials(url, user, api): first_validation = True if not user: - log.error('Ftrack Username is not set! Exiting.') + print('ERROR: Ftrack Username is not set! Exiting.') first_validation = False if not api: - log.error('Ftrack API key is not set! Exiting.') + print('ERROR: Ftrack API key is not set! Exiting.') first_validation = False if not first_validation: return False @@ -83,21 +80,21 @@ def validate_credentials(url, user, api): ) session.close() except Exception as e: - log.error( - 'Can\'t log into Ftrack with used credentials:' + print( + 'ERROR: Can\'t log into Ftrack with used credentials:' ' Ftrack server: "{}" // Username: {} // API key: {}'.format( url, user, api )) return False - log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( user, api )) return True def process_event_paths(event_paths): - log.debug('Processing event paths: {}.'.format(str(event_paths))) + print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) return_paths = [] not_found = [] if not event_paths: @@ -150,80 +147,64 @@ def main_loop(ftrack_url, username, api_key, event_paths): processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None - backup_name = "BackupThread" - backup_port = 10007 - backup_path = "{}/sub_event_backup.py".format(file_path) - backup_thread = None - ftrack_accessible = False mongo_accessible = False + printed_ftrack_error = False + printed_mongo_error = False + # Main loop while True: # Check if accessible Ftrack and Mongo url - if not ftrack_accessible or not mongo_accessible: - if not ftrack_accessible: - ftrack_accessible = check_ftrack_url(ftrack_url) + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) - if not mongo_accessible: - mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) # Run threads only if Ftrack is accessible - if not ftrack_accessible: + if not ftrack_accessible or not mongo_accessible: + if not mongo_accessible and not printed_mongo_error: + print("Can't access Mongo {}".format(mongo_url)) + + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {}".format(ftrack_url)) + + printed_ftrack_error = True + printed_mongo_error = True + time.sleep(1) continue + printed_ftrack_error = False + printed_mongo_error = False + # Run backup thread which does not requeire mongo to work - if not mongo_accessible: - if storer_thread is not None: - storer_thread.stop() - storer_thread.join() - storer_thread = None + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() - if processor_thread is not None: - processor_thread.stop() - processor_thread.join() - processor_thread = None + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False - if backup_thread is None: - backup_thread = socket_thread.SocketThread( - backup_name, backup_port, backup_path - ) - backup_thread.start() + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() - else: - if backup_thread is not None: - if backup_thread.finished: - backup_thread.join() - backup_thread = None - else: - backup_thread.process_to_die() - - if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif not storer_thread.isAlive(): - storer_thread.join() - storer_thread = None - ftrack_accessible = False - mongo_accessible = False - - if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): - processor_thread.join() - processor_thread = None - ftrack_accessible = False - mongo_accessible = False + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False time.sleep(1) @@ -353,14 +334,14 @@ def main(argv): # Process events path event_paths, not_found = process_event_paths(event_paths) if not_found: - log.warning( - 'These paths were not found: {}'.format(str(not_found)) + print( + 'WARNING: These paths were not found: {}'.format(str(not_found)) ) if not event_paths: if not_found: - log.error('Any of entered paths is valid or can be accesible.') + print('ERROR: Any of entered paths is valid or can be accesible.') else: - log.error('Paths to events are not set. Exiting.') + print('ERROR: Paths to events are not set. Exiting.') return 1 if kwargs.storecred: From bcf8bf17a521fb8b0643fbc786317bec192bfefb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Oct 2019 14:02:49 +0200 Subject: [PATCH 09/37] session instance validation also checks for process session and raises exception if does not match ftrack_api session --- pype/ftrack/lib/ftrack_base_handler.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 9821bfd81f..05c84eea84 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -32,18 +32,18 @@ class BaseHandler(object): def __init__(self, session, plugins_presets={}): '''Expects a ftrack_api.Session instance''' self.log = Logger().get_logger(self.__class__.__name__) - if not isinstance(session, ftrack_api.session.Session): - self.log.warning(( + if not( + isinstance(session, ftrack_api.session.Session) or + isinstance(session, ProcessSession) + ): + raise Exception(( "Session object entered with args is instance of \"{}\"" - " but expected instance is \"{}\"." + " but expected instances are \"{}\" and \"{}\"" ).format( str(type(session)), - str(ftrack_api.session.Session.__qualname__) + str(ftrack_api.session.Session), + str(ProcessSession) )) - self.register = self.register_without_session - self._session = None - - return self._session = session From b7c29540612443d77f435a00e9d628a089c1e5d0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:26:31 +0200 Subject: [PATCH 10/37] changed default ports --- .../ftrack_server/parallel_event_server/event_server_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index 3814cd0549..15afb6672d 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -138,12 +138,12 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Threads data storer_name = "StorerThread" - storer_port = 10005 + storer_port = 10001 storer_path = "{}/sub_event_storer.py".format(file_path) storer_thread = None processor_name = "ProcessorThread" - processor_port = 10006 + processor_port = 10011 processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None From d1bfa2412e417b455ba716e42e180d964aec7339 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:33:01 +0200 Subject: [PATCH 11/37] pymongo connection error cause that subprocess will end --- .../session_processor.py | 28 +++++++++++-------- .../parallel_event_server/sub_event_storer.py | 13 ++++++--- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py index 55bdf0fba9..d51da28938 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py @@ -31,11 +31,14 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): super(ProcessEventHub, self).__init__(*args, **kwargs) def prepare_dbcon(self): - self.dbcon.install() - if not self.is_table_created: - self.dbcon.create_table(self.table_name, capped=False) - self.dbcon.active_table = self.table_name - self.is_table_created = True + try: + self.dbcon.install() + if not self.is_table_created: + self.dbcon.create_table(self.table_name, capped=False) + self.dbcon.active_table = self.table_name + self.is_table_created = True + except pymongo.errors.AutoReconnect: + sys.exit(0) def wait(self, duration=None): '''Wait for events and handle as they arrive. @@ -46,7 +49,6 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): ''' started = time.time() - self.prepare_dbcon() while True: try: @@ -55,11 +57,14 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): if not self.load_events(): time.sleep(0.5) else: - self._handle(event) - self.dbcon.update_one( - {"id": event["id"]}, - {"$set": {"pype_data.is_processed": True}} - ) + try: + self._handle(event) + self.dbcon.update_one( + {"id": event["id"]}, + {"$set": {"pype_data.is_processed": True}} + ) + except pymongo.errors.AutoReconnect: + sys.exit(0) # Additional special processing of events. if event['topic'] == 'ftrack.meta.disconnected': break @@ -78,7 +83,6 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): } try: event = ftrack_api.event.base.Event(**new_event_data) - print(event) except Exception: self.logger.exception(L( 'Failed to convert payload into event: {0}', diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py index 295504f389..bf2dd0eefa 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -23,9 +23,12 @@ table_name = "ftrack_events" ignore_topics = [] def install_db(): - dbcon.install() - dbcon.create_table(table_name, capped=False) - dbcon.active_table = table_name + try: + dbcon.install() + dbcon.create_table(table_name, capped=False) + dbcon.active_table = table_name + except pymongo.errors.AutoReconnect: + sys.exit(0) def launch(event): if event.get("topic") in ignore_topics: @@ -47,6 +50,9 @@ def launch(event): except pymongo.errors.DuplicateKeyError: log.debug("Event: {} already exists".format(event_id)) + except pymongo.errors.AutoReconnect: + sys.exit(0) + except Exception as exc: log.error( "Event: {} failed to store".format(event_id), @@ -76,7 +82,6 @@ def main(args): session = StorerSession(auto_connect_event_hub=True, sock=sock) register(session) server = FtrackServer("event") - log.info(os.environ["FTRACK_EVENTS_PATH"]) log.debug("Launched Ftrack Event storer") server.run_server(session, load_files=False) From 4921783d1a25976275dc0ebf22333d0114d2b7e7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:36:54 +0200 Subject: [PATCH 12/37] terminate signal is registered too to be able terminate subprocesses --- .../ftrack_server/parallel_event_server/socket_thread.py | 6 +++--- .../parallel_event_server/sub_event_processor.py | 1 + .../ftrack_server/parallel_event_server/sub_event_storer.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py index deede2ed03..1ec4eca57f 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -68,7 +68,7 @@ class SocketThread(threading.Thread): if (time.time() - time_socket) > self.MAX_TIMEOUT: self.log.error("Connection timeout passed. Terminating.") self._is_running = False - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() break continue @@ -89,7 +89,7 @@ class SocketThread(threading.Thread): "Connection timeout passed. Terminating." ) self._is_running = False - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() break continue @@ -109,6 +109,6 @@ class SocketThread(threading.Thread): # Clean up the connection connection.close() if self.subproc.poll() is None: - os.kill(self.subproc.pid, signal.SIGINT) + self.subproc.terminate() self.finished = True diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py index bd3118a893..a795775a94 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -42,5 +42,6 @@ if __name__ == "__main__": sys.exit(0) signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) main(sys.argv) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py index bf2dd0eefa..864b4b7c1d 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py @@ -98,5 +98,6 @@ if __name__ == "__main__": sys.exit(0) signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) main(sys.argv) From f1c873fedc7ec05bb4d67436d534e0348e6fd376 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:38:32 +0200 Subject: [PATCH 13/37] minor fixes, occupied socket ports are skipped and logger is renamed --- .../parallel_event_server/event_server_cli.py | 10 ++++++++++ .../parallel_event_server/socket_thread.py | 13 +++++++++---- .../parallel_event_server/sub_event_processor.py | 2 +- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py index 15afb6672d..55670178e2 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py @@ -170,6 +170,16 @@ def main_loop(ftrack_url, username, api_key, event_paths): if not ftrack_accessible and not printed_ftrack_error: print("Can't access Ftrack {}".format(ftrack_url)) + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + printed_ftrack_error = True printed_mongo_error = True diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py index 1ec4eca57f..e787004afc 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py @@ -24,7 +24,6 @@ class SocketThread(threading.Thread): def stop(self): self._is_running = False - super().stop() def process_to_die(self): if not self.connection: @@ -40,9 +39,15 @@ class SocketThread(threading.Thread): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock = sock - # Bind the socket to the port - server_address = ("localhost", self.port) - sock.bind(server_address) + # Bind the socket to the port - skip already used ports + while True: + try: + server_address = ("localhost", self.port) + sock.bind(server_address) + break + except OSError: + self.port += 1 + self.log.debug( "Running Socked thread on {}:{}".format(*server_address) ) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py index a795775a94..c94223deae 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py @@ -9,7 +9,7 @@ from pype.ftrack.ftrack_server import FtrackServer from session_processor import ProcessSession from pypeapp import Logger -log = Logger().get_logger("Event storer") +log = Logger().get_logger("Event processor") def main(args): From 4290af23e7845e981aa80898a90d21d2bfa49969 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 14:57:53 +0200 Subject: [PATCH 14/37] moved event server files in hierarchy --- pype/ftrack/ftrack_server/__init__.py | 11 +- pype/ftrack/ftrack_server/event_server_cli.py | 193 +++++++-- .../parallel_event_server/__init__.py | 0 .../parallel_event_server/event_server_cli.py | 371 ------------------ .../session_processor.py | 0 .../session_storer.py | 0 .../socket_thread.py | 0 .../sub_event_processor.py | 6 +- .../sub_event_storer.py | 7 +- pype/ftrack/lib/ftrack_base_handler.py | 5 +- 10 files changed, 176 insertions(+), 417 deletions(-) delete mode 100644 pype/ftrack/ftrack_server/parallel_event_server/__init__.py delete mode 100644 pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py rename pype/ftrack/ftrack_server/{parallel_event_server => }/session_processor.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/session_storer.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/socket_thread.py (100%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/sub_event_processor.py (86%) rename pype/ftrack/ftrack_server/{parallel_event_server => }/sub_event_storer.py (90%) diff --git a/pype/ftrack/ftrack_server/__init__.py b/pype/ftrack/ftrack_server/__init__.py index 4a64ab8848..f8f876bbb6 100644 --- a/pype/ftrack/ftrack_server/__init__.py +++ b/pype/ftrack/ftrack_server/__init__.py @@ -1,7 +1,8 @@ from .ftrack_server import FtrackServer -from . import event_server_cli -__all__ = [ - 'event_server_cli', - 'FtrackServer' -] +import event_server_cli +import session_processor +import session_storer +import socket_thread +import sub_event_processor +import sub_event_storer diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index e06a626468..83ecd0d47a 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -1,18 +1,21 @@ import os import sys +import signal +import socket import argparse +import time +from urllib.parse import urlparse + import requests from pype.vendor import ftrack_api -from pype.ftrack import credentials +from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer -from pypeapp import Logger - -log = Logger().get_logger('Ftrack event server', "ftrack-event-server-cli") +import socket_thread -def check_url(url): +def check_ftrack_url(url, log_errors=True): if not url: - log.error('Ftrack URL is not set!') + print('ERROR: Ftrack URL is not set!') return None url = url.strip('/ ') @@ -25,24 +28,46 @@ def check_url(url): try: result = requests.get(url, allow_redirects=False) except requests.exceptions.RequestException: - log.error('Entered Ftrack URL is not accesible!') - return None + if log_errors: + print('ERROR: Entered Ftrack URL is not accesible!') + return False if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): - log.error('Entered Ftrack URL is not accesible!') - return None + if log_errors: + print('ERROR: Entered Ftrack URL is not accesible!') + return False - log.debug('Ftrack server {} is accessible.'.format(url)) + print('DEBUG: Ftrack server {} is accessible.'.format(url)) return url + +def check_mongo_url(host, port, log_error=False): + sock = None + try: + sock = socket.create_connection( + (host, port), + timeout=1 + ) + return True + except socket.error as err: + if log_error: + print("Can't connect to MongoDB at {}:{} because: {}".format( + host, port, err + )) + return False + finally: + if sock is not None: + sock.close() + + def validate_credentials(url, user, api): first_validation = True if not user: - log.error('Ftrack Username is not set! Exiting.') + print('ERROR: Ftrack Username is not set! Exiting.') first_validation = False if not api: - log.error('Ftrack API key is not set! Exiting.') + print('ERROR: Ftrack API key is not set! Exiting.') first_validation = False if not first_validation: return False @@ -55,21 +80,21 @@ def validate_credentials(url, user, api): ) session.close() except Exception as e: - log.error( - 'Can\'t log into Ftrack with used credentials:' + print( + 'ERROR: Can\'t log into Ftrack with used credentials:' ' Ftrack server: "{}" // Username: {} // API key: {}'.format( url, user, api )) return False - log.debug('Credentials Username: "{}", API key: "{}" are valid.'.format( + print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( user, api )) return True def process_event_paths(event_paths): - log.debug('Processing event paths: {}.'.format(str(event_paths))) + print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) return_paths = [] not_found = [] if not event_paths: @@ -87,14 +112,112 @@ def process_event_paths(event_paths): return os.pathsep.join(return_paths), not_found -def run_event_server(ftrack_url, username, api_key, event_paths): - os.environ['FTRACK_SERVER'] = ftrack_url - os.environ['FTRACK_API_USER'] = username - os.environ['FTRACK_API_KEY'] = api_key - os.environ['FTRACK_EVENTS_PATH'] = event_paths +def main_loop(ftrack_url, username, api_key, event_paths): + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + # Get mongo hostname and port for testing mongo connection + mongo_url = os.environ["AVALON_MONGO"].strip('/ ') + result = urlparse(mongo_url) + url_items = result.netloc.split("@") + mongo_url = url_items[0] + if len(url_items) == 2: + mongo_url = url_items[1] + + mongo_url = "://".join([result.scheme, mongo_url]) + result = urlparse(mongo_url) + + mongo_hostname = result.hostname + mongo_port = result.port + + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + # Threads data + storer_name = "StorerThread" + storer_port = 10001 + storer_path = "{}/sub_event_storer.py".format(file_path) + storer_thread = None + + processor_name = "ProcessorThread" + processor_port = 10011 + processor_path = "{}/sub_event_processor.py".format(file_path) + processor_thread = None + + ftrack_accessible = False + mongo_accessible = False + + printed_ftrack_error = False + printed_mongo_error = False + + # Main loop + while True: + # Check if accessible Ftrack and Mongo url + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + if not mongo_accessible: + mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible or not mongo_accessible: + if not mongo_accessible and not printed_mongo_error: + print("Can't access Mongo {}".format(mongo_url)) + + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {}".format(ftrack_url)) + + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + printed_ftrack_error = True + printed_mongo_error = True + + time.sleep(1) + continue + + printed_ftrack_error = False + printed_mongo_error = False + + # Run backup thread which does not requeire mongo to work + if storer_thread is None: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif not storer_thread.isAlive(): + storer_thread.join() + storer_thread = None + ftrack_accessible = False + mongo_accessible = False + + if processor_thread is None: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + # If thread failed test Ftrack and Mongo connection + elif processor_thread.isAlive(): + processor_thread.join() + processor_thread = None + ftrack_accessible = False + mongo_accessible = False + + time.sleep(1) - server = FtrackServer('event') - server.run_server() def main(argv): ''' @@ -210,7 +333,7 @@ def main(argv): api_key = kwargs.ftrackapikey # Check url regex and accessibility - ftrack_url = check_url(ftrack_url) + ftrack_url = check_ftrack_url(ftrack_url) if not ftrack_url: return 1 @@ -221,21 +344,29 @@ def main(argv): # Process events path event_paths, not_found = process_event_paths(event_paths) if not_found: - log.warning( - 'These paths were not found: {}'.format(str(not_found)) + print( + 'WARNING: These paths were not found: {}'.format(str(not_found)) ) if not event_paths: if not_found: - log.error('Any of entered paths is valid or can be accesible.') + print('ERROR: Any of entered paths is valid or can be accesible.') else: - log.error('Paths to events are not set. Exiting.') + print('ERROR: Paths to events are not set. Exiting.') return 1 if kwargs.storecred: credentials._save_credentials(username, api_key, True) - run_event_server(ftrack_url, username, api_key, event_paths) + main_loop(ftrack_url, username, api_key, event_paths) -if (__name__ == ('__main__')): +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/__init__.py b/pype/ftrack/ftrack_server/parallel_event_server/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py b/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py deleted file mode 100644 index 55670178e2..0000000000 --- a/pype/ftrack/ftrack_server/parallel_event_server/event_server_cli.py +++ /dev/null @@ -1,371 +0,0 @@ -import os -import sys -import signal -import socket -import argparse -import time -from urllib.parse import urlparse - -import requests -from pype.vendor import ftrack_api -from pype.ftrack.lib import credentials -from pype.ftrack.ftrack_server import FtrackServer -import socket_thread - - -def check_ftrack_url(url, log_errors=True): - if not url: - print('ERROR: Ftrack URL is not set!') - return None - - url = url.strip('/ ') - - if 'http' not in url: - if url.endswith('ftrackapp.com'): - url = 'https://' + url - else: - url = 'https://{0}.ftrackapp.com'.format(url) - try: - result = requests.get(url, allow_redirects=False) - except requests.exceptions.RequestException: - if log_errors: - print('ERROR: Entered Ftrack URL is not accesible!') - return False - - if (result.status_code != 200 or 'FTRACK_VERSION' not in result.headers): - if log_errors: - print('ERROR: Entered Ftrack URL is not accesible!') - return False - - print('DEBUG: Ftrack server {} is accessible.'.format(url)) - - return url - - -def check_mongo_url(host, port, log_error=False): - sock = None - try: - sock = socket.create_connection( - (host, port), - timeout=1 - ) - return True - except socket.error as err: - if log_error: - print("Can't connect to MongoDB at {}:{} because: {}".format( - host, port, err - )) - return False - finally: - if sock is not None: - sock.close() - - -def validate_credentials(url, user, api): - first_validation = True - if not user: - print('ERROR: Ftrack Username is not set! Exiting.') - first_validation = False - if not api: - print('ERROR: Ftrack API key is not set! Exiting.') - first_validation = False - if not first_validation: - return False - - try: - session = ftrack_api.Session( - server_url=url, - api_user=user, - api_key=api - ) - session.close() - except Exception as e: - print( - 'ERROR: Can\'t log into Ftrack with used credentials:' - ' Ftrack server: "{}" // Username: {} // API key: {}'.format( - url, user, api - )) - return False - - print('DEBUG: Credentials Username: "{}", API key: "{}" are valid.'.format( - user, api - )) - return True - - -def process_event_paths(event_paths): - print('DEBUG: Processing event paths: {}.'.format(str(event_paths))) - return_paths = [] - not_found = [] - if not event_paths: - return return_paths, not_found - - if isinstance(event_paths, str): - event_paths = event_paths.split(os.pathsep) - - for path in event_paths: - if os.path.exists(path): - return_paths.append(path) - else: - not_found.append(path) - - return os.pathsep.join(return_paths), not_found - - -def main_loop(ftrack_url, username, api_key, event_paths): - # Set Ftrack environments - os.environ["FTRACK_SERVER"] = ftrack_url - os.environ["FTRACK_API_USER"] = username - os.environ["FTRACK_API_KEY"] = api_key - os.environ["FTRACK_EVENTS_PATH"] = event_paths - - # Get mongo hostname and port for testing mongo connection - mongo_url = os.environ["AVALON_MONGO"].strip('/ ') - result = urlparse(mongo_url) - url_items = result.netloc.split("@") - mongo_url = url_items[0] - if len(url_items) == 2: - mongo_url = url_items[1] - - mongo_url = "://".join([result.scheme, mongo_url]) - result = urlparse(mongo_url) - - mongo_hostname = result.hostname - mongo_port = result.port - - # Current file - file_path = os.path.dirname(os.path.realpath(__file__)) - - # Threads data - storer_name = "StorerThread" - storer_port = 10001 - storer_path = "{}/sub_event_storer.py".format(file_path) - storer_thread = None - - processor_name = "ProcessorThread" - processor_port = 10011 - processor_path = "{}/sub_event_processor.py".format(file_path) - processor_thread = None - - ftrack_accessible = False - mongo_accessible = False - - printed_ftrack_error = False - printed_mongo_error = False - - # Main loop - while True: - # Check if accessible Ftrack and Mongo url - if not ftrack_accessible: - ftrack_accessible = check_ftrack_url(ftrack_url) - - if not mongo_accessible: - mongo_accessible = check_mongo_url(mongo_hostname, mongo_port) - - # Run threads only if Ftrack is accessible - if not ftrack_accessible or not mongo_accessible: - if not mongo_accessible and not printed_mongo_error: - print("Can't access Mongo {}".format(mongo_url)) - - if not ftrack_accessible and not printed_ftrack_error: - print("Can't access Ftrack {}".format(ftrack_url)) - - if storer_thread is not None: - storer_thread.stop() - storer_thread.join() - storer_thread = None - - if processor_thread is not None: - processor_thread.stop() - processor_thread.join() - processor_thread = None - - printed_ftrack_error = True - printed_mongo_error = True - - time.sleep(1) - continue - - printed_ftrack_error = False - printed_mongo_error = False - - # Run backup thread which does not requeire mongo to work - if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif not storer_thread.isAlive(): - storer_thread.join() - storer_thread = None - ftrack_accessible = False - mongo_accessible = False - - if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() - - # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): - processor_thread.join() - processor_thread = None - ftrack_accessible = False - mongo_accessible = False - - time.sleep(1) - - -def main(argv): - ''' - There are 4 values neccessary for event server: - 1.) Ftrack url - "studio.ftrackapp.com" - 2.) Username - "my.username" - 3.) API key - "apikey-long11223344-6665588-5565" - 4.) Path/s to events - "X:/path/to/folder/with/events" - - All these values can be entered with arguments or environment variables. - - arguments: - "-ftrackurl {url}" - "-ftrackuser {username}" - "-ftrackapikey {api key}" - "-ftrackeventpaths {path to events}" - - environment variables: - FTRACK_SERVER - FTRACK_API_USER - FTRACK_API_KEY - FTRACK_EVENTS_PATH - - Credentials (Username & API key): - - Credentials can be stored for auto load on next start - - To *Store/Update* these values add argument "-storecred" - - They will be stored to appsdir file when login is successful - - To *Update/Override* values with enviromnet variables is also needed to: - - *don't enter argument for that value* - - add argument "-noloadcred" (currently stored credentials won't be loaded) - - Order of getting values: - 1.) Arguments are always used when entered. - - entered values through args have most priority! (in each case) - 2.) Credentials are tried to load from appsdir file. - - skipped when credentials were entered through args or credentials - are not stored yet - - can be skipped with "-noloadcred" argument - 3.) Environment variables are last source of values. - - will try to get not yet set values from environments - - Best practice: - - set environment variables FTRACK_SERVER & FTRACK_EVENTS_PATH - - launch event_server_cli with args: - ~/event_server_cli.py -ftrackuser "{username}" -ftrackapikey "{API key}" -storecred - - next time launch event_server_cli.py only with set environment variables - FTRACK_SERVER & FTRACK_EVENTS_PATH - ''' - parser = argparse.ArgumentParser(description='Ftrack event server') - parser.add_argument( - "-ftrackurl", type=str, metavar='FTRACKURL', - help=( - "URL to ftrack server where events should handle" - " (default from environment: $FTRACK_SERVER)" - ) - ) - parser.add_argument( - "-ftrackuser", type=str, - help=( - "Username should be the username of the user in ftrack" - " to record operations against." - " (default from environment: $FTRACK_API_USER)" - ) - ) - parser.add_argument( - "-ftrackapikey", type=str, - help=( - "Should be the API key to use for authentication" - " (default from environment: $FTRACK_API_KEY)" - ) - ) - parser.add_argument( - "-ftrackeventpaths", nargs='+', - help=( - "List of paths where events are stored." - " (default from environment: $FTRACK_EVENTS_PATH)" - ) - ) - parser.add_argument( - '-storecred', - help=( - "Entered credentials will be also stored" - " to apps dir for future usage" - ), - action="store_true" - ) - parser.add_argument( - '-noloadcred', - help="Load creadentials from apps dir", - action="store_true" - ) - - ftrack_url = os.environ.get('FTRACK_SERVER') - username = os.environ.get('FTRACK_API_USER') - api_key = os.environ.get('FTRACK_API_KEY') - event_paths = os.environ.get('FTRACK_EVENTS_PATH') - - kwargs, args = parser.parse_known_args(argv) - - if kwargs.ftrackurl: - ftrack_url = kwargs.ftrackurl - - if kwargs.ftrackeventpaths: - event_paths = kwargs.ftrackeventpaths - - if not kwargs.noloadcred: - cred = credentials._get_credentials(True) - username = cred.get('username') - api_key = cred.get('apiKey') - - if kwargs.ftrackuser: - username = kwargs.ftrackuser - - if kwargs.ftrackapikey: - api_key = kwargs.ftrackapikey - - # Check url regex and accessibility - ftrack_url = check_ftrack_url(ftrack_url) - if not ftrack_url: - return 1 - - # Validate entered credentials - if not validate_credentials(ftrack_url, username, api_key): - return 1 - - # Process events path - event_paths, not_found = process_event_paths(event_paths) - if not_found: - print( - 'WARNING: These paths were not found: {}'.format(str(not_found)) - ) - if not event_paths: - if not_found: - print('ERROR: Any of entered paths is valid or can be accesible.') - else: - print('ERROR: Paths to events are not set. Exiting.') - return 1 - - if kwargs.storecred: - credentials._save_credentials(username, api_key, True) - - main_loop(ftrack_url, username, api_key, event_paths) - - -if (__name__ == ('__main__')): - # Register interupt signal - def signal_handler(sig, frame): - print("You pressed Ctrl+C. Process ended.") - sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - - sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/session_processor.py rename to pype/ftrack/ftrack_server/session_processor.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/session_storer.py b/pype/ftrack/ftrack_server/session_storer.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/session_storer.py rename to pype/ftrack/ftrack_server/session_storer.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py similarity index 100% rename from pype/ftrack/ftrack_server/parallel_event_server/socket_thread.py rename to pype/ftrack/ftrack_server/socket_thread.py diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py similarity index 86% rename from pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py rename to pype/ftrack/ftrack_server/sub_event_processor.py index c94223deae..b53c294981 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -5,8 +5,8 @@ import signal import socket import pymongo -from pype.ftrack.ftrack_server import FtrackServer -from session_processor import ProcessSession +from ftrack_server import FtrackServer +import session_processor from pypeapp import Logger log = Logger().get_logger("Event processor") @@ -24,7 +24,7 @@ def main(args): sock.sendall(b"CreatedProcess") try: - session = ProcessSession(auto_connect_event_hub=True, sock=sock) + session = session_processor.ProcessSession(auto_connect_event_hub=True, sock=sock) server = FtrackServer('event') log.debug("Launched Ftrack Event processor") server.run_server(session) diff --git a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py similarity index 90% rename from pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py rename to pype/ftrack/ftrack_server/sub_event_storer.py index 864b4b7c1d..918e1e1bea 100644 --- a/pype/ftrack/ftrack_server/parallel_event_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -5,9 +5,9 @@ import signal import socket import pymongo -from pype.ftrack.ftrack_server import FtrackServer +from ftrack_server import FtrackServer from pype.ftrack.lib.custom_db_connector import DbConnector -from pype.ftrack.ftrack_server.parallel_event_server.session_storer import StorerSession +from session_storer import StorerSession from pypeapp import Logger log = Logger().get_logger("Event storer") @@ -47,9 +47,6 @@ def launch(event): dbcon.update({"id": event_id}, event_data, upsert=True) log.debug("Event: {} stored".format(event_id)) - except pymongo.errors.DuplicateKeyError: - log.debug("Event: {} already exists".format(event_id)) - except pymongo.errors.AutoReconnect: sys.exit(0) diff --git a/pype/ftrack/lib/ftrack_base_handler.py b/pype/ftrack/lib/ftrack_base_handler.py index 05c84eea84..13e1cae9a9 100644 --- a/pype/ftrack/lib/ftrack_base_handler.py +++ b/pype/ftrack/lib/ftrack_base_handler.py @@ -3,6 +3,7 @@ import time from pypeapp import Logger from pype.vendor import ftrack_api from pype.vendor.ftrack_api import session as fa_session +from pype.ftrack.ftrack_server import session_processor class MissingPermision(Exception): @@ -34,7 +35,7 @@ class BaseHandler(object): self.log = Logger().get_logger(self.__class__.__name__) if not( isinstance(session, ftrack_api.session.Session) or - isinstance(session, ProcessSession) + isinstance(session, session_processor.ProcessSession) ): raise Exception(( "Session object entered with args is instance of \"{}\"" @@ -42,7 +43,7 @@ class BaseHandler(object): ).format( str(type(session)), str(ftrack_api.session.Session), - str(ProcessSession) + str(session_processor.ProcessSession) )) self._session = session From 19f810ff57e7a55a70ed7cba9afc10924a6e9c48 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:01:51 +0200 Subject: [PATCH 15/37] added check active table to custom db connector --- pype/ftrack/lib/custom_db_connector.py | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 505ac96610..f4cc00444b 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -20,6 +20,9 @@ import requests import pymongo from pymongo.client_session import ClientSession +class NotActiveTable(Exception): + pass + def auto_reconnect(func): """Handling auto reconnect in 3 retry times""" @functools.wraps(func) @@ -37,12 +40,23 @@ def auto_reconnect(func): return decorated +def check_active_table(func): + """Handling auto reconnect in 3 retry times""" + @functools.wraps(func) + def decorated(obj, *args, **kwargs): + if not obj.active_table: + raise NotActiveTable("Active table is not set. (This is bug)") + return func(obj, *args, **kwargs) + + return decorated + + class DbConnector: log = logging.getLogger(__name__) timeout = 1000 - def __init__(self, mongo_url, database_name, table_name): + def __init__(self, mongo_url, database_name, table_name=None): self._mongo_client = None self._sentry_client = None self._sentry_logging_handler = None @@ -115,6 +129,7 @@ class DbConnector: def collections(self): return self._database.collection_names() + @check_active_table @auto_reconnect def insert_one(self, item, session=None): assert isinstance(item, dict), "item must be of type " @@ -123,6 +138,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def insert_many(self, items, ordered=True, session=None): # check if all items are valid @@ -136,6 +152,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find(self, filter, projection=None, sort=None, session=None): return self._database[self.active_table].find( @@ -145,6 +162,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def find_one(self, filter, projection=None, sort=None, session=None): assert isinstance(filter, dict), "filter must be " @@ -156,6 +174,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def replace_one(self, filter, replacement, session=None): return self._database[self.active_table].replace_one( @@ -163,6 +182,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def update_one(self, filter, update, session=None): return self._database[self.active_table].update_one( @@ -170,6 +190,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def update_many(self, filter, update, session=None): return self._database[self.active_table].update_many( @@ -177,12 +198,14 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def distinct(self, *args, **kwargs): return self._database[self.active_table].distinct( *args, **kwargs ) + @check_active_table @auto_reconnect def drop_collection(self, name_or_collection, session=None): return self._database[self.active_table].drop( @@ -190,6 +213,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def delete_one(filter, collation=None, session=None): return self._database[self.active_table].delete_one( @@ -198,6 +222,7 @@ class DbConnector: session=session ) + @check_active_table @auto_reconnect def delete_many(filter, collation=None, session=None): return self._database[self.active_table].delete_many( From 1ca674f33f18e7821aa47f039f0b808b50293b99 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:05:07 +0200 Subject: [PATCH 16/37] added atexit to custom db connector to run uninstall on exit --- pype/ftrack/lib/custom_db_connector.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index f4cc00444b..222a565ded 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -13,6 +13,7 @@ import logging import tempfile import functools import contextlib +import atexit import requests @@ -71,7 +72,7 @@ class DbConnector: """Establish a persistent connection to the database""" if self._is_installed: return - + atexit.register(self.uninstall) logging.basicConfig() self._mongo_client = pymongo.MongoClient( @@ -113,6 +114,7 @@ class DbConnector: self._mongo_client = None self._database = None self._is_installed = False + atexit.unregister(self.uninstall) def tables(self): """List available tables From 45370876e33b1b3c188b2c95932989e2ccad458e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 15:29:05 +0200 Subject: [PATCH 17/37] replaced args with options(kwargs) in custom db connector --- pype/ftrack/lib/custom_db_connector.py | 95 ++++++++++++-------------- 1 file changed, 43 insertions(+), 52 deletions(-) diff --git a/pype/ftrack/lib/custom_db_connector.py b/pype/ftrack/lib/custom_db_connector.py index 222a565ded..11fd197555 100644 --- a/pype/ftrack/lib/custom_db_connector.py +++ b/pype/ftrack/lib/custom_db_connector.py @@ -68,6 +68,12 @@ class DbConnector: self.active_table = table_name + def __getattribute__(self, attr): + try: + return super().__getattribute__(attr) + except AttributeError: + return self._database[self.active_table].__getattribute__(attr) + def install(self): """Establish a persistent connection to the database""" if self._is_installed: @@ -116,6 +122,15 @@ class DbConnector: self._is_installed = False atexit.unregister(self.uninstall) + def create_table(self, name, **options): + if self.exist_table(name): + return + + return self._database.create_collection(name, **options) + + def exist_table(self, table_name): + return table_name in self.tables() + def tables(self): """List available tables Returns: @@ -133,102 +148,78 @@ class DbConnector: @check_active_table @auto_reconnect - def insert_one(self, item, session=None): + def insert_one(self, item, **options): assert isinstance(item, dict), "item must be of type " - return self._database[self.active_table].insert_one( - item, - session=session - ) + return self._database[self.active_table].insert_one(item, **options) @check_active_table @auto_reconnect - def insert_many(self, items, ordered=True, session=None): + def insert_many(self, items, ordered=True, **options): # check if all items are valid assert isinstance(items, list), "`items` must be of type " for item in items: assert isinstance(item, dict), "`item` must be of type " - return self._database[self.active_table].insert_many( - items, - ordered=ordered, - session=session - ) + options["ordered"] = ordered + return self._database[self.active_table].insert_many(items, **options) @check_active_table @auto_reconnect - def find(self, filter, projection=None, sort=None, session=None): - return self._database[self.active_table].find( - filter=filter, - projection=projection, - sort=sort, - session=session - ) + def find(self, filter, projection=None, sort=None, **options): + options["projection"] = projection + options["sort"] = sort + return self._database[self.active_table].find(filter, **options) @check_active_table @auto_reconnect - def find_one(self, filter, projection=None, sort=None, session=None): + def find_one(self, filter, projection=None, sort=None, **options): assert isinstance(filter, dict), "filter must be " - return self._database[self.active_table].find_one( - filter=filter, - projection=projection, - sort=sort, - session=session - ) + options["projection"] = projection + options["sort"] = sort + return self._database[self.active_table].find_one(filter, **options) @check_active_table @auto_reconnect - def replace_one(self, filter, replacement, session=None): + def replace_one(self, filter, replacement, **options): return self._database[self.active_table].replace_one( - filter, replacement, - session=session + filter, replacement, **options ) @check_active_table @auto_reconnect - def update_one(self, filter, update, session=None): + def update_one(self, filter, update, **options): return self._database[self.active_table].update_one( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect - def update_many(self, filter, update, session=None): + def update_many(self, filter, update, **options): return self._database[self.active_table].update_many( - filter, update, - session=session + filter, update, **options ) @check_active_table @auto_reconnect def distinct(self, *args, **kwargs): - return self._database[self.active_table].distinct( - *args, **kwargs - ) + return self._database[self.active_table].distinct(*args, **kwargs) @check_active_table @auto_reconnect - def drop_collection(self, name_or_collection, session=None): + def drop_collection(self, name_or_collection, **options): return self._database[self.active_table].drop( - name_or_collection, - session=session + name_or_collection, **options ) @check_active_table @auto_reconnect - def delete_one(filter, collation=None, session=None): - return self._database[self.active_table].delete_one( - filter, - collation=collation, - session=session - ) + def delete_one(self, filter, collation=None, **options): + options["collation"] = collation + return self._database[self.active_table].delete_one(filter, **options) @check_active_table @auto_reconnect - def delete_many(filter, collation=None, session=None): - return self._database[self.active_table].delete_many( - filter, - collation=collation, - session=session - ) + def delete_many(self, filter, collation=None, **options): + options["collation"] = collation + return self._database[self.active_table].delete_many(filter, **options) From ac290711b906379c5c6839b28f13de37baee7870 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:47:45 +0200 Subject: [PATCH 18/37] sort events by date there were stored --- pype/ftrack/ftrack_server/session_processor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index d51da28938..fd797cfb3a 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -6,6 +6,7 @@ import threading import time import requests import queue +import pymongo import ftrack_api import ftrack_api.session @@ -74,7 +75,12 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): break def load_events(self): - not_processed_events = self.dbcon.find({"pype_data.is_processed": False}) + not_processed_events = self.dbcon.find( + {"pype_data.is_processed": False} + ).sort( + [("pype_data.stored", pymongo.ASCENDING)] + ) + found = False for event_data in not_processed_events: new_event_data = { From aad641f233e76791626bb38979f30c5b65d6ef62 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:48:13 +0200 Subject: [PATCH 19/37] import fixes --- pype/ftrack/ftrack_server/__init__.py | 7 ------- pype/ftrack/ftrack_server/socket_thread.py | 7 ------- pype/ftrack/ftrack_server/sub_event_processor.py | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/pype/ftrack/ftrack_server/__init__.py b/pype/ftrack/ftrack_server/__init__.py index f8f876bbb6..0861a1bc08 100644 --- a/pype/ftrack/ftrack_server/__init__.py +++ b/pype/ftrack/ftrack_server/__init__.py @@ -1,8 +1 @@ from .ftrack_server import FtrackServer - -import event_server_cli -import session_processor -import session_storer -import socket_thread -import sub_event_processor -import sub_event_storer diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index e787004afc..8d700d7255 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -25,13 +25,6 @@ class SocketThread(threading.Thread): def stop(self): self._is_running = False - def process_to_die(self): - if not self.connection: - self.stop() - return - - self.connection.sendall(b"ptd") - def run(self): self._is_running = True time_socket = time.time() diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index b53c294981..a8f2dc5041 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -6,7 +6,7 @@ import socket import pymongo from ftrack_server import FtrackServer -import session_processor +from pype.ftrack.ftrack_server.session_processor import ProcessSession from pypeapp import Logger log = Logger().get_logger("Event processor") @@ -24,7 +24,7 @@ def main(args): sock.sendall(b"CreatedProcess") try: - session = session_processor.ProcessSession(auto_connect_event_hub=True, sock=sock) + session = ProcessSession(auto_connect_event_hub=True, sock=sock) server = FtrackServer('event') log.debug("Launched Ftrack Event processor") server.run_server(session) From ee8f09cf4635bda3c1a7d1ffafe34e0fa332fb71 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 16:52:07 +0200 Subject: [PATCH 20/37] added error logs --- pype/ftrack/ftrack_server/session_processor.py | 9 +++++++++ pype/ftrack/ftrack_server/sub_event_processor.py | 2 +- pype/ftrack/ftrack_server/sub_event_storer.py | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index fd797cfb3a..dbad705261 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -17,6 +17,9 @@ import ftrack_api.event from ftrack_api.logging import LazyLogMessage as L from pype.ftrack.lib.custom_db_connector import DbConnector +from pypeapp import Logger + +log = Logger().get_logger("Session processor") class ProcessEventHub(ftrack_api.event.hub.EventHub): @@ -39,6 +42,9 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): self.dbcon.active_table = self.table_name self.is_table_created = True except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) def wait(self, duration=None): @@ -65,6 +71,9 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): {"$set": {"pype_data.is_processed": True}} ) except pymongo.errors.AutoReconnect: + log.error(( + "Mongo server \"{}\" is not responding, exiting." + ).format(os.environ["AVALON_MONGO"])) sys.exit(0) # Additional special processing of events. if event['topic'] == 'ftrack.meta.disconnected': diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index a8f2dc5041..eefcfad6ab 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -44,4 +44,4 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - main(sys.argv) + return main(sys.argv) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index 918e1e1bea..dca9ae55e2 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -28,6 +28,9 @@ def install_db(): dbcon.create_table(table_name, capped=False) dbcon.active_table = table_name except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) def launch(event): @@ -48,6 +51,9 @@ def launch(event): log.debug("Event: {} stored".format(event_id)) except pymongo.errors.AutoReconnect: + log.error("Mongo server \"{}\" is not responding, exiting.".format( + os.environ["AVALON_MONGO"] + )) sys.exit(0) except Exception as exc: From 565c43971c1108065c1978c5f101c29b7076a876 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 17:33:32 +0200 Subject: [PATCH 21/37] added sigkill signal --- pype/ftrack/ftrack_server/event_server_cli.py | 2 ++ pype/ftrack/ftrack_server/sub_event_processor.py | 4 +++- pype/ftrack/ftrack_server/sub_event_storer.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 83ecd0d47a..2e9d95467e 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -368,5 +368,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index eefcfad6ab..20e72beb7c 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -43,5 +43,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) - return main(sys.argv) + main(sys.argv) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index dca9ae55e2..dc6fba293b 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -102,5 +102,7 @@ if __name__ == "__main__": signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) main(sys.argv) From bfa61e2e986ae621547df2e6e2d7e8e5af85c4ee Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 17:34:11 +0200 Subject: [PATCH 22/37] kill threads with subprocesses on exit --- pype/ftrack/ftrack_server/event_server_cli.py | 13 +++++++++++++ pype/ftrack/ftrack_server/sub_event_processor.py | 2 +- pype/ftrack/ftrack_server/sub_event_storer.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 2e9d95467e..81357da2fa 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -3,6 +3,7 @@ import sys import signal import socket import argparse +import atexit import time from urllib.parse import urlparse @@ -153,6 +154,18 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_ftrack_error = False printed_mongo_error = False + def on_exit(): + if processor_thread is not None: + processor_thread.stop() + processor_thread.join() + processor_thread = None + + if storer_thread is not None: + storer_thread.stop() + storer_thread.join() + storer_thread = None + + atexit.register(on_exit) # Main loop while True: # Check if accessible Ftrack and Mongo url diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index 20e72beb7c..46d2b861fa 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -46,4 +46,4 @@ if __name__ == "__main__": if hasattr(signal, "SIGKILL"): signal.signal(signal.SIGKILL, signal_handler) - main(sys.argv) + sys.exit(main(sys.argv)) diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index dc6fba293b..da67bc38eb 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -105,4 +105,4 @@ if __name__ == "__main__": if hasattr(signal, "SIGKILL"): signal.signal(signal.SIGKILL, signal_handler) - main(sys.argv) + sys.exit(main(sys.argv)) From 11f4451af1382d6be5c6110bf9ea42e97a4c7b54 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:02:53 +0200 Subject: [PATCH 23/37] added basic docstrings --- pype/ftrack/ftrack_server/event_server_cli.py | 12 ++++++++++++ pype/ftrack/ftrack_server/session_processor.py | 14 ++++++-------- pype/ftrack/ftrack_server/session_storer.py | 2 +- pype/ftrack/ftrack_server/socket_thread.py | 1 + 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 81357da2fa..815af50c5a 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -15,6 +15,7 @@ import socket_thread def check_ftrack_url(url, log_errors=True): + """Checks if Ftrack server is responding""" if not url: print('ERROR: Ftrack URL is not set!') return None @@ -44,6 +45,7 @@ def check_ftrack_url(url, log_errors=True): def check_mongo_url(host, port, log_error=False): + """Checks if mongo server is responding""" sock = None try: sock = socket.create_connection( @@ -114,6 +116,15 @@ def process_event_paths(event_paths): def main_loop(ftrack_url, username, api_key, event_paths): + """ This is main loop of event handling. + + Loop is handling threads which handles subprocesses of event storer and + processor. When one of threads is stopped it is tested to connect to + ftrack and mongo server. Threads are not started when ftrack or mongo + server is not accessible. When threads are started it is checked for socket + signals as heartbeat. Heartbeat must become at least once per 30sec + otherwise thread will be killed. + """ # Set Ftrack environments os.environ["FTRACK_SERVER"] = ftrack_url os.environ["FTRACK_API_USER"] = username @@ -154,6 +165,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_ftrack_error = False printed_mongo_error = False + # stop threads on exit def on_exit(): if processor_thread is not None: processor_thread.stop() diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index dbad705261..2dfa94dc94 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -48,13 +48,11 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): sys.exit(0) def wait(self, duration=None): - '''Wait for events and handle as they arrive. + """Overriden wait - If *duration* is specified, then only process events until duration is - reached. *duration* is in seconds though float values can be used for - smaller values. - - ''' + Event are loaded from Mongo DB when queue is empty. Handled event is + set as processed in Mongo DB. + """ started = time.time() self.prepare_dbcon() while True: @@ -84,6 +82,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): break def load_events(self): + """Load not processed events sorted by stored date""" not_processed_events = self.dbcon.find( {"pype_data.is_processed": False} ).sort( @@ -110,8 +109,7 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): return found def _handle_packet(self, code, packet_identifier, path, data): - '''Handle packet received from server.''' - # if self.is_waiting: + """Override `_handle_packet` which skip events and extend heartbeat""" code_name = self._code_name_mapping[code] if code_name == "event": return diff --git a/pype/ftrack/ftrack_server/session_storer.py b/pype/ftrack/ftrack_server/session_storer.py index 01b1228d55..b3201c9e4d 100644 --- a/pype/ftrack/ftrack_server/session_storer.py +++ b/pype/ftrack/ftrack_server/session_storer.py @@ -20,7 +20,7 @@ class StorerEventHub(ftrack_api.event.hub.EventHub): super(StorerEventHub, self).__init__(*args, **kwargs) def _handle_packet(self, code, packet_identifier, path, data): - '''Handle packet received from server.''' + """Override `_handle_packet` which extend heartbeat""" if self._code_name_mapping[code] == "heartbeat": # Reply with heartbeat. self.sock.sendall(b"storer") diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index 8d700d7255..ba6e0ec89b 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -9,6 +9,7 @@ from pypeapp import Logger class SocketThread(threading.Thread): MAX_TIMEOUT = 30 + """Thread that checks suprocess of storer of processor of events""" def __init__(self, name, port, filepath): super(SocketThread, self).__init__() self.log = Logger().get_logger("SocketThread", "Event Thread") From 7c59df5ec573ea47f1e0e4fca8011a7670de4c99 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:03:29 +0200 Subject: [PATCH 24/37] max timeout of heartbeat increased to 35 --- pype/ftrack/ftrack_server/socket_thread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index ba6e0ec89b..7b7d9cd852 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -8,8 +8,8 @@ from pypeapp import Logger class SocketThread(threading.Thread): - MAX_TIMEOUT = 30 """Thread that checks suprocess of storer of processor of events""" + MAX_TIMEOUT = 35 def __init__(self, name, port, filepath): super(SocketThread, self).__init__() self.log = Logger().get_logger("SocketThread", "Event Thread") From 3a5525edf68f2b4062e9bb680a36b6d435b84b62 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 23 Oct 2019 18:45:41 +0200 Subject: [PATCH 25/37] processed events older than 3 days are removed from mongo db --- pype/ftrack/ftrack_server/session_processor.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index 2dfa94dc94..686e89dc79 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -1,6 +1,7 @@ import logging import os import atexit +import datetime import tempfile import threading import time @@ -83,6 +84,12 @@ class ProcessEventHub(ftrack_api.event.hub.EventHub): def load_events(self): """Load not processed events sorted by stored date""" + ago_date = datetime.datetime.now() - datetime.timedelta(days=3) + result = self.dbcon.delete_many({ + "pype_data.stored": {"$lte": ago_date}, + "pype_data.is_processed": True + }) + not_processed_events = self.dbcon.find( {"pype_data.is_processed": False} ).sort( From e7c9a61b2e4e3e368f17bd92466c268e1fa96cab Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:21:44 +0200 Subject: [PATCH 26/37] added lib to ftrack_server with functios able to handle mongo url parts --- pype/ftrack/__init__.py | 2 +- pype/ftrack/ftrack_server/lib.py | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 pype/ftrack/ftrack_server/lib.py diff --git a/pype/ftrack/__init__.py b/pype/ftrack/__init__.py index bf18979e91..45ca8384b5 100644 --- a/pype/ftrack/__init__.py +++ b/pype/ftrack/__init__.py @@ -1,2 +1,2 @@ from .lib import * -from .ftrack_server import * +from .ftrack_server import FtrackServer diff --git a/pype/ftrack/ftrack_server/lib.py b/pype/ftrack/ftrack_server/lib.py new file mode 100644 index 0000000000..12159693fe --- /dev/null +++ b/pype/ftrack/ftrack_server/lib.py @@ -0,0 +1,68 @@ +import os +try: + from urllib.parse import urlparse, parse_qs +except ImportError: + from urlparse import urlparse, parse_qs + + +def ftrack_events_mongo_settings(): + host = None + port = None + username = None + password = None + collection = None + database = None + auth_db = "" + + if os.environ.get('FTRACK_EVENTS_MONGO_URL'): + result = urlparse(os.environ['FTRACK_EVENTS_MONGO_URL']) + + host = result.hostname + try: + port = result.port + except ValueError: + raise RuntimeError("invalid port specified") + username = result.username + password = result.password + try: + database = result.path.lstrip("/").split("/")[0] + collection = result.path.lstrip("/").split("/")[1] + except IndexError: + if not database: + raise RuntimeError("missing database name for logging") + try: + auth_db = parse_qs(result.query)['authSource'][0] + except KeyError: + # no auth db provided, mongo will use the one we are connecting to + pass + else: + host = os.environ.get('FTRACK_EVENTS_MONGO_HOST') + port = int(os.environ.get('FTRACK_EVENTS_MONGO_PORT', "0")) + database = os.environ.get('FTRACK_EVENTS_MONGO_DB') + username = os.environ.get('FTRACK_EVENTS_MONGO_USER') + password = os.environ.get('FTRACK_EVENTS_MONGO_PASSWORD') + collection = os.environ.get('FTRACK_EVENTS_MONGO_COL') + auth_db = os.environ.get('FTRACK_EVENTS_MONGO_AUTH_DB', 'avalon') + + return host, port, database, username, password, collection, auth_db + + +def get_ftrack_event_mongo_info(): + host, port, database, username, password, collection, auth_db = ftrack_events_mongo_settings() + user_pass = "" + if username and password: + user_pass = "{}:{}@".format(username, password) + + socket_path = "{}:{}".format(host, port) + + dab = "" + if database: + dab = "/{}".format(database) + + auth = "" + if auth_db: + auth = "?authSource={}".format(auth_db) + + url = "mongodb://{}{}{}{}".format(user_pass, socket_path, dab, auth) + + return url, database, collection From 24175a258766959de53dcc6fc2ffbee0f964847d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:22:28 +0200 Subject: [PATCH 27/37] atexit register has args now --- pype/ftrack/ftrack_server/event_server_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 815af50c5a..6796399ad8 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -166,7 +166,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_mongo_error = False # stop threads on exit - def on_exit(): + def on_exit(processor_thread, storer_thread): if processor_thread is not None: processor_thread.stop() processor_thread.join() @@ -177,7 +177,9 @@ def main_loop(ftrack_url, username, api_key, event_paths): storer_thread.join() storer_thread = None - atexit.register(on_exit) + atexit.register( + on_exit, processor_thread=processor_thread, storer_thread=storer_thread + ) # Main loop while True: # Check if accessible Ftrack and Mongo url From 4313cc5667bbb79195fdea32c7b94624c27eb961 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:23:52 +0200 Subject: [PATCH 28/37] mongo permissions error is sent with socket message --- .../ftrack/ftrack_server/session_processor.py | 27 ++++++++++++------- pype/ftrack/ftrack_server/sub_event_storer.py | 20 ++++++++++---- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pype/ftrack/ftrack_server/session_processor.py b/pype/ftrack/ftrack_server/session_processor.py index 686e89dc79..2f5818aab4 100644 --- a/pype/ftrack/ftrack_server/session_processor.py +++ b/pype/ftrack/ftrack_server/session_processor.py @@ -18,36 +18,45 @@ import ftrack_api.event from ftrack_api.logging import LazyLogMessage as L from pype.ftrack.lib.custom_db_connector import DbConnector +from pype.ftrack.ftrack_server.lib import get_ftrack_event_mongo_info from pypeapp import Logger log = Logger().get_logger("Session processor") class ProcessEventHub(ftrack_api.event.hub.EventHub): - dbcon = DbConnector( - mongo_url=os.environ["AVALON_MONGO"], - database_name="pype" - ) - table_name = "ftrack_events" + url, database, table_name = get_ftrack_event_mongo_info() + is_table_created = False def __init__(self, *args, **kwargs): + self.dbcon = DbConnector( + mongo_url=self.url, + database_name=self.database, + table_name=self.table_name + ) self.sock = kwargs.pop("sock") super(ProcessEventHub, self).__init__(*args, **kwargs) def prepare_dbcon(self): try: self.dbcon.install() - if not self.is_table_created: - self.dbcon.create_table(self.table_name, capped=False) - self.dbcon.active_table = self.table_name - self.is_table_created = True + dbcon._database.collection_names() except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( os.environ["AVALON_MONGO"] )) sys.exit(0) + except pymongo.errors.OperationFailure: + log.error(( + "Error with Mongo access, probably permissions." + "Check if exist database with name \"{}\"" + " and collection \"{}\" inside." + ).format(self.database, self.table_name)) + self.sock.sendall(b"MongoError") + sys.exit(0) + def wait(self, duration=None): """Overriden wait diff --git a/pype/ftrack/ftrack_server/sub_event_storer.py b/pype/ftrack/ftrack_server/sub_event_storer.py index da67bc38eb..6e30fb99e2 100644 --- a/pype/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/ftrack/ftrack_server/sub_event_storer.py @@ -6,18 +6,20 @@ import socket import pymongo from ftrack_server import FtrackServer +from pype.ftrack.ftrack_server.lib import get_ftrack_event_mongo_info from pype.ftrack.lib.custom_db_connector import DbConnector from session_storer import StorerSession from pypeapp import Logger log = Logger().get_logger("Event storer") +url, database, table_name = get_ftrack_event_mongo_info() dbcon = DbConnector( - mongo_url=os.environ["AVALON_MONGO"], - database_name="pype" + mongo_url=url, + database_name=database, + table_name=table_name ) -table_name = "ftrack_events" # ignore_topics = ["ftrack.meta.connected"] ignore_topics = [] @@ -25,14 +27,14 @@ ignore_topics = [] def install_db(): try: dbcon.install() - dbcon.create_table(table_name, capped=False) - dbcon.active_table = table_name + dbcon._database.collection_names() except pymongo.errors.AutoReconnect: log.error("Mongo server \"{}\" is not responding, exiting.".format( os.environ["AVALON_MONGO"] )) sys.exit(0) + def launch(event): if event.get("topic") in ignore_topics: return @@ -88,6 +90,14 @@ def main(args): log.debug("Launched Ftrack Event storer") server.run_server(session, load_files=False) + except pymongo.errors.OperationFailure: + log.error(( + "Error with Mongo access, probably permissions." + "Check if exist database with name \"{}\"" + " and collection \"{}\" inside." + ).format(database, table_name)) + sock.sendall(b"MongoError") + finally: log.debug("First closing socket") sock.close() From e0f7752205faef93ca376d0cf8f1b3966c7c91d0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:24:12 +0200 Subject: [PATCH 29/37] sub event processor print traceback --- pype/ftrack/ftrack_server/sub_event_processor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/ftrack/ftrack_server/sub_event_processor.py b/pype/ftrack/ftrack_server/sub_event_processor.py index 46d2b861fa..9444fe3ff0 100644 --- a/pype/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/ftrack/ftrack_server/sub_event_processor.py @@ -29,6 +29,10 @@ def main(args): log.debug("Launched Ftrack Event processor") server.run_server(session) + except Exception as exc: + import traceback + traceback.print_tb(exc.__traceback__) + finally: log.debug("First closing socket") sock.close() From 5d6f91d6141ded073b5109ede3bad47b9143e349 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:24:36 +0200 Subject: [PATCH 30/37] socket thread stores info if mongo error has happened during subprocess --- pype/ftrack/ftrack_server/socket_thread.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pype/ftrack/ftrack_server/socket_thread.py b/pype/ftrack/ftrack_server/socket_thread.py index 7b7d9cd852..d0a2868743 100644 --- a/pype/ftrack/ftrack_server/socket_thread.py +++ b/pype/ftrack/ftrack_server/socket_thread.py @@ -1,4 +1,5 @@ import os +import sys import time import signal import socket @@ -23,6 +24,8 @@ class SocketThread(threading.Thread): self._is_running = False self.finished = False + self.mongo_error = False + def stop(self): self._is_running = False @@ -97,6 +100,8 @@ class SocketThread(threading.Thread): break if data: + if data == b"MongoError": + self.mongo_error = True connection.sendall(data) except Exception as exc: @@ -110,4 +115,9 @@ class SocketThread(threading.Thread): if self.subproc.poll() is None: self.subproc.terminate() + lines = self.subproc.stdout.readlines() + if lines: + print("*** Socked Thread stdout ***") + for line in lines: + os.write(1, line) self.finished = True From 11f0c41e904b5c0b3fb28280c4801ed436848759 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:25:44 +0200 Subject: [PATCH 31/37] crash all server if mongo error happened --- pype/ftrack/ftrack_server/event_server_cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 6796399ad8..a016f1aaf4 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -225,6 +225,10 @@ def main_loop(ftrack_url, username, api_key, event_paths): # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): + if storer_thread.mongo_error: + raise Exception( + "Exiting because have issue with acces to MongoDB" + ) storer_thread.join() storer_thread = None ftrack_accessible = False @@ -237,7 +241,11 @@ def main_loop(ftrack_url, username, api_key, event_paths): processor_thread.start() # If thread failed test Ftrack and Mongo connection - elif processor_thread.isAlive(): + elif not processor_thread.isAlive(): + if storer_thread.mongo_error: + raise Exception( + "Exiting because have issue with acces to MongoDB" + ) processor_thread.join() processor_thread = None ftrack_accessible = False From 38a6c84fe55ebb17a188998a431125a63b4b12ca Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:26:38 +0200 Subject: [PATCH 32/37] its used lib to get mongo port and host in event server cli --- pype/ftrack/ftrack_server/event_server_cli.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index a016f1aaf4..0698f5728f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -11,6 +11,7 @@ import requests from pype.vendor import ftrack_api from pype.ftrack.lib import credentials from pype.ftrack.ftrack_server import FtrackServer +from pype.ftrack.ftrack_server.lib import ftrack_events_mongo_settings import socket_thread @@ -132,18 +133,9 @@ def main_loop(ftrack_url, username, api_key, event_paths): os.environ["FTRACK_EVENTS_PATH"] = event_paths # Get mongo hostname and port for testing mongo connection - mongo_url = os.environ["AVALON_MONGO"].strip('/ ') - result = urlparse(mongo_url) - url_items = result.netloc.split("@") - mongo_url = url_items[0] - if len(url_items) == 2: - mongo_url = url_items[1] - - mongo_url = "://".join([result.scheme, mongo_url]) - result = urlparse(mongo_url) - - mongo_hostname = result.hostname - mongo_port = result.port + mongo_list = ftrack_events_mongo_settings() + mongo_hostname = mongo_list[0] + mongo_port = mongo_list[1] # Current file file_path = os.path.dirname(os.path.realpath(__file__)) From 10d695603765fe9aecf5213b189c0ede700d9de2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 25 Oct 2019 18:27:31 +0200 Subject: [PATCH 33/37] event server will wait some time if subprocess crashed many times in row --- pype/ftrack/ftrack_server/event_server_cli.py | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 0698f5728f..768a4425bb 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -1,6 +1,7 @@ import os import sys import signal +import datetime import socket import argparse import atexit @@ -140,16 +141,24 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Current file file_path = os.path.dirname(os.path.realpath(__file__)) + min_fail_seconds = 5 + max_fail_count = 3 + wait_time_after_max_fail = 10 + # Threads data storer_name = "StorerThread" storer_port = 10001 storer_path = "{}/sub_event_storer.py".format(file_path) storer_thread = None + storer_last_failed = datetime.datetime.now() + storer_failed_count = 0 processor_name = "ProcessorThread" processor_port = 10011 processor_path = "{}/sub_event_processor.py".format(file_path) processor_thread = None + processor_last_failed = datetime.datetime.now() + processor_failed_count = 0 ftrack_accessible = False mongo_accessible = False @@ -210,10 +219,20 @@ def main_loop(ftrack_url, username, api_key, event_paths): # Run backup thread which does not requeire mongo to work if storer_thread is None: - storer_thread = socket_thread.SocketThread( - storer_name, storer_port, storer_path - ) - storer_thread.start() + if storer_failed_count < max_fail_count: + storer_thread = socket_thread.SocketThread( + storer_name, storer_port, storer_path + ) + storer_thread.start() + elif storer_failed_count == max_fail_count: + print(( + "Storer failed {}times I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + storer_failed_count += 1 + elif (( + datetime.datetime.now() - storer_last_failed + ).seconds > wait_time_after_max_fail): + storer_failed_count = 0 # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): @@ -226,11 +245,32 @@ def main_loop(ftrack_url, username, api_key, event_paths): ftrack_accessible = False mongo_accessible = False + _storer_last_failed = datetime.datetime.now() + delta_time = (_storer_last_failed - storer_last_failed).seconds + if delta_time < min_fail_seconds: + storer_failed_count += 1 + else: + storer_failed_count = 0 + storer_last_failed = _storer_last_failed + if processor_thread is None: - processor_thread = socket_thread.SocketThread( - processor_name, processor_port, processor_path - ) - processor_thread.start() + if processor_failed_count < max_fail_count: + processor_thread = socket_thread.SocketThread( + processor_name, processor_port, processor_path + ) + processor_thread.start() + + elif processor_failed_count == max_fail_count: + print(( + "Processor failed {}times in row" + " I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + processor_failed_count += 1 + + elif (( + datetime.datetime.now() - processor_last_failed + ).seconds > wait_time_after_max_fail): + processor_failed_count = 0 # If thread failed test Ftrack and Mongo connection elif not processor_thread.isAlive(): @@ -243,6 +283,17 @@ def main_loop(ftrack_url, username, api_key, event_paths): ftrack_accessible = False mongo_accessible = False + _processor_last_failed = datetime.datetime.now() + delta_time = ( + _processor_last_failed - processor_last_failed + ).seconds + + if delta_time < min_fail_seconds: + processor_failed_count += 1 + else: + processor_failed_count = 0 + processor_last_failed = _processor_last_failed + time.sleep(1) From 75ce8f1196d280aab1980ed6c97f52092b8c1eda Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 17:58:29 +0100 Subject: [PATCH 34/37] added subprocess file for oldway event server --- pype/ftrack/ftrack_server/sub_old_way.py | 100 +++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 pype/ftrack/ftrack_server/sub_old_way.py diff --git a/pype/ftrack/ftrack_server/sub_old_way.py b/pype/ftrack/ftrack_server/sub_old_way.py new file mode 100644 index 0000000000..92e7c0cf8c --- /dev/null +++ b/pype/ftrack/ftrack_server/sub_old_way.py @@ -0,0 +1,100 @@ +import os +import sys +import time +import datetime +import signal +import threading + +from ftrack_server import FtrackServer +from pype.vendor import ftrack_api +from pype.vendor.ftrack_api.event.hub import EventHub +from pypeapp import Logger + +log = Logger().get_logger("Event Server Old") + + +class TimerChecker(threading.Thread): + max_time_out = 35 + + def __init__(self, server, session): + self.server = server + self.session = session + self.is_running = False + self.failed = False + super().__init__() + + def stop(self): + self.is_running = False + + def run(self): + start = datetime.datetime.now() + self.is_running = True + connected = False + + while True: + if not self.is_running: + break + + if not self.session.event_hub.connected: + if not connected: + if (datetime.datetime.now() - start).seconds > self.max_time_out: + log.error(( + "Exiting event server. Session was not connected" + " to ftrack server in {} seconds." + ).format(self.max_time_out)) + self.failed = True + break + else: + log.error( + "Exiting event server. Event Hub is not connected." + ) + self.server.stop_session() + self.failed = True + break + else: + if not connected: + connected = True + + time.sleep(1) + + +def main(args): + check_thread = None + try: + server = FtrackServer('event') + session = ftrack_api.Session(auto_connect_event_hub=True) + + check_thread = TimerChecker(server, session) + check_thread.start() + + log.debug("Launching Ftrack Event Old Way Server") + server.run_server(session) + + except Exception as exc: + import traceback + traceback.print_tb(exc.__traceback__) + + finally: + log_info = True + if check_thread is not None: + check_thread.stop() + check_thread.join() + if check_thread.failed: + log_info = False + if log_info: + log.info("Exiting Event server subprocess") + return 1 + + +if __name__ == "__main__": + # Register interupt signal + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + if hasattr(signal, "SIGKILL"): + signal.signal(signal.SIGKILL, signal_handler) + + sys.exit(main(sys.argv)) From 5b70ddf32f9d0c836eb99337fcf6fcac42b4ac9d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 17:59:36 +0100 Subject: [PATCH 35/37] added `oldway` argument to event server cli --- pype/ftrack/ftrack_server/event_server_cli.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index 768a4425bb..ade80117af 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -127,11 +127,6 @@ def main_loop(ftrack_url, username, api_key, event_paths): signals as heartbeat. Heartbeat must become at least once per 30sec otherwise thread will be killed. """ - # Set Ftrack environments - os.environ["FTRACK_SERVER"] = ftrack_url - os.environ["FTRACK_API_USER"] = username - os.environ["FTRACK_API_KEY"] = api_key - os.environ["FTRACK_EVENTS_PATH"] = event_paths # Get mongo hostname and port for testing mongo connection mongo_list = ftrack_events_mongo_settings() @@ -385,7 +380,11 @@ def main(argv): help="Load creadentials from apps dir", action="store_true" ) - + parser.add_argument( + '-oldway', + help="Load creadentials from apps dir", + action="store_true" + ) ftrack_url = os.environ.get('FTRACK_SERVER') username = os.environ.get('FTRACK_API_USER') api_key = os.environ.get('FTRACK_API_KEY') @@ -410,6 +409,7 @@ def main(argv): if kwargs.ftrackapikey: api_key = kwargs.ftrackapikey + oldway = kwargs.oldway # Check url regex and accessibility ftrack_url = check_ftrack_url(ftrack_url) if not ftrack_url: @@ -435,7 +435,16 @@ def main(argv): if kwargs.storecred: credentials._save_credentials(username, api_key, True) - main_loop(ftrack_url, username, api_key, event_paths) + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + os.environ["FTRACK_EVENTS_PATH"] = event_paths + + if oldway: + return old_way_server(ftrack_url) + + return main_loop(ftrack_url) if __name__ == "__main__": From 92ccd406c57b13dca335fb73e53b34cf00048ea2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 18:00:20 +0100 Subject: [PATCH 36/37] created exception for mongo permission error --- pype/ftrack/ftrack_server/event_server_cli.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index ade80117af..b0dfb48a1f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -16,6 +16,14 @@ from pype.ftrack.ftrack_server.lib import ftrack_events_mongo_settings import socket_thread +class MongoPermissionsError(Exception): + """Is used when is created multiple objects of same RestApi class.""" + def __init__(self, message=None): + if not message: + message = "Exiting because have issue with acces to MongoDB" + super().__init__(message) + + def check_ftrack_url(url, log_errors=True): """Checks if Ftrack server is responding""" if not url: @@ -162,6 +170,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): printed_mongo_error = False # stop threads on exit + # TODO check if works and args have thread objects! def on_exit(processor_thread, storer_thread): if processor_thread is not None: processor_thread.stop() @@ -232,9 +241,7 @@ def main_loop(ftrack_url, username, api_key, event_paths): # If thread failed test Ftrack and Mongo connection elif not storer_thread.isAlive(): if storer_thread.mongo_error: - raise Exception( - "Exiting because have issue with acces to MongoDB" - ) + raise MongoPermissionsError() storer_thread.join() storer_thread = None ftrack_accessible = False From 6f93743c8df32c81c1dad71dfdc747882434c55a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 29 Oct 2019 18:01:02 +0100 Subject: [PATCH 37/37] added old_way_server which can handle same way as was before and restart event server when ftrack connection fails --- pype/ftrack/ftrack_server/event_server_cli.py | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/ftrack_server/event_server_cli.py b/pype/ftrack/ftrack_server/event_server_cli.py index b0dfb48a1f..2106da3b5f 100644 --- a/pype/ftrack/ftrack_server/event_server_cli.py +++ b/pype/ftrack/ftrack_server/event_server_cli.py @@ -2,6 +2,7 @@ import os import sys import signal import datetime +import subprocess import socket import argparse import atexit @@ -125,7 +126,77 @@ def process_event_paths(event_paths): return os.pathsep.join(return_paths), not_found -def main_loop(ftrack_url, username, api_key, event_paths): +def old_way_server(ftrack_url): + # Current file + file_path = os.path.dirname(os.path.realpath(__file__)) + + min_fail_seconds = 5 + max_fail_count = 3 + wait_time_after_max_fail = 10 + + subproc = None + subproc_path = "{}/sub_old_way.py".format(file_path) + subproc_last_failed = datetime.datetime.now() + subproc_failed_count = 0 + + ftrack_accessible = False + printed_ftrack_error = False + + while True: + if not ftrack_accessible: + ftrack_accessible = check_ftrack_url(ftrack_url) + + # Run threads only if Ftrack is accessible + if not ftrack_accessible and not printed_ftrack_error: + print("Can't access Ftrack {} <{}>".format( + ftrack_url, str(datetime.datetime.now()) + )) + if subproc is not None: + if subproc.poll() is None: + subproc.terminate() + + subproc = None + + printed_ftrack_error = True + + time.sleep(1) + continue + + printed_ftrack_error = False + + if subproc is None: + if subproc_failed_count < max_fail_count: + subproc = subprocess.Popen( + ["python", subproc_path], + stdout=subprocess.PIPE + ) + elif subproc_failed_count == max_fail_count: + print(( + "Storer failed {}times I'll try to run again {}s later" + ).format(str(max_fail_count), str(wait_time_after_max_fail))) + subproc_failed_count += 1 + elif (( + datetime.datetime.now() - subproc_last_failed + ).seconds > wait_time_after_max_fail): + subproc_failed_count = 0 + + # If thread failed test Ftrack and Mongo connection + elif subproc.poll() is not None: + subproc = None + ftrack_accessible = False + + _subproc_last_failed = datetime.datetime.now() + delta_time = (_subproc_last_failed - subproc_last_failed).seconds + if delta_time < min_fail_seconds: + subproc_failed_count += 1 + else: + subproc_failed_count = 0 + subproc_last_failed = _subproc_last_failed + + time.sleep(1) + + +def main_loop(ftrack_url): """ This is main loop of event handling. Loop is handling threads which handles subprocesses of event storer and