From 898ac4b7f47c99edb5fcb5bc32a60a89e5142000 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 4 Dec 2020 12:11:31 +0100 Subject: [PATCH 01/19] add sync server to settings --- pype/modules/sync_server/providers/gdrive.py | 9 +- pype/modules/sync_server/sync_server.py | 4 +- .../defaults/project_settings/global.json | 13 + .../defaults/project_settings/maya.json | 2 +- .../project_settings/standalonepublisher.json | 2 +- .../defaults/system_settings/modules.json | 17 +- .../projects_schema/schema_main.json | 134 +++--- .../schema_project_global.json | 5 + .../schema_project_syncserver.json | 60 +++ .../system_schema/schema_modules.json | 426 +++++++++++------- 10 files changed, 424 insertions(+), 248 deletions(-) create mode 100644 pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 0e68b1963a..9e30324432 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -6,7 +6,7 @@ from googleapiclient import errors from .abstract_provider import AbstractProvider from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from pype.api import Logger -from pype.api import config +from pype.api import get_system_settings from ..utils import time_function SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly', @@ -597,7 +597,12 @@ class GDriveHandler(AbstractProvider): """ provider_presets = None try: - provider_presets = config.get_presets()["sync_server"]["gdrive"] + provider_presets = ( + get_system_settings()["modules"] + ["sync_server"] + ["providers"] + ["gdrive"] + ) except KeyError: log.info(("Sync Server: There are no presets for Gdrive " + "provider."). diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 97bda589f8..ef7a5616b7 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import config, Logger +from pype.api import get_system_settings, Logger import threading import asyncio @@ -92,7 +92,7 @@ class SyncServer(): self.connection = AvalonMongoDB() try: - self.presets = config.get_presets()["sync_server"]["config"] + self.presets = get_system_settings()["sync_server"]["config"] self.sync_server_thread = SyncServerThread(self) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 5f76f2d0f6..0528544e00 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -178,5 +178,18 @@ "editorial[ftrack.Folder]": {} } } + }, + "sync_server": { + "enabled": false, + "config": { + "local_id": "", + "retry_cnt": "", + "loop_delay": "", + "active_site": "", + "remote_site": "" + }, + "providers": { + "gdrive": {} + } } } \ No newline at end of file diff --git a/pype/settings/defaults/project_settings/maya.json b/pype/settings/defaults/project_settings/maya.json index 9193ea2b52..bf5b87cd9f 100644 --- a/pype/settings/defaults/project_settings/maya.json +++ b/pype/settings/defaults/project_settings/maya.json @@ -316,4 +316,4 @@ "ValidateNoAnimation": false } } -} +} \ No newline at end of file diff --git a/pype/settings/defaults/project_settings/standalonepublisher.json b/pype/settings/defaults/project_settings/standalonepublisher.json index b8015f2832..67b4826692 100644 --- a/pype/settings/defaults/project_settings/standalonepublisher.json +++ b/pype/settings/defaults/project_settings/standalonepublisher.json @@ -123,4 +123,4 @@ "help": "Script exported from matchmoving application" } } -} +} \ No newline at end of file diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index a36a3b75cf..c251ac95ca 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -169,6 +169,21 @@ "enabled": false, "workspace_name": "studio name" }, + "sync_server": { + "enabled": false, + "config": { + "local_id": "local_0", + "retry_cnt": "3", + "loop_delay": "60", + "active_site": "studio", + "remote_site": "gdrive" + }, + "providers": { + "gdrive": { + "credentials_url": "" + } + } + }, "Deadline": { "enabled": true, "DEADLINE_REST_URL": "http://localhost:8082" @@ -202,4 +217,4 @@ "Idle Manager": { "enabled": true } -} +} \ No newline at end of file diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json index 5b3c399666..9f7eb8fce3 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json @@ -2,73 +2,71 @@ "key": "project", "type": "dict", "children": [ + { + "type": "anatomy", + "key": "project_anatomy", + "children": [ { - "type": "anatomy", - "key": "project_anatomy", - "children": [ - { - "type": "anatomy_roots", - "key": "roots", - "is_file": true - }, - { - "type": "schema", - "name": "schema_anatomy_templates" - }, - { - "type": "schema", - "name": "schema_anatomy_attributes" - }, - { - "type": "schema", - "name": "schema_anatomy_imageio" - } - ] - }, { - "type": "dict", - "key": "project_settings", - "children": [ - { - "type": "schema", - "name": "schema_project_global" - }, - { - "type": "schema", - "name": "schema_project_ftrack" - }, - { - "type": "schema", - "name": "schema_project_maya" - }, - { - "type": "schema", - "name": "schema_project_nuke" - }, - { - "type": "schema", - "name": "schema_project_hiero" - }, - { - "type": "schema", - "name": "schema_project_harmony" - }, - { - "type": "schema", - "name": "schema_project_celaction" - }, - { - "type": "schema", - "name": "schema_project_resolve" - }, - { - "type": "schema", - "name": "schema_project_standalonepublisher" - }, - { - "type": "schema", - "name": "schema_project_unreal" - } - ] - } - ] + "type": "anatomy_roots", + "key": "roots", + "is_file": true + }, + { + "type": "schema", + "name": "schema_anatomy_templates" + }, + { + "type": "schema", + "name": "schema_anatomy_attributes" + }, + { + "type": "schema", + "name": "schema_anatomy_imageio" + }] + }, + { + "type": "dict", + "key": "project_settings", + "children": [ + { + "type": "schema", + "name": "schema_project_global" + }, + { + "type": "schema", + "name": "schema_project_ftrack" + }, + { + "type": "schema", + "name": "schema_project_maya" + }, + { + "type": "schema", + "name": "schema_project_nuke" + }, + { + "type": "schema", + "name": "schema_project_hiero" + }, + { + "type": "schema", + "name": "schema_project_harmony" + }, + { + "type": "schema", + "name": "schema_project_celaction" + }, + { + "type": "schema", + "name": "schema_project_resolve" + }, + { + "type": "schema", + "name": "schema_project_standalonepublisher" + }, + { + "type": "schema", + "name": "schema_project_unreal" + }] + }] } diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json index 75731fe207..efc751dfc2 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json @@ -23,6 +23,11 @@ "key": "project_folder_structure", "label": "" }] + }, + + { + "type": "schema", + "name": "schema_project_syncserver" } ] } diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json new file mode 100644 index 0000000000..0ec96c597c --- /dev/null +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -0,0 +1,60 @@ +{ + "type": "dict", + "key": "sync_server", + "label": "Sync Server (currently unused)", + "collapsable": true, + "checkbox_key": "enabled", + "is_file": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "dict", + "key": "config", + "label": "Config", + "collapsable": true, + "children": [ + + { + "type": "text", + "key": "local_id", + "label": "Local ID" + }, + { + "type": "text", + "key": "retry_cnt", + "label": "Retry Count" + }, + { + "type": "text", + "key": "loop_delay", + "label": "Loop Delay" + }, + { + "type": "text", + "key": "active_site", + "label": "Active Site" + }, + { + "type": "text", + "key": "remote_site", + "label": "Remote Site" + } + ] + }, { + "type": "dict", + "key": "providers", + "label": "Providers", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "raw-json", + "key": "gdrive", + "label": "Gdrive Provider" + }] + }] +} diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index 31eaab2ede..392ef91655 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -4,185 +4,265 @@ "label": "Modules", "collapsable": true, "is_file": true, - "children": [{ + "children": [ + { + "type": "dict", + "key": "Avalon", + "label": "Avalon", + "collapsable": true, + "children": [ + { + "type": "text", + "key": "AVALON_MONGO", + "label": "Avalon Mongo URL" + }, + { + "type": "text", + "key": "AVALON_DB_DATA", + "label": "Avalon Mongo Data Location" + }, + { + "type": "text", + "key": "AVALON_THUMBNAIL_ROOT", + "label": "Thumbnail Storage Location" + }, + { + "key": "environment", + "label": "Environment", + "type": "raw-json", + "env_group_key": "avalon" + }] + }, + { + "type": "schema", + "name": "schema_ftrack" + }, + { + "type": "dict", + "key": "Rest Api", + "label": "Rest Api", + "collapsable": true, + "children": [ + { + "type": "number", + "key": "default_port", + "label": "Default Port", + "minimum": 1, + "maximum": 65535 + }, + { + "type": "list", + "key": "exclude_ports", + "label": "Exclude ports", + "object_type": + { + "type": "number", + "minimum": 1, + "maximum": 65535 + } + }] + }, + { + "type": "dict", + "key": "Timers Manager", + "label": "Timers Manager", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "number", + "decimal": 2, + "key": "full_time", + "label": "Max idle time" + }, + { + "type": "number", + "decimal": 2, + "key": "message_time", + "label": "When dialog will show" + }] + }, + { + "type": "dict", + "key": "Clockify", + "label": "Clockify", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "workspace_name", + "label": "Workspace name" + }] + }, + { + "type": "dict", + "key": "sync_server", + "label": "Sync Server", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { "type": "dict", - "key": "Avalon", - "label": "Avalon", + "key": "config", + "label": "Config", "collapsable": true, - "children": [{ - "type": "text", - "key": "AVALON_MONGO", - "label": "Avalon Mongo URL" - }, - { - "type": "text", - "key": "AVALON_DB_DATA", - "label": "Avalon Mongo Data Location" - }, - { - "type": "text", - "key": "AVALON_THUMBNAIL_ROOT", - "label": "Thumbnail Storage Location" - }, - { - "key": "environment", - "label": "Environment", - "type": "raw-json", - "env_group_key": "avalon" - } - ] - }, { - "type": "schema", - "name": "schema_ftrack" + "children": [ + + { + "type": "text", + "key": "local_id", + "label": "Local ID" + }, + { + "type": "text", + "key": "retry_cnt", + "label": "Retry Count" + }, + { + "type": "text", + "key": "loop_delay", + "label": "Loop Delay" + }, + { + "type": "text", + "key": "active_site", + "label": "Active Site" + }, + { + "type": "text", + "key": "remote_site", + "label": "Remote Site" + }] }, { "type": "dict", - "key": "Rest Api", - "label": "Rest Api", - "collapsable": true, - "children": [{ - "type": "number", - "key": "default_port", - "label": "Default Port", - "minimum": 1, - "maximum": 65535 - }, - { - "type": "list", - "key": "exclude_ports", - "label": "Exclude ports", - "object_type": { - "type": "number", - "minimum": 1, - "maximum": 65535 - } - } - ] - }, { - "type": "dict", - "key": "Timers Manager", - "label": "Timers Manager", + "key": "providers", + "label": "Providers", "collapsable": true, "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "number", - "decimal": 2, - "key": "full_time", - "label": "Max idle time" - }, { - "type": "number", - "decimal": 2, - "key": "message_time", - "label": "When dialog will show" - } - ] - }, { - "type": "dict", - "key": "Clockify", - "label": "Clockify", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "text", - "key": "workspace_name", - "label": "Workspace name" - } - ] - }, { - "type": "dict", - "key": "Deadline", - "label": "Deadline", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, { - "type": "text", - "key": "DEADLINE_REST_URL", - "label": "Deadline Resl URL" + "children": [ + { + "type": "raw-json", + "key": "gdrive", + "label": "Gdrive Provider" }] - }, { - "type": "dict", - "key": "Muster", - "label": "Muster", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, { - "type": "text", - "key": "MUSTER_REST_URL", - "label": "Muster Resl URL" - }, { - "type": "dict-modifiable", - "object_type": { - "type": "number", - "minimum": 0, - "maximum": 300 - }, - "is_group": true, - "key": "templates_mapping", - "label": "Templates mapping", - "is_file": true - }] - }, { - "type": "dict", - "key": "Logging", - "label": "Logging", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, { - "type": "dict", - "key": "User setting", - "label": "User setting", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, { - "type": "dict", - "key": "Standalone Publish", - "label": "Standalone Publish", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, { - "type": "dict", - "key": "Idle Manager", - "label": "Idle Manager", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - } - ] + }] + }, + { + "type": "dict", + "key": "Deadline", + "label": "Deadline", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "DEADLINE_REST_URL", + "label": "Deadline Resl URL" + }] + }, + { + "type": "dict", + "key": "Muster", + "label": "Muster", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "MUSTER_REST_URL", + "label": "Muster Resl URL" + }, + { + "type": "dict-modifiable", + "object_type": + { + "type": "number", + "minimum": 0, + "maximum": 300 + }, + "is_group": true, + "key": "templates_mapping", + "label": "Templates mapping", + "is_file": true + }] + }, + { + "type": "dict", + "key": "Logging", + "label": "Logging", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }] + }, + { + "type": "dict", + "key": "User setting", + "label": "User setting", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }] + }, + { + "type": "dict", + "key": "Standalone Publish", + "label": "Standalone Publish", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }] + }, + { + "type": "dict", + "key": "Idle Manager", + "label": "Idle Manager", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }] + }] } From c8860eaf1994ce45d32cd57dfcbfacffacff7c3d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 7 Dec 2020 20:50:24 +0100 Subject: [PATCH 02/19] Sync Server - fixed usage of settings Implemented settings per project Updated defaults settings --- pype/modules/sync_server/providers/gdrive.py | 10 +- pype/modules/sync_server/providers/lib.py | 18 +- pype/modules/sync_server/sync_server.py | 206 ++++++++++-------- .../defaults/project_settings/global.json | 20 +- .../defaults/system_settings/modules.json | 18 +- 5 files changed, 152 insertions(+), 120 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 9e30324432..2207fdf3a3 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -25,10 +25,14 @@ class GDriveHandler(AbstractProvider): slow and should be run only when necessary. Currently is set to lazy creation, created only after first call when necessary. - Configuration for provider is in pype-config/presets/gdrive.json + Configuration for provider is in + 'settings/defaults/project_settings/global.json' + + Settings could be overwritten per project. Example of config: "gdrive": { - site name + "provider": "gdrive", - type of provider, label must be registered "credentials_url": "/my_secret_folder/credentials.json", "root": { - could be "root": "/My Drive" for single root "root_one": "/My Drive", @@ -39,12 +43,12 @@ class GDriveHandler(AbstractProvider): FOLDER_STR = 'application/vnd.google-apps.folder' MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive - def __init__(self, site_name, tree=None): + def __init__(self, site_name, tree=None, presets=None): self.presets = None self.active = False self.site_name = site_name - self.presets = self.get_presets().get(site_name, None) + self.presets = presets if not self.presets: log.info("Sync Server: There are no presets for {}.". format(site_name)) diff --git a/pype/modules/sync_server/providers/lib.py b/pype/modules/sync_server/providers/lib.py index 5aaa7a78fd..a6a52f0624 100644 --- a/pype/modules/sync_server/providers/lib.py +++ b/pype/modules/sync_server/providers/lib.py @@ -29,7 +29,7 @@ class ProviderFactory: """ self.providers[provider] = (creator, batch_limit) - def get_provider(self, provider, site_name, tree=None): + def get_provider(self, provider, site_name, tree=None, presets=None): """ Returns new instance of provider client for specific site. One provider could have multiple sites. @@ -42,11 +42,13 @@ class ProviderFactory: site_name (string): descriptor of site, different service accounts must have different site name tree (dictionary): - folder paths to folder id structure + presets (dictionary): config for provider and site (eg. + "credentials_url"..) Returns: (implementation of AbstractProvider) """ creator_info = self._get_creator_info(provider) - site = creator_info[0](site_name, tree) # call init + site = creator_info[0](site_name, tree, presets) # call init return site @@ -68,10 +70,16 @@ class ProviderFactory: def _get_creator_info(self, provider): """ Collect all necessary info for provider. Currently only creator - class and batch limit + class and batch limit. Args: provider (string): 'gdrive' etc Returns: + (tuple): (creator, batch_limit) + creator is class of a provider (ex: GDriveHandler) + batch_limit denotes how many files synced at single loop + its provided via 'register_provider' as its needed even + before provider class is initialized itself + (setting it as a class variable didn't work) """ creator_info = self.providers.get(provider) if not creator_info: @@ -81,4 +89,8 @@ class ProviderFactory: factory = ProviderFactory() +# this says that there is implemented provider with a label 'gdrive' +# there is implementing 'GDriveHandler' class +# 7 denotes number of files that could be synced in single loop - learned by +# trial and error factory.register_provider('gdrive', GDriveHandler, 7) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index ef7a5616b7..0eff2b3b20 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import get_system_settings, Logger +from pype.api import get_system_settings, get_project_settings, Logger import threading import asyncio @@ -92,79 +92,89 @@ class SyncServer(): self.connection = AvalonMongoDB() try: - self.presets = get_system_settings()["sync_server"]["config"] + self.presets = self.get_synced_presets() + self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + except KeyError as exp: + log.debug(("There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) - self.active_site = self.presets["active_site"] - self.remote_site = self.presets["remote_site"] - - # try to activate providers, need to have valid credentials - self.active_sites = [] - for provider in lib.factory.providers.keys(): - for site in lib.factory.providers[provider][0].get_presets(). \ - keys(): - handler = lib.factory.get_provider(provider, site) - if handler.is_active(): - self.active_sites.append((provider, site)) - except KeyError: - log.debug(("There are not set presets for SyncServer." - " No credentials provided, no syncing possible"). - format(str(self.presets))) - - @property - def active_site(self): + def get_synced_presets(self): """ - Returns active 'local' site (could be personal location on user - laptop or general 'studio' mounted disk. - Its 'mine' part of synchronization. - + Collects all projects which have enabled syncing and their settings Returns: - (string) + (dict): of settings, keys are project names """ - return self._active_site + sync_presets = {} + for collection in self.connection.database.collection_names(False): + sync_settings = self.get_synced_preset(collection) + if sync_settings: + sync_presets[collection] = sync_settings - @active_site.setter - def active_site(self, value): + return sync_presets + + def get_synced_preset(self, project_name): + """ Handles pulling sync_server's settings for enabled 'project_name' + + Args: + project_name (str): used in project settings + Returns: + (dict): settings dictionary for the enabled project, + empty if no settings or sync is disabled """ - Sets 'mine' part of synchronization process. It is expected only - single site is active at the time. Active site could be changed - though on different location (user working in studio has - 'active_site' = 'studio', when user is at home changes - 'active_site' to 'john_doe_local_001'. + settings = get_project_settings(project_name) + sync_settings = settings.get("global")["sync_server"] + if not sync_settings: + log.debug("No project setting for sync_server, not syncing.") + return {} + if sync_settings.get("enabled"): + return sync_settings + return {} + + def set_active_sites(self, settings): + """ + Sets 'self.active_sites' as a dictionary from provided 'settings' + + Format: + { 'project_name' : ('provider_name', 'site_name') } Args: - value (string): label for site, needs to match representation's - 'files.site'.keys() - - Returns: - (string) + settings (dict): all enabled project sync setting (sites labesl, + retries count etc.) """ - self._active_site = value + self.active_sites = {} + for project_name, project_setting in settings.items(): + for site_name, config in project_setting.get("sites").items(): + handler = lib.factory.get_provider(config["provider"], + site_name, + presets=config) + if handler.is_active(): + if not self.active_sites.get('project_name'): + self.active_sites[project_name] = [] - @property - def remote_site(self): - """ - Remote side of synchronization, where "to synchronize to". - Currently expected only single remote destination ('gdrive'..), - but prepared for multiple. - Denotes 'theirs' side of synchronization. + self.active_sites[project_name].append( + (config["provider"], site_name)) - Returns: - (list) of strings (['gdrive']) - """ - return [self._remote_site] + if not self.active_sites: + log.debug("No sync sites active, no working credentials provided") - @remote_site.setter - def remote_site(self, value): - self._remote_site = value + def get_active_sites(self, project_name): + """ + Returns active sites (provider configured and able to connect) per + project. - def get_collections(self): + Args: + project_name (str): used as a key in dict + + Returns: + (dict): + Format: + { 'project_name' : ('provider_name', 'site_name') } """ - Returns: - (list) of strings with collection names in avalon DB - """ - return self.connection.database.collection_names(False) + return self.active_sites[project_name] @time_function def get_sync_representations(self, collection, active_site, remote_site): @@ -191,8 +201,8 @@ class SyncServer(): log.debug("Check representations for : {}".format(collection)) self.connection.Session["AVALON_PROJECT"] = collection # retry_cnt - number of attempts to sync specific file before giving up - retries_arr = self._get_retries_arr() - active_providers_str = ",".join(remote_site) + retries_arr = self._get_retries_arr(collection) + #active_providers_str = ",".join(remote_site) query = { "type": "representation", "$or": [ @@ -206,7 +216,7 @@ class SyncServer(): }}, { "files.sites": { "$elemMatch": { - "name": {"$in": [active_providers_str]}, + "name": {"$in": [remote_site]}, "created_dt": {"$exists": False}, "tries": {"$in": retries_arr} } @@ -223,7 +233,7 @@ class SyncServer(): }}, { "files.sites": { "$elemMatch": { - "name": {"$in": [active_providers_str]}, + "name": {"$in": [remote_site]}, "created_dt": {"$exists": True} } } @@ -237,18 +247,19 @@ class SyncServer(): return representations - def check_status(self, file, provider_name): + def check_status(self, file, provider_name, config_preset): """ Check synchronization status for single 'file' of single 'representation' by single 'provider'. (Eg. check if 'scene.ma' of lookdev.v10 should be synced to GDrive Always is comparing local record, eg. site with - 'name' == self.presets["active_site"] + 'name' == self.presets[PROJECT_NAME]['config']["active_site"] Args: file (dictionary): of file from representation in Mongo provider_name (string): - gdrive etc. + config_preset (dict): config about active site, retries Returns: (string) - one of SyncStatus """ @@ -262,25 +273,25 @@ class SyncServer(): tries = self._get_tries_count_from_rec(provider_rec) # file will be skipped if unsuccessfully tried over threshold # error metadata needs to be purged manually in DB to reset - if tries < self.presets["retry_cnt"]: + if tries < config_preset["retry_cnt"]: return SyncStatus.DO_UPLOAD else: _, local_rec = self._get_provider_rec( sites, - self.presets["active_site"]) or {} + config_preset["active_site"]) or {} if not local_rec or not local_rec.get("created_dt"): tries = self._get_tries_count_from_rec(local_rec) # file will be skipped if unsuccessfully tried over # threshold times, error metadata needs to be purged # manually in DB to reset - if tries < self.presets["retry_cnt"]: + if tries < config_preset["retry_cnt"]: return SyncStatus.DO_DOWNLOAD return SyncStatus.DO_NOTHING async def upload(self, file, representation, provider_name, site_name, - tree=None): + tree=None, preset=None): """ Upload single 'file' of a 'representation' to 'provider'. Source url is taken from 'file' portion, where {root} placeholder @@ -297,6 +308,7 @@ class SyncServer(): site_name (string): site on provider, single provider(gdrive) could have multiple sites (different accounts, credentials) tree (dictionary): injected memory structure for performance + preset (dictionary): site config ('credentials_url', 'root'...) """ # create ids sequentially, upload file in parallel later @@ -304,7 +316,8 @@ class SyncServer(): # this part modifies structure on 'remote_site', only single # thread can do that at a time, upload/download to prepared # structure should be run in parallel - handler = lib.factory.get_provider(provider_name, site_name, tree) + handler = lib.factory.get_provider(provider_name, site_name, + tree=tree, presets=preset) remote_file = self._get_remote_file_path(file, handler.get_roots_config() ) @@ -328,7 +341,7 @@ class SyncServer(): return file_id async def download(self, file, representation, provider_name, - site_name, tree=None): + site_name, tree=None, preset=None): """ Downloads file to local folder denoted in representation.Context. @@ -339,12 +352,14 @@ class SyncServer(): site_name (string): site on provider, single provider(gdrive) could have multiple sites (different accounts, credentials) tree (dictionary): injected memory structure for performance + preset (dictionary): site config ('credentials_url', 'root'...) Returns: (string) - 'name' of local file """ with self.lock: - handler = lib.factory.get_provider(provider_name, site_name, tree) + handler = lib.factory.get_provider(provider_name, site_name, + tree=tree, presets=preset) remote_file = self._get_remote_file_path(file, handler.get_roots_config() ) @@ -538,14 +553,14 @@ class SyncServer(): update ) - def get_loop_delay(self): + def get_loop_delay(self, project_name): """ Return count of seconds before next synchronization loop starts after finish of previous loop. Returns: (int): in seconds """ - return self.presets["loop_delay"] + return int(self.presets[project_name]["config"]["loop_delay"]) def _get_success_dict(self, file_index, site_index, new_file_id): """ @@ -642,7 +657,7 @@ class SyncServer(): path = path.format(**root_config) return path - def _get_retries_arr(self): + def _get_retries_arr(self, project_name): """ Returns array with allowed values in 'tries' field. If repre contains these values, it means it was tried to be synchronized @@ -651,7 +666,8 @@ class SyncServer(): Returns: (list) """ - arr = [i for i in range(self.presets["retry_cnt"])] + retry_cnt = self.presets[project_name].get("config")["retry_cnt"] + arr = [i for i in range(int(retry_cnt))] arr.append(None) return arr @@ -706,15 +722,16 @@ class SyncServerThread(threading.Thread): while self.is_running: import time start_time = None - for collection in self.module.get_collections(): + for collection, preset in self.module.get_synced_presets().\ + items(): start_time = time.time() sync_repres = self.module.get_sync_representations( collection, - self.module.active_site, - self.module.remote_site + preset.get('config')["active_site"], + preset.get('config')["remote_site"] ) - local = self.module.active_site + local = preset.get('config')["active_site"] task_files_to_process = [] files_processed_info = [] # process only unique file paths in one batch @@ -723,9 +740,12 @@ class SyncServerThread(threading.Thread): # upload process can find already uploaded file and # reuse same id processed_file_path = set() - for active_site in self.module.active_sites: - provider, site = active_site - handler = lib.factory.get_provider(provider, site) + for check_site in self.module.get_active_sites(collection): + provider, site = check_site + site_preset = preset.get('sites')[site] + handler = lib.factory.get_provider(provider, + site, + presets=site_preset) limit = lib.factory.get_provider_batch_limit(provider) # first call to get_provider could be expensive, its # building folder tree structure in memory @@ -741,8 +761,10 @@ class SyncServerThread(threading.Thread): if file_path in processed_file_path: continue - status = self.module.check_status(file, - provider) + status = self.module.check_status( + file, + provider, + preset.get('config')) if status == SyncStatus.DO_UPLOAD: tree = handler.get_tree() limit -= 1 @@ -751,7 +773,8 @@ class SyncServerThread(threading.Thread): sync, provider, site, - tree)) + tree, + site_preset)) task_files_to_process.append(task) # store info for exception handling files_processed_info.append((file, @@ -762,11 +785,12 @@ class SyncServerThread(threading.Thread): tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.download(file, - sync, - provider, - site, - tree)) + self.module.download(file, + sync, + provider, + site, + tree, + site_preset)) task_files_to_process.append(task) files_processed_info.append((file, @@ -794,7 +818,7 @@ class SyncServerThread(threading.Thread): duration = time.time() - start_time log.debug("One loop took {:.2f}s".format(duration)) - await asyncio.sleep(self.module.get_loop_delay()) + await asyncio.sleep(self.module.get_loop_delay(collection)) except ConnectionResetError: log.warning("ConnectionResetError in sync loop, trying next loop", exc_info=True) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 0528544e00..e68292516a 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -180,16 +180,20 @@ } }, "sync_server": { - "enabled": false, + "enabled": true, "config": { - "local_id": "", - "retry_cnt": "", - "loop_delay": "", - "active_site": "", - "remote_site": "" + "local_id": "local_0", + "retry_cnt": "3", + "loop_delay": "60", + "active_site": "studio", + "remote_site": "gdrive" }, - "providers": { - "gdrive": {} + "sites": { + "gdrive": { + "provider": "gdrive", + "credentials_url": "", + "root": "" + } } } } \ No newline at end of file diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index c251ac95ca..c7fe8435b6 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -169,23 +169,11 @@ "enabled": false, "workspace_name": "studio name" }, - "sync_server": { - "enabled": false, - "config": { - "local_id": "local_0", - "retry_cnt": "3", - "loop_delay": "60", - "active_site": "studio", - "remote_site": "gdrive" - }, - "providers": { - "gdrive": { - "credentials_url": "" - } - } + "Sync Server": { + "enabled": true }, "Deadline": { - "enabled": true, + "enabled": false, "DEADLINE_REST_URL": "http://localhost:8082" }, "Muster": { From 2423b348b1602d9e85c78d5f87978a62bc46df57 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 8 Dec 2020 14:55:18 +0100 Subject: [PATCH 03/19] Sync Server - fixed usage of settings Updated integrate_new.py Updated documentation --- pype/modules/sync_server/README.md | 69 +++++++++++-------- pype/modules/sync_server/sync_server.py | 31 ++++++--- pype/plugins/global/publish/integrate_new.py | 17 +++-- .../defaults/project_settings/global.json | 4 +- .../defaults/system_settings/modules.json | 2 +- 5 files changed, 80 insertions(+), 43 deletions(-) diff --git a/pype/modules/sync_server/README.md b/pype/modules/sync_server/README.md index e6c055c1fb..8ecf849a4e 100644 --- a/pype/modules/sync_server/README.md +++ b/pype/modules/sync_server/README.md @@ -17,61 +17,74 @@ Quick HOWTOs: I want to start syncing my newly published files: ------------------------------------------------ -Get credentials for service account, share target folder on Gdrive with it -Set path to stored credentils file in gdrive.json -Set name of site, root folder in gdrive.json -Update config.json/remote_site to name of site you set in previous step -Start Pype and publish +- Check that Sync server is enabled globally in + `pype/settings/defaults/system_settings/modules.json` + +- Get credentials for service account, share target folder on Gdrive with it + +- Set path to stored credentials file in + `pype/settings/defaults/project_settings/global.json`.`credentials_url` + +- Set name of site, root folder and provider('gdrive' in case of Google Drive) in + `pype/settings/defaults/project_settings/global.json`.`sites` + +- Update `pype/settings/defaults/project_settings/global.json`.`remote_site` +to name of site you set in previous step. + +- Check that project setting is enabled (in this `global.json` file) + +- Start Pype and publish My published file is not syncing: -------------------------------- -Check that representation record contains for all 'files.site' skeleton in -format: {name: "MY_CONFIGURED_REMOTE_SITE"} -Check if that record doesn't have already 'created_dt' filled. That would +- Check that representation record contains for all 'files.sites' skeleton in +format: `{name: "MY_CONFIGURED_REMOTE_SITE"}` +- Check if that record doesn't have already 'created_dt' filled. That would denote that file was synced but someone might have had removed it on remote site. -If that records contains field "error", check that "tries" field doesn't +- If that records contains field "error", check that "tries" field doesn't contain same value as threshold in config.json.retry_cnt. If it does fix the problem mentioned in 'error' field, delete 'tries' field. I want to sync my already published files: ----------------------------------------- -Configure your Pype for syncing (see first section of Howtos). -Manually add skeleton {name: "MY_CONFIGURED_REMOTE_SITE"} to all +- Configure your Pype for syncing (see first section of Howtos). +- Manually add skeleton {name: "MY_CONFIGURED_REMOTE_SITE"} to all representation.files.sites: -db.getCollection('MY_PROJECT').update({type:"representation"}, -{$set:{"files.$[].sites.MY_CONFIGURED_REMOTE_SITE" : {}}}, true, true) +`db.getCollection('MY_PROJECT').update({type:"representation"}, +{$set:{"files.$[].sites.MY_CONFIGURED_REMOTE_SITE" : {}}}, true, true)` Needed configuration: -------------------- -pype-config/presets/config.json: - "local_id": "local_0", -- identifier of user pype - "retry_cnt": 3, -- how many times try to synch file in case of error - "loop_delay": 60, -- how many seconds between sync loops - "active_site": "studio", -- which site user current, 'studio' by default, +`pype/settings/defaults/project_settings/global.json`.`sync_server`: + - `"local_id": "local_0",` -- identifier of user pype + - `"retry_cnt": 3,` -- how many times try to synch file in case of error + - `"loop_delay": 60,` -- how many seconds between sync loops + - `"active_site": "studio",` -- which site user current, 'studio' by default, could by same as 'local_id' if user is working from home without connection to studio infrastructure - "remote_site": "gdrive" -- key for site to synchronize to. Must match to site - configured in 'gdrive.json'. + - `"remote_site": "gdrive"` -- key for site to synchronize to. Must match to site + configured lower in this file. Used in IntegrateNew to prepare skeleton for syncing in the representation record. Leave empty if no syncing is wanted. This is a general configuration, 'local_id', 'active_site' and 'remote_site' will be set and changed by some GUI in the future. -pype-config/presets/gdrive.json: - "gdrive": { - site name, must be unique - "credentials_url": "/my_secret_folder/credentials.json", +`pype/settings/defaults/project_settings/global.json`.`sync_server`.`sites`: + ```- "gdrive": { - site name, must be unique + - "provider": "gdrive" -- type of provider, must be registered in 'sync_server\providers\lib.py' + - "credentials_url": "/my_secret_folder/credentials.json", -- path to credentials for service account - "root": { -- "root": "/My Drive" in simple scenario, config here for + - "root": { -- "root": "/My Drive" in simple scenario, config here for -- multiroot projects - "root_one": "/My Drive/work_folder", - "root_tow": "/My Drive/publish_folder" - } - } + - "root_one": "/My Drive/work_folder", + - "root_tow": "/My Drive/publish_folder" + } + }`` diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 0eff2b3b20..d5c2e8b73a 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import get_system_settings, get_project_settings, Logger +from pype.api import get_project_settings, get_system_settings, Logger import threading import asyncio @@ -92,12 +92,24 @@ class SyncServer(): self.connection = AvalonMongoDB() try: + module_presets = get_system_settings().\ + get("modules").get("Sync Server") + + if not module_presets: + raise ValueError("No system setting for sync.") + + if not module_presets.get("enabled"): + log.info("Sync server disabled system wide.") + return + self.presets = self.get_synced_presets() self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + except ValueError: + log.info("No system setting for sync. Not syncing.") except KeyError as exp: - log.debug(("There are not set presets for SyncServer OR " + log.info(("There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.presets)), exc_info=True) @@ -114,6 +126,9 @@ class SyncServer(): if sync_settings: sync_presets[collection] = sync_settings + if not sync_presets: + log.info("No enabled and configured projects for sync.") + return sync_presets def get_synced_preset(self, project_name): @@ -128,7 +143,7 @@ class SyncServer(): settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: - log.debug("No project setting for sync_server, not syncing.") + log.info("No project setting for sync_server, not syncing.") return {} if sync_settings.get("enabled"): return sync_settings @@ -159,7 +174,7 @@ class SyncServer(): (config["provider"], site_name)) if not self.active_sites: - log.debug("No sync sites active, no working credentials provided") + log.info("No sync sites active, no working credentials provided") def get_active_sites(self, project_name): """ @@ -273,7 +288,7 @@ class SyncServer(): tries = self._get_tries_count_from_rec(provider_rec) # file will be skipped if unsuccessfully tried over threshold # error metadata needs to be purged manually in DB to reset - if tries < config_preset["retry_cnt"]: + if tries < int(config_preset["retry_cnt"]): return SyncStatus.DO_UPLOAD else: _, local_rec = self._get_provider_rec( @@ -285,7 +300,7 @@ class SyncServer(): # file will be skipped if unsuccessfully tried over # threshold times, error metadata needs to be purged # manually in DB to reset - if tries < config_preset["retry_cnt"]: + if tries < int(config_preset["retry_cnt"]): return SyncStatus.DO_DOWNLOAD return SyncStatus.DO_NOTHING @@ -445,7 +460,7 @@ class SyncServer(): if self.presets and self.active_sites: self.sync_server_thread.start() else: - log.debug("No presets or active providers. " + + log.info("No presets or active providers. " + "Synchronization not possible.") def tray_exit(self): @@ -462,7 +477,7 @@ class SyncServer(): if not self.is_running: return try: - log.debug("Stopping sync server server") + log.info("Stopping sync server server") self.sync_server_thread.is_running = False self.sync_server_thread.stop() except Exception: diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index add201c246..7fa0a7b4d6 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -929,16 +929,25 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): Returns: rec: dictionary with filled info """ + local_site = 'studio' # default + remote_site = None + sync_server_presets = None try: - sync_server_presets = config.get_presets()["sync_server"]["config"] + settings = pype.api.get_current_project_settings() + sync_settings = settings.get("global")["sync_server"] + if not sync_settings: + log.debug("No settings for synchronization for " + + "current project. No remote site added.") + elif sync_settings["enabled"]: + sync_server_presets = sync_settings["config"] + local_site = sync_server_presets.get("active_site", + "studio").strip() + remote_site = sync_server_presets.get("remote_site") except KeyError: log.debug(("There are not set presets for SyncServer." " No credentials provided, no synching possible"). format(str(sync_server_presets))) - local_site = sync_server_presets.get("active_site", "studio").strip() - remote_site = sync_server_presets.get("remote_site") - rec = { "_id": io.ObjectId(), "path": path diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index e68292516a..8b2bf22163 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -180,7 +180,7 @@ } }, "sync_server": { - "enabled": true, + "enabled": false, "config": { "local_id": "local_0", "retry_cnt": "3", @@ -192,7 +192,7 @@ "gdrive": { "provider": "gdrive", "credentials_url": "", - "root": "" + "root": "/sync_testing/test" } } } diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index c7fe8435b6..6a2649f025 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -170,7 +170,7 @@ "workspace_name": "studio name" }, "Sync Server": { - "enabled": true + "enabled": false }, "Deadline": { "enabled": false, From 8fcfc590036514bc3fc57646d5abb230edf9b311 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 12:07:22 +0100 Subject: [PATCH 04/19] Hound --- pype/modules/sync_server/sync_server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index d5c2e8b73a..058ed2ef31 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -108,11 +108,12 @@ class SyncServer(): self.sync_server_thread = SyncServerThread(self) except ValueError: log.info("No system setting for sync. Not syncing.") - except KeyError as exp: - log.info(("There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.presets)), exc_info=True) + except KeyError: + log.info(( + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) def get_synced_presets(self): """ @@ -217,7 +218,6 @@ class SyncServer(): self.connection.Session["AVALON_PROJECT"] = collection # retry_cnt - number of attempts to sync specific file before giving up retries_arr = self._get_retries_arr(collection) - #active_providers_str = ",".join(remote_site) query = { "type": "representation", "$or": [ From baeebef538ba6ca3d777753307f4e246faa58768 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 15:04:48 +0100 Subject: [PATCH 05/19] Fix settings schema, modules shouldnt be wrapped under Sync Server Modified schema for configuration of remote sites --- .../schema_project_syncserver.json | 38 +++-- .../system_schema/schema_modules.json | 151 +----------------- 2 files changed, 29 insertions(+), 160 deletions(-) diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index 0ec96c597c..396e4ca2dc 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -44,17 +44,33 @@ "label": "Remote Site" } ] - }, { - "type": "dict", - "key": "providers", - "label": "Providers", + }, { + "type": "dict-modifiable", "collapsable": true, - "checkbox_key": "enabled", - "children": [ + "key": "sites", + "label": "Sites", + "collapsable_key": false, + "is_file": true, + "object_type": { - "type": "raw-json", - "key": "gdrive", - "label": "Gdrive Provider" - }] - }] + "type": "dict", + "children": [ + { + "type": "text", + "key": "provider", + "label": "Provider" + }, + { + "type": "text", + "key": "credentials_url", + "label": "Credentials url" + }, + { + "type": "text", + "key": "root", + "label": "Root" + }] + } + } + ] } diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index f5b8f7eed0..06cdb02f15 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -116,155 +116,8 @@ "type": "boolean", "key": "enabled", "label": "Enabled" - }, - { - "type": "dict", - "key": "Rest Api", - "label": "Rest Api", - "collapsable": true, - "children": [{ - "type": "number", - "key": "default_port", - "label": "Default Port", - "minimum": 1, - "maximum": 65535 - }, - { - "type": "list", - "key": "exclude_ports", - "label": "Exclude ports", - "object_type": { - "type": "number", - "minimum": 1, - "maximum": 65535 - } - } - ] - }, { - "type": "dict", - "key": "Timers Manager", - "label": "Timers Manager", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "number", - "decimal": 2, - "key": "full_time", - "label": "Max idle time" - }, { - "type": "number", - "decimal": 2, - "key": "message_time", - "label": "When dialog will show" - } - ] - }, { - "type": "dict", - "key": "Clockify", - "label": "Clockify", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, - { - "type": "text", - "key": "workspace_name", - "label": "Workspace name" - } - ] - }, { - "type": "dict", - "key": "deadline", - "label": "Deadline", - "collapsable": true, - "children": [ - - { - "type": "text", - "key": "DEADLINE_REST_URL", - "label": "Deadline Resl URL" - }] - }, { - "type": "dict", - "key": "muster", - "label": "Muster", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }, { - "type": "text", - "key": "MUSTER_REST_URL", - "label": "Muster Rest URL" - }, { - "type": "dict-modifiable", - "object_type": { - "type": "number", - "minimum": 0, - "maximum": 300 - }, - "is_group": true, - "key": "templates_mapping", - "label": "Templates mapping", - "is_file": true - }] - }, { - "type": "dict", - "key": "Logging", - "label": "Logging", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, { - "type": "dict", - "key": "User setting", - "label": "User setting", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, { - "type": "dict", - "key": "Standalone Publish", - "label": "Standalone Publish", - "collapsable": true, - "checkbox_key": "enabled", - "children": [{ - "type": "boolean", - "key": "enabled", - "label": "Enabled" - }] - }, - { - "type": "dict", - "key": "providers", - "label": "Providers", - "collapsable": true, - "checkbox_key": "enabled", - "children": [ - { - "type": "raw-json", - "key": "gdrive", - "label": "Gdrive Provider" - }] - }] + } + ] }, { "type": "dict", From 8641d51ff62b4cfebd4307bbfb83cd81fbdfb629 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 18:32:01 +0100 Subject: [PATCH 06/19] Fail hard if no config for sync server is present in project settings --- pype/plugins/global/publish/integrate_new.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 7fa0a7b4d6..d98c68ae80 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -934,11 +934,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sync_server_presets = None try: settings = pype.api.get_current_project_settings() - sync_settings = settings.get("global")["sync_server"] - if not sync_settings: - log.debug("No settings for synchronization for " + - "current project. No remote site added.") - elif sync_settings["enabled"]: + sync_settings = settings["global"]["sync_server"] + + if sync_settings["enabled"]: sync_server_presets = sync_settings["config"] local_site = sync_server_presets.get("active_site", "studio").strip() From 4af38b13186fa4e36c6c519f9e6373562fd4b743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 9 Dec 2020 16:13:14 +0100 Subject: [PATCH 07/19] make user settings compatible with python 2 hosts --- pype.py | 2 + pype/hosts/maya/menu.py | 2 +- pype/lib/user_settings.py | 149 +++++++++++++++++++++++++------------- requirements.txt | 1 + 4 files changed, 104 insertions(+), 50 deletions(-) diff --git a/pype.py b/pype.py index 325a073001..7f319f0d32 100644 --- a/pype.py +++ b/pype.py @@ -145,6 +145,8 @@ def boot(): os.environ["PYPE_ROOT"] = pype_root repos = os.listdir(os.path.join(pype_root, "repos")) repos = [os.path.join(pype_root, "repos", repo) for repo in repos] + # add self to python paths + repos.insert(0, pype_root) for repo in repos: sys.path.append(repo) diff --git a/pype/hosts/maya/menu.py b/pype/hosts/maya/menu.py index 9dadd8d1f5..288502a1e1 100644 --- a/pype/hosts/maya/menu.py +++ b/pype/hosts/maya/menu.py @@ -8,7 +8,7 @@ from ...lib import BuildWorkfile import maya.cmds as cmds self = sys.modules[__name__] -self._menu = os.environ['PYPE_STUDIO_NAME'] +self._menu = os.environ.get('PYPE_STUDIO_NAME') or "Pype" log = logging.getLogger(__name__) diff --git a/pype/lib/user_settings.py b/pype/lib/user_settings.py index e2d1bded8e..0b40eccb65 100644 --- a/pype/lib/user_settings.py +++ b/pype/lib/user_settings.py @@ -2,21 +2,37 @@ """Package to deal with saving and retrieving user specific settings.""" import os from datetime import datetime -from abc import ABC, abstractmethod -import configparser +from abc import ABCMeta, abstractmethod import json -from typing import Any -from functools import lru_cache -from pathlib import Path + +# disable lru cache in Python 2 +try: + from functools import lru_cache +except ImportError: + def lru_cache(maxsize): + def max_size(func): + def wrapper(*args, **kwargs): + value = func(*args, **kwargs) + return value + return wrapper + return max_size + +# ConfigParser was renamed in python3 to configparser +try: + import configparser +except ImportError: + import ConfigParser as configparser + import platform import appdirs -import keyring +import six from ..version import __version__ -class ASettingRegistry(ABC): +@six.add_metaclass(ABCMeta) +class ASettingRegistry(): """Abstract class defining structure of **SettingRegistry** class. It is implementing methods to store secure items into keyring, otherwise @@ -28,18 +44,22 @@ class ASettingRegistry(ABC): """ - def __init__(self, name: str): + def __init__(self, name): + # type: (str) -> ASettingRegistry super(ASettingRegistry, self).__init__() - # hack for cx_freeze and Windows keyring backend - if platform.system() == "Windows": - from keyring.backends import Windows - keyring.set_keyring(Windows.WinVaultKeyring()) + if six.PY3: + import keyring + # hack for cx_freeze and Windows keyring backend + if platform.system() == "Windows": + from keyring.backends import Windows + keyring.set_keyring(Windows.WinVaultKeyring()) self._name = name self._items = {} - def set_item(self, name: str, value: str) -> None: + def set_item(self, name, value): + # type: (str, str) -> None """Set item to settings registry. Args: @@ -50,7 +70,8 @@ class ASettingRegistry(ABC): self._set_item(name, value) @abstractmethod - def _set_item(self, name: str, value: str) -> None: + def _set_item(self, name, value): + # type: (str, str) -> None # Implement it pass @@ -58,7 +79,8 @@ class ASettingRegistry(ABC): self._items[name] = value self._set_item(name, value) - def get_item(self, name: str) -> str: + def get_item(self, name): + # type: (str) -> str """Get item from settings registry. Args: @@ -74,27 +96,27 @@ class ASettingRegistry(ABC): return self._get_item(name) @abstractmethod - def _get_item(self, name: str) -> str: + def _get_item(self, name): + # type: (str) -> str # Implement it pass def __getitem__(self, name): return self._get_item(name) - def delete_item(self, name: str): + def delete_item(self, name): + # type: (str) -> None """Delete item from settings registry. Args: name (str): Name of the item. - Returns: - value (str): Value of the item. - """ self._delete_item(name) @abstractmethod - def _delete_item(self, name: str): + def _delete_item(self, name): + # type: (str) -> None """Delete item from settings. Note: @@ -107,7 +129,8 @@ class ASettingRegistry(ABC): del self._items[name] self._delete_item(name) - def set_secure_item(self, name: str, value: str) -> None: + def set_secure_item(self, name, value): + # type: (str, str) -> None """Set sensitive item into system's keyring. This uses `Keyring module`_ to save sensitive stuff into system's @@ -121,10 +144,15 @@ class ASettingRegistry(ABC): https://github.com/jaraco/keyring """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring keyring.set_password(self._name, name, value) @lru_cache(maxsize=32) - def get_secure_item(self, name: str) -> str: + def get_secure_item(self, name): + # type: (str) -> str """Get value of sensitive item from system's keyring. See also `Keyring module`_ @@ -142,13 +170,19 @@ class ASettingRegistry(ABC): https://github.com/jaraco/keyring """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring value = keyring.get_password(self._name, name) if not value: raise ValueError( - f"Item {self._name}:{name} does not exist in keyring.") + "Item {}:{} does not exist in keyring.".format( + self._name, name)) return value - def delete_secure_item(self, name: str) -> None: + def delete_secure_item(self, name): + # type: (str) -> None """Delete value stored in system's keyring. See also `Keyring module`_ @@ -160,6 +194,10 @@ class ASettingRegistry(ABC): https://github.com/jaraco/keyring """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring self.get_secure_item.cache_clear() keyring.delete_password(self._name, name) @@ -171,19 +209,21 @@ class IniSettingRegistry(ASettingRegistry): """ - def __init__(self, name, path: str): + def __init__(self, name, path): + # type: (str, str) -> IniSettingRegistry super(IniSettingRegistry, self).__init__(name) # get registry file - self._registry_file = os.path.join(path, f"{name}.ini") + self._registry_file = os.path.join(path, "{}.ini".format(name)) if not os.path.exists(self._registry_file): with open(self._registry_file, mode="w") as cfg: print("# Settings registry", cfg) - print(f"# Generated by Pype {__version__}", cfg) + print("# Generated by Pype {}".format(__version__), cfg) now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - print(f"# {now} ") + print("# {}".format(now), cfg) def set_item_section( - self, section: str, name: str, value: str) -> None: + self, section, name, value): + # type: (str, str, str) -> None """Set item to specific section of ini registry. If section doesn't exists, it is created. @@ -206,11 +246,12 @@ class IniSettingRegistry(ASettingRegistry): with open(self._registry_file, mode="w") as cfg: config.write(cfg) - def _set_item(self, name: str, value: str) -> None: - + def _set_item(self, name, value): + # type: (str, str) -> None self.set_item_section("MAIN", name, value) - def set_item(self, name: str, value: str) -> None: + def set_item(self, name, value): + # type: (str, str) -> None """Set item to settings ini file. This saves item to ``DEFAULT`` section of ini as each item there @@ -225,7 +266,8 @@ class IniSettingRegistry(ASettingRegistry): # we cast value to str as ini options values must be strings. super(IniSettingRegistry, self).set_item(name, str(value)) - def get_item(self, name: str) -> str: + def get_item(self, name): + # type: (str) -> str """Gets item from settings ini file. This gets settings from ``DEFAULT`` section of ini file as each item @@ -244,7 +286,8 @@ class IniSettingRegistry(ASettingRegistry): return super(IniSettingRegistry, self).get_item(name) @lru_cache(maxsize=32) - def get_item_from_section(self, section: str, name: str) -> str: + def get_item_from_section(self, section, name): + # type: (str, str) -> str """Get item from section of ini file. This will read ini file and try to get item value from specified @@ -268,13 +311,15 @@ class IniSettingRegistry(ASettingRegistry): value = config[section][name] except KeyError: raise ValueError( - f"Registry doesn't contain value {section}:{name}") + "Registry doesn't contain value {}:{}".format(section, name)) return value - def _get_item(self, name: str) -> str: + def _get_item(self, name): + # type: (str) -> str return self.get_item_from_section("MAIN", name) - def delete_item_from_section(self, section: str, name: str) -> None: + def delete_item_from_section(self, section, name): + # type: (str, str) -> None """Delete item from section in ini file. Args: @@ -292,7 +337,7 @@ class IniSettingRegistry(ASettingRegistry): _ = config[section][name] except KeyError: raise ValueError( - f"Registry doesn't contain value {section}:{name}") + "Registry doesn't contain value {}:{}".format(section, name)) config.remove_option(section, name) # if section is empty, delete it @@ -315,10 +360,11 @@ class IniSettingRegistry(ASettingRegistry): class JSONSettingRegistry(ASettingRegistry): """Class using json file as storage.""" - def __init__(self, name, path: str): + def __init__(self, name, path): + # type: (str, str) -> JSONSettingRegistry super(JSONSettingRegistry, self).__init__(name) #: str: name of registry file - self._registry_file = Path(os.path.join(path, f"{name}.json")) + self._registry_file = os.path.join(path, "{}.json".format(name)) now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") header = { "__metadata__": { @@ -328,14 +374,15 @@ class JSONSettingRegistry(ASettingRegistry): "registry": {} } - if not self._registry_file.parent.exists(): - self._registry_file.parent.mkdir(parents=True) + if not os.path.exists(os.path.dirname(self._registry_file)): + os.makedirs(os.path.dirname(self._registry_file), exist_ok=True) if not os.path.exists(self._registry_file): with open(self._registry_file, mode="w") as cfg: json.dump(header, cfg, indent=4) @lru_cache(maxsize=32) - def _get_item(self, name: str) -> Any: + def _get_item(self, name): + # type: (str) -> object """Get item value from registry json. Note: @@ -348,10 +395,11 @@ class JSONSettingRegistry(ASettingRegistry): value = data["registry"][name] except KeyError: raise ValueError( - f"Registry doesn't contain value {name}") + "Registry doesn't contain value {}".format(name)) return value - def get_item(self, name: str) -> Any: + def get_item(self, name): + # type: (str) -> object """Get item value from registry json. Args: @@ -366,7 +414,8 @@ class JSONSettingRegistry(ASettingRegistry): """ return self._get_item(name) - def _set_item(self, name: str, value: Any) -> None: + def _set_item(self, name, value): + # type: (str, object) -> None """Set item value to registry json. Note: @@ -380,7 +429,8 @@ class JSONSettingRegistry(ASettingRegistry): cfg.seek(0) json.dump(data, cfg, indent=4) - def set_item(self, name: str, value: Any) -> None: + def set_item(self, name, value): + # type: (str, object) -> None """Set item and its value into json registry file. Args: @@ -390,7 +440,8 @@ class JSONSettingRegistry(ASettingRegistry): """ self._set_item(name, value) - def _delete_item(self, name: str): + def _delete_item(self, name): + # type: (str) -> None self._get_item.cache_clear() with open(self._registry_file, "r+") as cfg: data = json.load(cfg) diff --git a/requirements.txt b/requirements.txt index 18b63041d6..ac197b9b1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ pytest-print pyqt5 Qt.py speedcopy +six Sphinx sphinx-rtd-theme sphinxcontrib-websupport From ecf220156eec101eb6f3c88f4798f54d144ea209 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 4 Dec 2020 12:11:31 +0100 Subject: [PATCH 08/19] add sync server to settings --- pype/modules/sync_server/providers/gdrive.py | 9 +- pype/modules/sync_server/sync_server.py | 4 +- .../defaults/project_settings/global.json | 13 ++ .../defaults/system_settings/modules.json | 15 ++ .../projects_schema/schema_main.json | 134 +++++++++--------- .../schema_project_global.json | 5 + .../schema_project_syncserver.json | 60 ++++++++ .../system_schema/schema_modules.json | 11 ++ 8 files changed, 179 insertions(+), 72 deletions(-) create mode 100644 pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 0e68b1963a..9e30324432 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -6,7 +6,7 @@ from googleapiclient import errors from .abstract_provider import AbstractProvider from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from pype.api import Logger -from pype.api import config +from pype.api import get_system_settings from ..utils import time_function SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly', @@ -597,7 +597,12 @@ class GDriveHandler(AbstractProvider): """ provider_presets = None try: - provider_presets = config.get_presets()["sync_server"]["gdrive"] + provider_presets = ( + get_system_settings()["modules"] + ["sync_server"] + ["providers"] + ["gdrive"] + ) except KeyError: log.info(("Sync Server: There are no presets for Gdrive " + "provider."). diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 97bda589f8..ef7a5616b7 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import config, Logger +from pype.api import get_system_settings, Logger import threading import asyncio @@ -92,7 +92,7 @@ class SyncServer(): self.connection = AvalonMongoDB() try: - self.presets = config.get_presets()["sync_server"]["config"] + self.presets = get_system_settings()["sync_server"]["config"] self.sync_server_thread = SyncServerThread(self) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index da56fd34e7..b3cc1bcaae 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -178,5 +178,18 @@ "editorial[ftrack.Folder]": {} } } + }, + "sync_server": { + "enabled": false, + "config": { + "local_id": "", + "retry_cnt": "", + "loop_delay": "", + "active_site": "", + "remote_site": "" + }, + "providers": { + "gdrive": {} + } } } diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index 93c099a43e..a985a789b5 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -146,6 +146,21 @@ "enabled": false, "workspace_name": "studio name" }, + "sync_server": { + "enabled": false, + "config": { + "local_id": "local_0", + "retry_cnt": "3", + "loop_delay": "60", + "active_site": "studio", + "remote_site": "gdrive" + }, + "providers": { + "gdrive": { + "credentials_url": "" + } + } + }, "deadline": { "enabled": true, "DEADLINE_REST_URL": "http://localhost:8082" diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json index 5b3c399666..9f7eb8fce3 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_main.json @@ -2,73 +2,71 @@ "key": "project", "type": "dict", "children": [ + { + "type": "anatomy", + "key": "project_anatomy", + "children": [ { - "type": "anatomy", - "key": "project_anatomy", - "children": [ - { - "type": "anatomy_roots", - "key": "roots", - "is_file": true - }, - { - "type": "schema", - "name": "schema_anatomy_templates" - }, - { - "type": "schema", - "name": "schema_anatomy_attributes" - }, - { - "type": "schema", - "name": "schema_anatomy_imageio" - } - ] - }, { - "type": "dict", - "key": "project_settings", - "children": [ - { - "type": "schema", - "name": "schema_project_global" - }, - { - "type": "schema", - "name": "schema_project_ftrack" - }, - { - "type": "schema", - "name": "schema_project_maya" - }, - { - "type": "schema", - "name": "schema_project_nuke" - }, - { - "type": "schema", - "name": "schema_project_hiero" - }, - { - "type": "schema", - "name": "schema_project_harmony" - }, - { - "type": "schema", - "name": "schema_project_celaction" - }, - { - "type": "schema", - "name": "schema_project_resolve" - }, - { - "type": "schema", - "name": "schema_project_standalonepublisher" - }, - { - "type": "schema", - "name": "schema_project_unreal" - } - ] - } - ] + "type": "anatomy_roots", + "key": "roots", + "is_file": true + }, + { + "type": "schema", + "name": "schema_anatomy_templates" + }, + { + "type": "schema", + "name": "schema_anatomy_attributes" + }, + { + "type": "schema", + "name": "schema_anatomy_imageio" + }] + }, + { + "type": "dict", + "key": "project_settings", + "children": [ + { + "type": "schema", + "name": "schema_project_global" + }, + { + "type": "schema", + "name": "schema_project_ftrack" + }, + { + "type": "schema", + "name": "schema_project_maya" + }, + { + "type": "schema", + "name": "schema_project_nuke" + }, + { + "type": "schema", + "name": "schema_project_hiero" + }, + { + "type": "schema", + "name": "schema_project_harmony" + }, + { + "type": "schema", + "name": "schema_project_celaction" + }, + { + "type": "schema", + "name": "schema_project_resolve" + }, + { + "type": "schema", + "name": "schema_project_standalonepublisher" + }, + { + "type": "schema", + "name": "schema_project_unreal" + }] + }] } diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json index 75731fe207..efc751dfc2 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_global.json @@ -23,6 +23,11 @@ "key": "project_folder_structure", "label": "" }] + }, + + { + "type": "schema", + "name": "schema_project_syncserver" } ] } diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json new file mode 100644 index 0000000000..0ec96c597c --- /dev/null +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -0,0 +1,60 @@ +{ + "type": "dict", + "key": "sync_server", + "label": "Sync Server (currently unused)", + "collapsable": true, + "checkbox_key": "enabled", + "is_file": true, + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "dict", + "key": "config", + "label": "Config", + "collapsable": true, + "children": [ + + { + "type": "text", + "key": "local_id", + "label": "Local ID" + }, + { + "type": "text", + "key": "retry_cnt", + "label": "Retry Count" + }, + { + "type": "text", + "key": "loop_delay", + "label": "Loop Delay" + }, + { + "type": "text", + "key": "active_site", + "label": "Active Site" + }, + { + "type": "text", + "key": "remote_site", + "label": "Remote Site" + } + ] + }, { + "type": "dict", + "key": "providers", + "label": "Providers", + "collapsable": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "raw-json", + "key": "gdrive", + "label": "Gdrive Provider" + }] + }] +} diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index 62aaafc27b..b8c33655a0 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -99,6 +99,17 @@ } ] }, { + "type": "dict", + "key": "sync_server", + "label": "Sync Server", + "collapsable": true, + "checkbox_key": "enabled", + "children": [{ + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }] + }, { "type": "dict", "key": "deadline", "label": "Deadline", From 76e3d70ccac1e0c251aeabc695ea862f57fa4398 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 7 Dec 2020 20:50:24 +0100 Subject: [PATCH 09/19] Sync Server - fixed usage of settings Implemented settings per project Updated defaults settings --- pype/modules/sync_server/providers/gdrive.py | 10 +- pype/modules/sync_server/providers/lib.py | 18 +- pype/modules/sync_server/sync_server.py | 206 ++++++++++-------- .../defaults/project_settings/global.json | 20 +- .../defaults/system_settings/modules.json | 16 +- 5 files changed, 151 insertions(+), 119 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 9e30324432..2207fdf3a3 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -25,10 +25,14 @@ class GDriveHandler(AbstractProvider): slow and should be run only when necessary. Currently is set to lazy creation, created only after first call when necessary. - Configuration for provider is in pype-config/presets/gdrive.json + Configuration for provider is in + 'settings/defaults/project_settings/global.json' + + Settings could be overwritten per project. Example of config: "gdrive": { - site name + "provider": "gdrive", - type of provider, label must be registered "credentials_url": "/my_secret_folder/credentials.json", "root": { - could be "root": "/My Drive" for single root "root_one": "/My Drive", @@ -39,12 +43,12 @@ class GDriveHandler(AbstractProvider): FOLDER_STR = 'application/vnd.google-apps.folder' MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive - def __init__(self, site_name, tree=None): + def __init__(self, site_name, tree=None, presets=None): self.presets = None self.active = False self.site_name = site_name - self.presets = self.get_presets().get(site_name, None) + self.presets = presets if not self.presets: log.info("Sync Server: There are no presets for {}.". format(site_name)) diff --git a/pype/modules/sync_server/providers/lib.py b/pype/modules/sync_server/providers/lib.py index 5aaa7a78fd..a6a52f0624 100644 --- a/pype/modules/sync_server/providers/lib.py +++ b/pype/modules/sync_server/providers/lib.py @@ -29,7 +29,7 @@ class ProviderFactory: """ self.providers[provider] = (creator, batch_limit) - def get_provider(self, provider, site_name, tree=None): + def get_provider(self, provider, site_name, tree=None, presets=None): """ Returns new instance of provider client for specific site. One provider could have multiple sites. @@ -42,11 +42,13 @@ class ProviderFactory: site_name (string): descriptor of site, different service accounts must have different site name tree (dictionary): - folder paths to folder id structure + presets (dictionary): config for provider and site (eg. + "credentials_url"..) Returns: (implementation of AbstractProvider) """ creator_info = self._get_creator_info(provider) - site = creator_info[0](site_name, tree) # call init + site = creator_info[0](site_name, tree, presets) # call init return site @@ -68,10 +70,16 @@ class ProviderFactory: def _get_creator_info(self, provider): """ Collect all necessary info for provider. Currently only creator - class and batch limit + class and batch limit. Args: provider (string): 'gdrive' etc Returns: + (tuple): (creator, batch_limit) + creator is class of a provider (ex: GDriveHandler) + batch_limit denotes how many files synced at single loop + its provided via 'register_provider' as its needed even + before provider class is initialized itself + (setting it as a class variable didn't work) """ creator_info = self.providers.get(provider) if not creator_info: @@ -81,4 +89,8 @@ class ProviderFactory: factory = ProviderFactory() +# this says that there is implemented provider with a label 'gdrive' +# there is implementing 'GDriveHandler' class +# 7 denotes number of files that could be synced in single loop - learned by +# trial and error factory.register_provider('gdrive', GDriveHandler, 7) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index ef7a5616b7..0eff2b3b20 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import get_system_settings, Logger +from pype.api import get_system_settings, get_project_settings, Logger import threading import asyncio @@ -92,79 +92,89 @@ class SyncServer(): self.connection = AvalonMongoDB() try: - self.presets = get_system_settings()["sync_server"]["config"] + self.presets = self.get_synced_presets() + self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + except KeyError as exp: + log.debug(("There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) - self.active_site = self.presets["active_site"] - self.remote_site = self.presets["remote_site"] - - # try to activate providers, need to have valid credentials - self.active_sites = [] - for provider in lib.factory.providers.keys(): - for site in lib.factory.providers[provider][0].get_presets(). \ - keys(): - handler = lib.factory.get_provider(provider, site) - if handler.is_active(): - self.active_sites.append((provider, site)) - except KeyError: - log.debug(("There are not set presets for SyncServer." - " No credentials provided, no syncing possible"). - format(str(self.presets))) - - @property - def active_site(self): + def get_synced_presets(self): """ - Returns active 'local' site (could be personal location on user - laptop or general 'studio' mounted disk. - Its 'mine' part of synchronization. - + Collects all projects which have enabled syncing and their settings Returns: - (string) + (dict): of settings, keys are project names """ - return self._active_site + sync_presets = {} + for collection in self.connection.database.collection_names(False): + sync_settings = self.get_synced_preset(collection) + if sync_settings: + sync_presets[collection] = sync_settings - @active_site.setter - def active_site(self, value): + return sync_presets + + def get_synced_preset(self, project_name): + """ Handles pulling sync_server's settings for enabled 'project_name' + + Args: + project_name (str): used in project settings + Returns: + (dict): settings dictionary for the enabled project, + empty if no settings or sync is disabled """ - Sets 'mine' part of synchronization process. It is expected only - single site is active at the time. Active site could be changed - though on different location (user working in studio has - 'active_site' = 'studio', when user is at home changes - 'active_site' to 'john_doe_local_001'. + settings = get_project_settings(project_name) + sync_settings = settings.get("global")["sync_server"] + if not sync_settings: + log.debug("No project setting for sync_server, not syncing.") + return {} + if sync_settings.get("enabled"): + return sync_settings + return {} + + def set_active_sites(self, settings): + """ + Sets 'self.active_sites' as a dictionary from provided 'settings' + + Format: + { 'project_name' : ('provider_name', 'site_name') } Args: - value (string): label for site, needs to match representation's - 'files.site'.keys() - - Returns: - (string) + settings (dict): all enabled project sync setting (sites labesl, + retries count etc.) """ - self._active_site = value + self.active_sites = {} + for project_name, project_setting in settings.items(): + for site_name, config in project_setting.get("sites").items(): + handler = lib.factory.get_provider(config["provider"], + site_name, + presets=config) + if handler.is_active(): + if not self.active_sites.get('project_name'): + self.active_sites[project_name] = [] - @property - def remote_site(self): - """ - Remote side of synchronization, where "to synchronize to". - Currently expected only single remote destination ('gdrive'..), - but prepared for multiple. - Denotes 'theirs' side of synchronization. + self.active_sites[project_name].append( + (config["provider"], site_name)) - Returns: - (list) of strings (['gdrive']) - """ - return [self._remote_site] + if not self.active_sites: + log.debug("No sync sites active, no working credentials provided") - @remote_site.setter - def remote_site(self, value): - self._remote_site = value + def get_active_sites(self, project_name): + """ + Returns active sites (provider configured and able to connect) per + project. - def get_collections(self): + Args: + project_name (str): used as a key in dict + + Returns: + (dict): + Format: + { 'project_name' : ('provider_name', 'site_name') } """ - Returns: - (list) of strings with collection names in avalon DB - """ - return self.connection.database.collection_names(False) + return self.active_sites[project_name] @time_function def get_sync_representations(self, collection, active_site, remote_site): @@ -191,8 +201,8 @@ class SyncServer(): log.debug("Check representations for : {}".format(collection)) self.connection.Session["AVALON_PROJECT"] = collection # retry_cnt - number of attempts to sync specific file before giving up - retries_arr = self._get_retries_arr() - active_providers_str = ",".join(remote_site) + retries_arr = self._get_retries_arr(collection) + #active_providers_str = ",".join(remote_site) query = { "type": "representation", "$or": [ @@ -206,7 +216,7 @@ class SyncServer(): }}, { "files.sites": { "$elemMatch": { - "name": {"$in": [active_providers_str]}, + "name": {"$in": [remote_site]}, "created_dt": {"$exists": False}, "tries": {"$in": retries_arr} } @@ -223,7 +233,7 @@ class SyncServer(): }}, { "files.sites": { "$elemMatch": { - "name": {"$in": [active_providers_str]}, + "name": {"$in": [remote_site]}, "created_dt": {"$exists": True} } } @@ -237,18 +247,19 @@ class SyncServer(): return representations - def check_status(self, file, provider_name): + def check_status(self, file, provider_name, config_preset): """ Check synchronization status for single 'file' of single 'representation' by single 'provider'. (Eg. check if 'scene.ma' of lookdev.v10 should be synced to GDrive Always is comparing local record, eg. site with - 'name' == self.presets["active_site"] + 'name' == self.presets[PROJECT_NAME]['config']["active_site"] Args: file (dictionary): of file from representation in Mongo provider_name (string): - gdrive etc. + config_preset (dict): config about active site, retries Returns: (string) - one of SyncStatus """ @@ -262,25 +273,25 @@ class SyncServer(): tries = self._get_tries_count_from_rec(provider_rec) # file will be skipped if unsuccessfully tried over threshold # error metadata needs to be purged manually in DB to reset - if tries < self.presets["retry_cnt"]: + if tries < config_preset["retry_cnt"]: return SyncStatus.DO_UPLOAD else: _, local_rec = self._get_provider_rec( sites, - self.presets["active_site"]) or {} + config_preset["active_site"]) or {} if not local_rec or not local_rec.get("created_dt"): tries = self._get_tries_count_from_rec(local_rec) # file will be skipped if unsuccessfully tried over # threshold times, error metadata needs to be purged # manually in DB to reset - if tries < self.presets["retry_cnt"]: + if tries < config_preset["retry_cnt"]: return SyncStatus.DO_DOWNLOAD return SyncStatus.DO_NOTHING async def upload(self, file, representation, provider_name, site_name, - tree=None): + tree=None, preset=None): """ Upload single 'file' of a 'representation' to 'provider'. Source url is taken from 'file' portion, where {root} placeholder @@ -297,6 +308,7 @@ class SyncServer(): site_name (string): site on provider, single provider(gdrive) could have multiple sites (different accounts, credentials) tree (dictionary): injected memory structure for performance + preset (dictionary): site config ('credentials_url', 'root'...) """ # create ids sequentially, upload file in parallel later @@ -304,7 +316,8 @@ class SyncServer(): # this part modifies structure on 'remote_site', only single # thread can do that at a time, upload/download to prepared # structure should be run in parallel - handler = lib.factory.get_provider(provider_name, site_name, tree) + handler = lib.factory.get_provider(provider_name, site_name, + tree=tree, presets=preset) remote_file = self._get_remote_file_path(file, handler.get_roots_config() ) @@ -328,7 +341,7 @@ class SyncServer(): return file_id async def download(self, file, representation, provider_name, - site_name, tree=None): + site_name, tree=None, preset=None): """ Downloads file to local folder denoted in representation.Context. @@ -339,12 +352,14 @@ class SyncServer(): site_name (string): site on provider, single provider(gdrive) could have multiple sites (different accounts, credentials) tree (dictionary): injected memory structure for performance + preset (dictionary): site config ('credentials_url', 'root'...) Returns: (string) - 'name' of local file """ with self.lock: - handler = lib.factory.get_provider(provider_name, site_name, tree) + handler = lib.factory.get_provider(provider_name, site_name, + tree=tree, presets=preset) remote_file = self._get_remote_file_path(file, handler.get_roots_config() ) @@ -538,14 +553,14 @@ class SyncServer(): update ) - def get_loop_delay(self): + def get_loop_delay(self, project_name): """ Return count of seconds before next synchronization loop starts after finish of previous loop. Returns: (int): in seconds """ - return self.presets["loop_delay"] + return int(self.presets[project_name]["config"]["loop_delay"]) def _get_success_dict(self, file_index, site_index, new_file_id): """ @@ -642,7 +657,7 @@ class SyncServer(): path = path.format(**root_config) return path - def _get_retries_arr(self): + def _get_retries_arr(self, project_name): """ Returns array with allowed values in 'tries' field. If repre contains these values, it means it was tried to be synchronized @@ -651,7 +666,8 @@ class SyncServer(): Returns: (list) """ - arr = [i for i in range(self.presets["retry_cnt"])] + retry_cnt = self.presets[project_name].get("config")["retry_cnt"] + arr = [i for i in range(int(retry_cnt))] arr.append(None) return arr @@ -706,15 +722,16 @@ class SyncServerThread(threading.Thread): while self.is_running: import time start_time = None - for collection in self.module.get_collections(): + for collection, preset in self.module.get_synced_presets().\ + items(): start_time = time.time() sync_repres = self.module.get_sync_representations( collection, - self.module.active_site, - self.module.remote_site + preset.get('config')["active_site"], + preset.get('config')["remote_site"] ) - local = self.module.active_site + local = preset.get('config')["active_site"] task_files_to_process = [] files_processed_info = [] # process only unique file paths in one batch @@ -723,9 +740,12 @@ class SyncServerThread(threading.Thread): # upload process can find already uploaded file and # reuse same id processed_file_path = set() - for active_site in self.module.active_sites: - provider, site = active_site - handler = lib.factory.get_provider(provider, site) + for check_site in self.module.get_active_sites(collection): + provider, site = check_site + site_preset = preset.get('sites')[site] + handler = lib.factory.get_provider(provider, + site, + presets=site_preset) limit = lib.factory.get_provider_batch_limit(provider) # first call to get_provider could be expensive, its # building folder tree structure in memory @@ -741,8 +761,10 @@ class SyncServerThread(threading.Thread): if file_path in processed_file_path: continue - status = self.module.check_status(file, - provider) + status = self.module.check_status( + file, + provider, + preset.get('config')) if status == SyncStatus.DO_UPLOAD: tree = handler.get_tree() limit -= 1 @@ -751,7 +773,8 @@ class SyncServerThread(threading.Thread): sync, provider, site, - tree)) + tree, + site_preset)) task_files_to_process.append(task) # store info for exception handling files_processed_info.append((file, @@ -762,11 +785,12 @@ class SyncServerThread(threading.Thread): tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.download(file, - sync, - provider, - site, - tree)) + self.module.download(file, + sync, + provider, + site, + tree, + site_preset)) task_files_to_process.append(task) files_processed_info.append((file, @@ -794,7 +818,7 @@ class SyncServerThread(threading.Thread): duration = time.time() - start_time log.debug("One loop took {:.2f}s".format(duration)) - await asyncio.sleep(self.module.get_loop_delay()) + await asyncio.sleep(self.module.get_loop_delay(collection)) except ConnectionResetError: log.warning("ConnectionResetError in sync loop, trying next loop", exc_info=True) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index b3cc1bcaae..6fd8c3d57f 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -180,16 +180,20 @@ } }, "sync_server": { - "enabled": false, + "enabled": true, "config": { - "local_id": "", - "retry_cnt": "", - "loop_delay": "", - "active_site": "", - "remote_site": "" + "local_id": "local_0", + "retry_cnt": "3", + "loop_delay": "60", + "active_site": "studio", + "remote_site": "gdrive" }, - "providers": { - "gdrive": {} + "sites": { + "gdrive": { + "provider": "gdrive", + "credentials_url": "", + "root": "" + } } } } diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index a985a789b5..9bf5936861 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -146,20 +146,8 @@ "enabled": false, "workspace_name": "studio name" }, - "sync_server": { - "enabled": false, - "config": { - "local_id": "local_0", - "retry_cnt": "3", - "loop_delay": "60", - "active_site": "studio", - "remote_site": "gdrive" - }, - "providers": { - "gdrive": { - "credentials_url": "" - } - } + "Sync Server": { + "enabled": true }, "deadline": { "enabled": true, From f941917dabf4325284813d4eb603fcf731026d25 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 8 Dec 2020 14:55:18 +0100 Subject: [PATCH 10/19] Sync Server - fixed usage of settings Updated integrate_new.py Updated documentation --- pype/modules/sync_server/README.md | 69 +++++++++++-------- pype/modules/sync_server/sync_server.py | 31 ++++++--- pype/plugins/global/publish/integrate_new.py | 17 +++-- .../defaults/project_settings/global.json | 4 +- .../defaults/system_settings/modules.json | 2 +- 5 files changed, 80 insertions(+), 43 deletions(-) diff --git a/pype/modules/sync_server/README.md b/pype/modules/sync_server/README.md index e6c055c1fb..8ecf849a4e 100644 --- a/pype/modules/sync_server/README.md +++ b/pype/modules/sync_server/README.md @@ -17,61 +17,74 @@ Quick HOWTOs: I want to start syncing my newly published files: ------------------------------------------------ -Get credentials for service account, share target folder on Gdrive with it -Set path to stored credentils file in gdrive.json -Set name of site, root folder in gdrive.json -Update config.json/remote_site to name of site you set in previous step -Start Pype and publish +- Check that Sync server is enabled globally in + `pype/settings/defaults/system_settings/modules.json` + +- Get credentials for service account, share target folder on Gdrive with it + +- Set path to stored credentials file in + `pype/settings/defaults/project_settings/global.json`.`credentials_url` + +- Set name of site, root folder and provider('gdrive' in case of Google Drive) in + `pype/settings/defaults/project_settings/global.json`.`sites` + +- Update `pype/settings/defaults/project_settings/global.json`.`remote_site` +to name of site you set in previous step. + +- Check that project setting is enabled (in this `global.json` file) + +- Start Pype and publish My published file is not syncing: -------------------------------- -Check that representation record contains for all 'files.site' skeleton in -format: {name: "MY_CONFIGURED_REMOTE_SITE"} -Check if that record doesn't have already 'created_dt' filled. That would +- Check that representation record contains for all 'files.sites' skeleton in +format: `{name: "MY_CONFIGURED_REMOTE_SITE"}` +- Check if that record doesn't have already 'created_dt' filled. That would denote that file was synced but someone might have had removed it on remote site. -If that records contains field "error", check that "tries" field doesn't +- If that records contains field "error", check that "tries" field doesn't contain same value as threshold in config.json.retry_cnt. If it does fix the problem mentioned in 'error' field, delete 'tries' field. I want to sync my already published files: ----------------------------------------- -Configure your Pype for syncing (see first section of Howtos). -Manually add skeleton {name: "MY_CONFIGURED_REMOTE_SITE"} to all +- Configure your Pype for syncing (see first section of Howtos). +- Manually add skeleton {name: "MY_CONFIGURED_REMOTE_SITE"} to all representation.files.sites: -db.getCollection('MY_PROJECT').update({type:"representation"}, -{$set:{"files.$[].sites.MY_CONFIGURED_REMOTE_SITE" : {}}}, true, true) +`db.getCollection('MY_PROJECT').update({type:"representation"}, +{$set:{"files.$[].sites.MY_CONFIGURED_REMOTE_SITE" : {}}}, true, true)` Needed configuration: -------------------- -pype-config/presets/config.json: - "local_id": "local_0", -- identifier of user pype - "retry_cnt": 3, -- how many times try to synch file in case of error - "loop_delay": 60, -- how many seconds between sync loops - "active_site": "studio", -- which site user current, 'studio' by default, +`pype/settings/defaults/project_settings/global.json`.`sync_server`: + - `"local_id": "local_0",` -- identifier of user pype + - `"retry_cnt": 3,` -- how many times try to synch file in case of error + - `"loop_delay": 60,` -- how many seconds between sync loops + - `"active_site": "studio",` -- which site user current, 'studio' by default, could by same as 'local_id' if user is working from home without connection to studio infrastructure - "remote_site": "gdrive" -- key for site to synchronize to. Must match to site - configured in 'gdrive.json'. + - `"remote_site": "gdrive"` -- key for site to synchronize to. Must match to site + configured lower in this file. Used in IntegrateNew to prepare skeleton for syncing in the representation record. Leave empty if no syncing is wanted. This is a general configuration, 'local_id', 'active_site' and 'remote_site' will be set and changed by some GUI in the future. -pype-config/presets/gdrive.json: - "gdrive": { - site name, must be unique - "credentials_url": "/my_secret_folder/credentials.json", +`pype/settings/defaults/project_settings/global.json`.`sync_server`.`sites`: + ```- "gdrive": { - site name, must be unique + - "provider": "gdrive" -- type of provider, must be registered in 'sync_server\providers\lib.py' + - "credentials_url": "/my_secret_folder/credentials.json", -- path to credentials for service account - "root": { -- "root": "/My Drive" in simple scenario, config here for + - "root": { -- "root": "/My Drive" in simple scenario, config here for -- multiroot projects - "root_one": "/My Drive/work_folder", - "root_tow": "/My Drive/publish_folder" - } - } + - "root_one": "/My Drive/work_folder", + - "root_tow": "/My Drive/publish_folder" + } + }`` diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 0eff2b3b20..d5c2e8b73a 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,4 @@ -from pype.api import get_system_settings, get_project_settings, Logger +from pype.api import get_project_settings, get_system_settings, Logger import threading import asyncio @@ -92,12 +92,24 @@ class SyncServer(): self.connection = AvalonMongoDB() try: + module_presets = get_system_settings().\ + get("modules").get("Sync Server") + + if not module_presets: + raise ValueError("No system setting for sync.") + + if not module_presets.get("enabled"): + log.info("Sync server disabled system wide.") + return + self.presets = self.get_synced_presets() self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + except ValueError: + log.info("No system setting for sync. Not syncing.") except KeyError as exp: - log.debug(("There are not set presets for SyncServer OR " + log.info(("There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.presets)), exc_info=True) @@ -114,6 +126,9 @@ class SyncServer(): if sync_settings: sync_presets[collection] = sync_settings + if not sync_presets: + log.info("No enabled and configured projects for sync.") + return sync_presets def get_synced_preset(self, project_name): @@ -128,7 +143,7 @@ class SyncServer(): settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: - log.debug("No project setting for sync_server, not syncing.") + log.info("No project setting for sync_server, not syncing.") return {} if sync_settings.get("enabled"): return sync_settings @@ -159,7 +174,7 @@ class SyncServer(): (config["provider"], site_name)) if not self.active_sites: - log.debug("No sync sites active, no working credentials provided") + log.info("No sync sites active, no working credentials provided") def get_active_sites(self, project_name): """ @@ -273,7 +288,7 @@ class SyncServer(): tries = self._get_tries_count_from_rec(provider_rec) # file will be skipped if unsuccessfully tried over threshold # error metadata needs to be purged manually in DB to reset - if tries < config_preset["retry_cnt"]: + if tries < int(config_preset["retry_cnt"]): return SyncStatus.DO_UPLOAD else: _, local_rec = self._get_provider_rec( @@ -285,7 +300,7 @@ class SyncServer(): # file will be skipped if unsuccessfully tried over # threshold times, error metadata needs to be purged # manually in DB to reset - if tries < config_preset["retry_cnt"]: + if tries < int(config_preset["retry_cnt"]): return SyncStatus.DO_DOWNLOAD return SyncStatus.DO_NOTHING @@ -445,7 +460,7 @@ class SyncServer(): if self.presets and self.active_sites: self.sync_server_thread.start() else: - log.debug("No presets or active providers. " + + log.info("No presets or active providers. " + "Synchronization not possible.") def tray_exit(self): @@ -462,7 +477,7 @@ class SyncServer(): if not self.is_running: return try: - log.debug("Stopping sync server server") + log.info("Stopping sync server server") self.sync_server_thread.is_running = False self.sync_server_thread.stop() except Exception: diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index add201c246..7fa0a7b4d6 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -929,16 +929,25 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): Returns: rec: dictionary with filled info """ + local_site = 'studio' # default + remote_site = None + sync_server_presets = None try: - sync_server_presets = config.get_presets()["sync_server"]["config"] + settings = pype.api.get_current_project_settings() + sync_settings = settings.get("global")["sync_server"] + if not sync_settings: + log.debug("No settings for synchronization for " + + "current project. No remote site added.") + elif sync_settings["enabled"]: + sync_server_presets = sync_settings["config"] + local_site = sync_server_presets.get("active_site", + "studio").strip() + remote_site = sync_server_presets.get("remote_site") except KeyError: log.debug(("There are not set presets for SyncServer." " No credentials provided, no synching possible"). format(str(sync_server_presets))) - local_site = sync_server_presets.get("active_site", "studio").strip() - remote_site = sync_server_presets.get("remote_site") - rec = { "_id": io.ObjectId(), "path": path diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 6fd8c3d57f..ed7dc52cac 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -180,7 +180,7 @@ } }, "sync_server": { - "enabled": true, + "enabled": false, "config": { "local_id": "local_0", "retry_cnt": "3", @@ -192,7 +192,7 @@ "gdrive": { "provider": "gdrive", "credentials_url": "", - "root": "" + "root": "/sync_testing/test" } } } diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index 9bf5936861..558e078c5e 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -147,7 +147,7 @@ "workspace_name": "studio name" }, "Sync Server": { - "enabled": true + "enabled": false }, "deadline": { "enabled": true, From 7a6cdd735323bf8a7c0d449bb97c68385e6e159a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 12:07:22 +0100 Subject: [PATCH 11/19] Hound --- pype/modules/sync_server/sync_server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index d5c2e8b73a..058ed2ef31 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -108,11 +108,12 @@ class SyncServer(): self.sync_server_thread = SyncServerThread(self) except ValueError: log.info("No system setting for sync. Not syncing.") - except KeyError as exp: - log.info(("There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.presets)), exc_info=True) + except KeyError: + log.info(( + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) def get_synced_presets(self): """ @@ -217,7 +218,6 @@ class SyncServer(): self.connection.Session["AVALON_PROJECT"] = collection # retry_cnt - number of attempts to sync specific file before giving up retries_arr = self._get_retries_arr(collection) - #active_providers_str = ",".join(remote_site) query = { "type": "representation", "$or": [ From 938b116bd57ddc8b9a2c9fe8edb1d11f43e2d0c6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 15:04:48 +0100 Subject: [PATCH 12/19] Fix settings schema, modules shouldnt be wrapped under Sync Server Modified schema for configuration of remote sites --- .../schema_project_syncserver.json | 38 +++++++++++++------ .../system_schema/schema_modules.json | 4 +- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index 0ec96c597c..396e4ca2dc 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -44,17 +44,33 @@ "label": "Remote Site" } ] - }, { - "type": "dict", - "key": "providers", - "label": "Providers", + }, { + "type": "dict-modifiable", "collapsable": true, - "checkbox_key": "enabled", - "children": [ + "key": "sites", + "label": "Sites", + "collapsable_key": false, + "is_file": true, + "object_type": { - "type": "raw-json", - "key": "gdrive", - "label": "Gdrive Provider" - }] - }] + "type": "dict", + "children": [ + { + "type": "text", + "key": "provider", + "label": "Provider" + }, + { + "type": "text", + "key": "credentials_url", + "label": "Credentials url" + }, + { + "type": "text", + "key": "root", + "label": "Root" + }] + } + } + ] } diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index b8c33655a0..91ab624755 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -98,7 +98,7 @@ "label": "Workspace name" } ] - }, { + }, { "type": "dict", "key": "sync_server", "label": "Sync Server", @@ -109,7 +109,7 @@ "key": "enabled", "label": "Enabled" }] - }, { + },{ "type": "dict", "key": "deadline", "label": "Deadline", From ea5560c3f2750a2c9790f91278e57d29bfb96fcd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 9 Dec 2020 18:32:01 +0100 Subject: [PATCH 13/19] Fail hard if no config for sync server is present in project settings --- pype/plugins/global/publish/integrate_new.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 7fa0a7b4d6..d98c68ae80 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -934,11 +934,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sync_server_presets = None try: settings = pype.api.get_current_project_settings() - sync_settings = settings.get("global")["sync_server"] - if not sync_settings: - log.debug("No settings for synchronization for " + - "current project. No remote site added.") - elif sync_settings["enabled"]: + sync_settings = settings["global"]["sync_server"] + + if sync_settings["enabled"]: sync_server_presets = sync_settings["config"] local_site = sync_server_presets.get("active_site", "studio").strip() From 2e619be3d7cd5913bf1aeecf8efad69daa495887 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 10 Dec 2020 10:53:22 +0100 Subject: [PATCH 14/19] Removed avalon.io import --- pype/modules/sync_server/sync_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 058ed2ef31..5f2fbaf2fb 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -10,7 +10,7 @@ from datetime import datetime from .providers import lib import os -from avalon import io +from bson.objectid import ObjectId from avalon.api import AvalonMongoDB from .utils import time_function @@ -541,7 +541,7 @@ class SyncServer(): """ # TODO - implement reset for ALL files or ALL sites query = { - "_id": io.ObjectId(representation_id) + "_id": ObjectId(representation_id) } self.connection.Session["AVALON_PROJECT"] = collection representation = list(self.connection.find(query)) From 29763581e9af6e56d86bae3a4e4f3de244a9c540 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 10 Dec 2020 19:02:00 +0100 Subject: [PATCH 15/19] Fix schema, standardize to 'Sync Server' as a key for now Handled system wide disabling in integrate_new --- pype/modules/sync_server/__init__.py | 2 +- pype/modules/sync_server/sync_server.py | 78 +++++++++++++++---- pype/plugins/global/publish/integrate_new.py | 13 +--- .../defaults/project_settings/global.json | 4 +- .../schema_project_syncserver.json | 2 +- .../system_schema/schema_modules.json | 2 +- 6 files changed, 71 insertions(+), 30 deletions(-) diff --git a/pype/modules/sync_server/__init__.py b/pype/modules/sync_server/__init__.py index d7590e24e0..7452b5be1a 100644 --- a/pype/modules/sync_server/__init__.py +++ b/pype/modules/sync_server/__init__.py @@ -1,4 +1,4 @@ -from .sync_server import SyncServer +from pype.modules.sync_server.sync_server import SyncServer def tray_init(tray_widget, main_widget): diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 5f2fbaf2fb..23d1e13bc3 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,8 @@ -from pype.api import get_project_settings, get_system_settings, Logger +from pype.api import ( + get_project_settings, + get_system_settings, + Logger, + get_current_project_settings) import threading import asyncio @@ -92,20 +96,11 @@ class SyncServer(): self.connection = AvalonMongoDB() try: - module_presets = get_system_settings().\ - get("modules").get("Sync Server") + if SyncServer.is_enabled(True): + self.presets = self.get_synced_presets() + self.set_active_sites(self.presets) - if not module_presets: - raise ValueError("No system setting for sync.") - - if not module_presets.get("enabled"): - log.info("Sync server disabled system wide.") - return - - self.presets = self.get_synced_presets() - self.set_active_sites(self.presets) - - self.sync_server_thread = SyncServerThread(self) + self.sync_server_thread = SyncServerThread(self) except ValueError: log.info("No system setting for sync. Not syncing.") except KeyError: @@ -115,6 +110,57 @@ class SyncServer(): "no syncing possible"). format(str(self.presets)), exc_info=True) + @staticmethod + def is_enabled(raise_error=False): + """" + Returns true if synchronization in globally enabled by settings + + raise_error (bool): if missing settings should result in + exception + """ + module_presets = get_system_settings(). \ + get("modules").get("Sync Server") + + if not module_presets: + if raise_error: + raise ValueError("No system setting for sync.") + log.info("No system setting for sync.") + return False + + if not module_presets.get("enabled"): + log.info("Sync server disabled system wide.") + return False + + return True + + @staticmethod + def get_sites_for_project(project_name=None): + """ + Checks if sync is enabled globally and on project. + In that case return local and remote site + + Args: + project_name (str): + + Returns: + (tuple): of strings, labels for (local_site, remote_site) + """ + if SyncServer.is_enabled(False): + if project_name: + settings = get_project_settings(project_name) + else: + settings = get_current_project_settings() + + sync_server_presets = settings["global"]["Sync Server"]["config"] + if settings["global"]["Sync Server"]["enabled"]: + local_site = sync_server_presets.get("active_site", + "studio").strip() + remote_site = sync_server_presets.get("remote_site") + + return local_site, remote_site + + return 'studio', None + def get_synced_presets(self): """ Collects all projects which have enabled syncing and their settings @@ -142,9 +188,9 @@ class SyncServer(): empty if no settings or sync is disabled """ settings = get_project_settings(project_name) - sync_settings = settings.get("global")["sync_server"] + sync_settings = settings.get("global")["Sync Server"] if not sync_settings: - log.info("No project setting for sync_server, not syncing.") + log.info("No project setting for Sync Server, not syncing.") return {} if sync_settings.get("enabled"): return sync_settings diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index d98c68ae80..debfa47aca 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -16,6 +16,7 @@ from avalon.vendor import filelink import pype.api from datetime import datetime from pype.api import config +from pype.modules.sync_server import SyncServer # this is needed until speedcopy for linux is fixed if sys.platform == "win32": @@ -933,15 +934,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): remote_site = None sync_server_presets = None try: - settings = pype.api.get_current_project_settings() - sync_settings = settings["global"]["sync_server"] - - if sync_settings["enabled"]: - sync_server_presets = sync_settings["config"] - local_site = sync_server_presets.get("active_site", - "studio").strip() - remote_site = sync_server_presets.get("remote_site") - except KeyError: + if SyncServer.is_enabled(raise_error=True): + local_site, remote_site = SyncServer.get_sites_for_project() + except ValueError: log.debug(("There are not set presets for SyncServer." " No credentials provided, no synching possible"). format(str(sync_server_presets))) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index ed7dc52cac..44c164362a 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -179,7 +179,7 @@ } } }, - "sync_server": { + "Sync Server": { "enabled": false, "config": { "local_id": "local_0", @@ -196,4 +196,4 @@ } } } -} +} \ No newline at end of file diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index 396e4ca2dc..ebc94d0cfc 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -1,6 +1,6 @@ { "type": "dict", - "key": "sync_server", + "key": "Sync Server", "label": "Sync Server (currently unused)", "collapsable": true, "checkbox_key": "enabled", diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index bcd7bcb8ca..88e3010603 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -100,7 +100,7 @@ ] }, { "type": "dict", - "key": "sync_server", + "key": "Sync Server", "label": "Sync Server", "collapsable": true, "checkbox_key": "enabled", From 589b9afc534a5b342aad469474925ecaac055dbc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 15 Dec 2020 11:49:42 +0100 Subject: [PATCH 16/19] Implemente Module Manager Reworked key value to 'sync_server' according to new configuration --- pype/modules/__init__.py | 4 +- pype/modules/sync_server/sync_server.py | 176 ++++++++++-------- pype/plugins/global/publish/integrate_new.py | 10 +- .../defaults/project_settings/global.json | 2 +- .../schema_project_syncserver.json | 2 +- .../system_schema/schema_modules.json | 2 +- 6 files changed, 111 insertions(+), 85 deletions(-) diff --git a/pype/modules/__init__.py b/pype/modules/__init__.py index 11157f24b1..beae043785 100644 --- a/pype/modules/__init__.py +++ b/pype/modules/__init__.py @@ -34,6 +34,7 @@ from .logging import LoggingModule from .muster import MusterModule from .standalonepublish import StandAlonePublishModule from .websocket_server import WebsocketModule +from .sync_server import SyncServer __all__ = ( @@ -67,5 +68,6 @@ __all__ = ( "MusterModule", "StandAlonePublishModule", - "WebsocketModule" + "WebsocketModule", + "SyncServer" ) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 23d1e13bc3..b6cb9a7623 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -18,8 +18,18 @@ from bson.objectid import ObjectId from avalon.api import AvalonMongoDB from .utils import time_function +from .. import PypeModule, ITrayModule -log = Logger().get_logger("SyncServer") +import six +from pype.lib import PypeLogger +from .. import PypeModule, ITrayService + +if six.PY2: + web = asyncio = STATIC_DIR = WebSocketAsync = None +else: + import asyncio + +log = PypeLogger().get_logger("SyncServer") class SyncStatus(Enum): @@ -28,7 +38,7 @@ class SyncStatus(Enum): DO_DOWNLOAD = 2 -class SyncServer(): +class SyncServer(PypeModule, ITrayService): """ Synchronization server that is syncing published files from local to any of implemented providers (like GDrive, S3 etc.) @@ -85,56 +95,105 @@ class SyncServer(): # different limits imposed by its API # set 0 to no limit REPRESENTATION_LIMIT = 100 + DEFAULT_SITE = 'studio' + + name = "sync_server" + label = "Sync Server" + + def initialize(self, module_settings): + """ + Called during Module Manager creation. + + Collects needed data, checks asyncio presence. + Sets 'enabled' according to global settings for the module. + Shouldnt be doing any initialization, thats a job for 'tray_init' + """ + sync_server_settings = module_settings[self.name] + self.enabled = sync_server_settings["enabled"] + if asyncio is None: + raise AssertionError( + "SyncServer module requires Python 3.5 or higher." + ) + self.lock = None # some parts of code need to run sequentally, not + # in async + self.connection = None # connection to avalon DB to update state + self.presets = None # settings for all enabled projects for sync + self.sync_server_thread = None # asyncio requires new thread + + def connect_with_modules(self, *_a, **kw): + return + + def tray_init(self): + """ + Actual initialization of Sync Server. + + Called when tray is initialized, it checks if module should be + enabled. If not, no initialization necessary. + """ + if not self.enabled: + return - def __init__(self): - self.qaction = None - self.failed_icon = None - self._is_running = False self.presets = None self.lock = threading.Lock() - self.connection = AvalonMongoDB() try: - if SyncServer.is_enabled(True): - self.presets = self.get_synced_presets() - self.set_active_sites(self.presets) + self.presets = self.get_synced_presets() + self.set_active_sites(self.presets) - self.sync_server_thread = SyncServerThread(self) + self.sync_server_thread = SyncServerThread(self) except ValueError: log.info("No system setting for sync. Not syncing.") except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.presets)), exc_info=True) + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) - @staticmethod - def is_enabled(raise_error=False): - """" - Returns true if synchronization in globally enabled by settings - - raise_error (bool): if missing settings should result in - exception + def tray_start(self): """ - module_presets = get_system_settings(). \ - get("modules").get("Sync Server") + Triggered when Tray is started. - if not module_presets: - if raise_error: - raise ValueError("No system setting for sync.") - log.info("No system setting for sync.") - return False + Checks if configuration presets are available and if there is + any provider ('gdrive', 'S3') that is activated + (eg. has valid credentials). - if not module_presets.get("enabled"): - log.info("Sync server disabled system wide.") - return False + Returns: + None + """ + if self.presets and self.active_sites: + self.sync_server_thread.start() + else: + log.info("No presets or active providers. " + + "Synchronization not possible.") - return True + def tray_exit(self): + """ + Stops sync thread if running. - @staticmethod - def get_sites_for_project(project_name=None): + Called from Module Manager + """ + if not self.sync_server_thread: + return + + if not self.is_running: + return + try: + log.info("Stopping sync server server") + self.sync_server_thread.is_running = False + self.sync_server_thread.stop() + except Exception: + log.warning( + "Error has happened during Killing sync server", + exc_info=True + ) + + @property + def is_running(self): + return self.sync_server_thread.is_running + + def get_sites_for_project(self, project_name=None): """ Checks if sync is enabled globally and on project. In that case return local and remote site @@ -145,21 +204,21 @@ class SyncServer(): Returns: (tuple): of strings, labels for (local_site, remote_site) """ - if SyncServer.is_enabled(False): + if self.enabled: if project_name: settings = get_project_settings(project_name) else: settings = get_current_project_settings() - sync_server_presets = settings["global"]["Sync Server"]["config"] - if settings["global"]["Sync Server"]["enabled"]: + sync_server_presets = settings["global"]["sync_server"]["config"] + if settings["global"]["sync_server"]["enabled"]: local_site = sync_server_presets.get("active_site", "studio").strip() remote_site = sync_server_presets.get("remote_site") return local_site, remote_site - return 'studio', None + return self.DEFAULT_SITE, None def get_synced_presets(self): """ @@ -188,7 +247,7 @@ class SyncServer(): empty if no settings or sync is disabled """ settings = get_project_settings(project_name) - sync_settings = settings.get("global")["Sync Server"] + sync_settings = settings.get("global")["sync_server"] if not sync_settings: log.info("No project setting for Sync Server, not syncing.") return {} @@ -495,43 +554,6 @@ class SyncServer(): source_file=source_file, error_str=error_str)) - def tray_start(self): - """ - Triggered when Tray is started. Checks if configuration presets - are available and if there is any provider ('gdrive', 'S3') that - is activated (eg. has valid credentials). - Returns: - None - """ - if self.presets and self.active_sites: - self.sync_server_thread.start() - else: - log.info("No presets or active providers. " + - "Synchronization not possible.") - - def tray_exit(self): - self.stop() - - def thread_stopped(self): - self._is_running = False - - @property - def is_running(self): - return self.sync_server_thread.is_running - - def stop(self): - if not self.is_running: - return - try: - log.info("Stopping sync server server") - self.sync_server_thread.is_running = False - self.sync_server_thread.stop() - except Exception: - log.warning( - "Error has happened during Killing sync server", - exc_info=True - ) - def _get_file_info(self, files, _id): """ Return record from list of records which name matches to 'provider' diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index debfa47aca..26e5fff699 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -15,8 +15,7 @@ from avalon import io from avalon.vendor import filelink import pype.api from datetime import datetime -from pype.api import config -from pype.modules.sync_server import SyncServer +from pype.modules import ModulesManager # this is needed until speedcopy for linux is fixed if sys.platform == "win32": @@ -933,9 +932,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None sync_server_presets = None + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] try: - if SyncServer.is_enabled(raise_error=True): - local_site, remote_site = SyncServer.get_sites_for_project() + if sync_server.enabled: + local_site, remote_site = sync_server.get_sites_for_project() except ValueError: log.debug(("There are not set presets for SyncServer." " No credentials provided, no synching possible"). diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 44c164362a..5913277df3 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -179,7 +179,7 @@ } } }, - "Sync Server": { + "sync_server": { "enabled": false, "config": { "local_id": "local_0", diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index ebc94d0cfc..396e4ca2dc 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -1,6 +1,6 @@ { "type": "dict", - "key": "Sync Server", + "key": "sync_server", "label": "Sync Server (currently unused)", "collapsable": true, "checkbox_key": "enabled", diff --git a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json index 72b7a8e130..74fbac744b 100644 --- a/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json +++ b/pype/tools/settings/settings/gui_schemas/system_schema/schema_modules.json @@ -103,7 +103,7 @@ ] }, { "type": "dict", - "key": "Sync Server", + "key": "sync_server", "label": "Sync Server", "collapsable": true, "checkbox_key": "enabled", From aa94ac132e804033051844fd2d0b371af1339808 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 15 Dec 2020 11:55:09 +0100 Subject: [PATCH 17/19] Hound --- pype/modules/sync_server/sync_server.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index b6cb9a7623..167be665f5 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,11 +1,8 @@ from pype.api import ( get_project_settings, - get_system_settings, - Logger, get_current_project_settings) import threading -import asyncio import concurrent.futures from concurrent.futures._base import CancelledError @@ -18,7 +15,6 @@ from bson.objectid import ObjectId from avalon.api import AvalonMongoDB from .utils import time_function -from .. import PypeModule, ITrayModule import six from pype.lib import PypeLogger @@ -114,8 +110,8 @@ class SyncServer(PypeModule, ITrayService): raise AssertionError( "SyncServer module requires Python 3.5 or higher." ) - self.lock = None # some parts of code need to run sequentally, not - # in async + # some parts of code need to run sequentially, not in async + self.lock = None self.connection = None # connection to avalon DB to update state self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread @@ -146,10 +142,10 @@ class SyncServer(PypeModule, ITrayService): log.info("No system setting for sync. Not syncing.") except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.presets)), exc_info=True) + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.presets)), exc_info=True) def tray_start(self): """ From 5c3d983a0be378e286cb792f3dd27f5f04396750 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 15 Dec 2020 12:56:51 +0100 Subject: [PATCH 18/19] fix settings name in defaults to sync_settings --- pype/settings/defaults/system_settings/modules.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index c714004eb1..0f4b0b37f3 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -117,7 +117,7 @@ "enabled": false, "workspace_name": "studio name" }, - "Sync Server": { + "sync_server": { "enabled": false }, "deadline": { @@ -153,4 +153,4 @@ "idle_manager": { "enabled": true } -} \ No newline at end of file +} From 62ca93fd6e049941ac3fbd53975162725e85e8cc Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Tue, 15 Dec 2020 12:57:11 +0100 Subject: [PATCH 19/19] add QT scaling factor to gloval env --- pype/settings/defaults/system_settings/general.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pype/settings/defaults/system_settings/general.json b/pype/settings/defaults/system_settings/general.json index ff739cc75f..bae0ed4e87 100644 --- a/pype/settings/defaults/system_settings/general.json +++ b/pype/settings/defaults/system_settings/general.json @@ -20,7 +20,8 @@ "PYPE_PROJECT_CONFIGS", "PYPE_PYTHON_EXE", "PYPE_OCIO_CONFIG", - "PYBLISH_GUI" + "PYBLISH_GUI", + "QT_AUTO_SCREEN_SCALE_FACTOR" ] }, "FFMPEG_PATH": { @@ -44,6 +45,7 @@ "darwin": "{VIRTUAL_ENV}/bin/python" }, "PYPE_OCIO_CONFIG": "{STUDIO_SOFT}/OpenColorIO-Configs", - "PYBLISH_GUI": "pyblish_pype" + "PYBLISH_GUI": "pyblish_pype", + "QT_AUTO_SCREEN_SCALE_FACTOR": "1" } } \ No newline at end of file