From 59753ddb1b551bb6bd6707ff624d69dfbdfb19e7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 21 Oct 2020 20:24:49 +0200 Subject: [PATCH] Sync Server - added multiroot support --- pype/modules/sync_server/README.md | 32 ++++++++++++++++ pype/modules/sync_server/providers/gdrive.py | 17 ++++++--- pype/modules/sync_server/sync_server.py | 40 ++++++++++++++------ 3 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 pype/modules/sync_server/README.md diff --git a/pype/modules/sync_server/README.md b/pype/modules/sync_server/README.md new file mode 100644 index 0000000000..4ee8310085 --- /dev/null +++ b/pype/modules/sync_server/README.md @@ -0,0 +1,32 @@ +Synchronization server +--------------------- +This server is scheduled at start of Pype, it periodically checks avalon DB +for 'representation' records which have in theirs files.sites record with +name: 'gdrive' without field 'created_dt'. +This denotes that this representation should be sync to GDrive. +Records like these are created by IntegrateNew process based on configuration. + +Needed configuration: +-------------------- +pype-config/presets/config.json: + "local_id": "local_0", -- identifier of user pype + "retry_cnt": 3, -- how many times try to synch file in case of error + "loop_delay": 60, -- how many seconds between sync loops + "active_site": "studio", -- which site user current, 'studio' by default, + could by same as 'local_id' if user is working + from home without connection to studio + infrastructure + "remote_site": "gdrive" -- key for site to synchronize to (currently only + 'gdrive' implemented, but could be any provider + implemented in 'pype/modules/sync_server') +pype-config/presets/gdrive.json: + "credentials_url": "/my_secret_folder/credentials.json", + -- path to credentials for service account + "root": { -- "root": "/My Drive" in simple scenario, this could be for + multiroot projects + "root_one": "/My Drive/work_folder", + "root_tow": "/My Drive/publish_folder" + } + + + diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 4d2958b3bd..de33246777 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -141,14 +141,19 @@ class GDriveHandler(AbstractProvider): self._tree = self._build_tree(self.list_folders()) return self._tree - def get_root_name(self): + def get_roots_config(self): """ - Return name of root folder. Needs to be used as a beginning of - absolute gdrive path + Returns value from presets of roots. It calculates with multi + roots. Config should be simple key value, or dictionary. + + Examples: + "root": "/My Drive" + OR + "root": {"root_ONE": "value", "root_TWO":"value} Returns: - (string) - plain name, no '/' + (string or dict) """ - return self.root["name"] + return self.presets["root"] def create_folder(self, path): """ @@ -169,7 +174,7 @@ class GDriveHandler(AbstractProvider): while parts: folders_to_create.append(parts.pop()) path = '/'.join(parts) - + path = path.strip() folder_id = self.folder_path_exists(path) # lowest common path if folder_id: while folders_to_create: diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index c023bfb032..fe3ef5c774 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,5 @@ from pype.api import config, Logger +from pypeapp.lib.anatomy import Roots from pype.lib import timeit import threading @@ -174,12 +175,20 @@ class SyncServer(): recognised by presence of document in 'files.sites', where key is a provider (GDrive, S3) and value is empty document or document without 'created_dt' field. (Don't put null to 'created_dt'!). + Querying of 'to-be-synched' files is offloaded to Mongod for better performance. Goal is to get as few representations as possible. + Args: + collection (string): name of collection (in most cases matches + project name + active_site (string): identifier of current active site (could be + 'local_0' when working from home, 'studio' when working in the + studio (default) + remote_site (string): identifier of remote site I want to sync to Returns: - (list) + (list) of dictionaries """ log.debug("Check representations for : {}".format(collection)) self.connection.Session["AVALON_PROJECT"] = collection @@ -296,7 +305,8 @@ class SyncServer(): # structure should be run in parallel handler = lib.factory.get_provider(provider_name, tree) remote_file = self._get_remote_file_path(file, - handler.get_root_name()) + handler.get_roots_config() + ) local_root = representation.get("context", {}).get("root") local_file = self._get_local_file_path(file, local_root) @@ -332,7 +342,8 @@ class SyncServer(): with self.lock: handler = lib.factory.get_provider(provider_name, tree) remote_file = self._get_remote_file_path(file, - handler.get_root_name()) + handler.get_roots_config() + ) local_root = representation.get("context", {}).get("root") local_file = self._get_local_file_path(file, local_root) @@ -399,9 +410,8 @@ class SyncServer(): error_str = '' source_file = file.get("path", "") - log.debug("File {} process {} {}".format(status, - source_file, - error_str)) + log.debug("File {source_file} process {status} {error_str}". + format(status, source_file, error_str)) def tray_start(self): """ @@ -606,20 +616,28 @@ class SyncServer(): """ if not local_root: raise ValueError("Unknown local root for file {}") - return file.get("path", "").replace('{root}', local_root) + roots = Roots().default_roots() + path = file.get("path", "") - def _get_remote_file_path(self, file, root_name): + return path.format(**{"root": local_root}) + + def _get_remote_file_path(self, file, root_config): """ Auxiliary function for replacing rootless path with real path Args: file (dictionary): file info, get 'path' to file with {root} - root_name (string): value of {root} for remote location + root_config (string or dict): value of {root} for remote location Returns: (string) - absolute path on remote location """ - target_root = '/{}'.format(root_name) - return file.get("path", "").replace('{root}', target_root) + log.debug("root_config::{}".format(root_config)) + if isinstance(root_config, str): + root_config = {'root': root_config} + + path = file.get("path", "") + path = path.format(**{"root": root_config}) + return path def _get_retries_arr(self): """