Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 12:54:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Sync Server - added multiroot support

Browse source
This commit is contained in:
Petr Kalis 2020-10-21 20:24:49 +02:00
parent fad183d1ab
commit 59753ddb1b
3 changed files with 72 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

32
pype/modules/sync_server/README.md Normal file
View file

@ -0,0 +1,32 @@
Synchronization server
---------------------
This server is scheduled at start of Pype, it periodically checks avalon DB
for 'representation' records which have in theirs files.sites record with
name: 'gdrive' without field 'created_dt'.
This denotes that this representation should be sync to GDrive.
Records like these are created by IntegrateNew process based on configuration.
Needed configuration:
--------------------
pype-config/presets/config.json:
"local_id": "local_0", -- identifier of user pype
"retry_cnt": 3, -- how many times try to synch file in case of error
"loop_delay": 60, -- how many seconds between sync loops
"active_site": "studio", -- which site user current, 'studio' by default,
could by same as 'local_id' if user is working
from home without connection to studio
infrastructure
"remote_site": "gdrive" -- key for site to synchronize to (currently only
'gdrive' implemented, but could be any provider
implemented in 'pype/modules/sync_server')
pype-config/presets/gdrive.json:
"credentials_url": "/my_secret_folder/credentials.json",
-- path to credentials for service account
"root": { -- "root": "/My Drive" in simple scenario, this could be for
multiroot projects
"root_one": "/My Drive/work_folder",
"root_tow": "/My Drive/publish_folder"
}

17
pype/modules/sync_server/providers/gdrive.py
View file

@ -141,14 +141,19 @@ class GDriveHandler(AbstractProvider):
self._tree = self._build_tree(self.list_folders())
return self._tree
def get_root_name(self):
def get_roots_config(self):
"""
Return name of root folder. Needs to be used as a beginning of
absolute gdrive path
Returns value from presets of roots. It calculates with multi
roots. Config should be simple key value, or dictionary.
Examples:
"root": "/My Drive"
OR
"root": {"root_ONE": "value", "root_TWO":"value}
Returns:
(string) - plain name, no '/'
(string or dict)
"""
return self.root["name"]
return self.presets["root"]
def create_folder(self, path):
"""
@ -169,7 +174,7 @@ class GDriveHandler(AbstractProvider):
while parts:
folders_to_create.append(parts.pop())
path = '/'.join(parts)
path = path.strip()
folder_id = self.folder_path_exists(path) # lowest common path
if folder_id:
while folders_to_create:

40
pype/modules/sync_server/sync_server.py
View file

@ -1,4 +1,5 @@
from pype.api import config, Logger
from pypeapp.lib.anatomy import Roots
from pype.lib import timeit
import threading
@ -174,12 +175,20 @@ class SyncServer():
recognised by presence of document in 'files.sites', where key is
a provider (GDrive, S3) and value is empty document or document
without 'created_dt' field. (Don't put null to 'created_dt'!).
Querying of 'to-be-synched' files is offloaded to Mongod for
better performance. Goal is to get as few representations as
possible.
Args:
collection (string): name of collection (in most cases matches
project name
active_site (string): identifier of current active site (could be
'local_0' when working from home, 'studio' when working in the
studio (default)
remote_site (string): identifier of remote site I want to sync to
Returns:
(list)
(list) of dictionaries
"""
log.debug("Check representations for : {}".format(collection))
self.connection.Session["AVALON_PROJECT"] = collection
@ -296,7 +305,8 @@ class SyncServer():
# structure should be run in parallel
handler = lib.factory.get_provider(provider_name, tree)
remote_file = self._get_remote_file_path(file,
handler.get_root_name())
handler.get_roots_config()
)
local_root = representation.get("context", {}).get("root")
local_file = self._get_local_file_path(file, local_root)
@ -332,7 +342,8 @@ class SyncServer():
with self.lock:
handler = lib.factory.get_provider(provider_name, tree)
remote_file = self._get_remote_file_path(file,
handler.get_root_name())
handler.get_roots_config()
)
local_root = representation.get("context", {}).get("root")
local_file = self._get_local_file_path(file, local_root)
@ -399,9 +410,8 @@ class SyncServer():
error_str = ''
source_file = file.get("path", "")
log.debug("File {} process {} {}".format(status,
source_file,
error_str))
log.debug("File {source_file} process {status} {error_str}".
format(status, source_file, error_str))
def tray_start(self):
"""
@ -606,20 +616,28 @@ class SyncServer():
"""
if not local_root:
raise ValueError("Unknown local root for file {}")
return file.get("path", "").replace('{root}', local_root)
roots = Roots().default_roots()
path = file.get("path", "")
def _get_remote_file_path(self, file, root_name):
return path.format(**{"root": local_root})
def _get_remote_file_path(self, file, root_config):
"""
Auxiliary function for replacing rootless path with real path
Args:
file (dictionary): file info, get 'path' to file with {root}
root_name (string): value of {root} for remote location
root_config (string or dict): value of {root} for remote location
Returns:
(string) - absolute path on remote location
"""
target_root = '/{}'.format(root_name)
return file.get("path", "").replace('{root}', target_root)
log.debug("root_config::{}".format(root_config))
if isinstance(root_config, str):
root_config = {'root': root_config}
path = file.get("path", "")
path = path.format(**{"root": root_config})
return path
def _get_retries_arr(self):
"""