Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

SyncServer - modified resolving of paths for local and remote

Browse source 
Fix - status for some failed was incorrectly set to Not available
Extended AbstracProvider with new method for resolvments of paths
Added defaults sites to configured sites
Name refactor
This commit is contained in:
Petr Kalis 2021-03-11 16:20:54 +01:00
parent fcfd537830
commit 83570d4998
5 changed files with 134 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

14
pype/modules/sync_server/providers/abstract_provider.py
View file

@ -93,3 +93,17 @@ class AbstractProvider(metaclass=ABCMeta):
only parents and their parents)
"""
pass
@abstractmethod
def resolve_path(self, path, root_config, anatomy=None):
"""
Replaces root placeholders with appropriate real value from
'root_configs' (from Settings or Local Settings) or Anatomy
(mainly for 'studio' site)
Args:
path(string): path with '{root[work]}/...'
root_config(dict): from Settings or Local Settings
anatomy (Anatomy): prepared anatomy object for project
"""
pass

10
pype/modules/sync_server/providers/gdrive.py
View file

@ -678,6 +678,16 @@ class GDriveHandler(AbstractProvider):
return
return provider_presets
def resolve_path(self, path, root_config, anatomy=None):
if not root_config.get("root"):
root_config = {"root": root_config}
try:
return path.format(**root_config)
except KeyError:
msg = "Error in resolving remote root, unknown key"
log.error(msg)
def _handle_q(self, q, trashed=False):
""" API list call contain trashed and hidden files/folder by default.
Usually we dont want those, must be included in query explicitly.

19
pype/modules/sync_server/providers/local_drive.py
View file

@ -85,6 +85,25 @@ class LocalDriveHandler(AbstractProvider):
def get_tree(self):
return
def resolve_path(self, path, root_config, anatomy=None):
if root_config and not root_config.get("root"):
root_config = {"root": root_config}
try:
if not root_config:
raise KeyError
path = path.format(**root_config)
except KeyError:
try:
path = anatomy.fill_root(path)
except KeyError:
msg = "Error in resolving local root from anatomy"
log.error(msg)
raise ValueError(msg)
return path
def _copy(self, source_path, target_path):
print("copying {}->{}".format(source_path, target_path))
shutil.copy(source_path, target_path)

164
pype/modules/sync_server/sync_server.py
View file

@ -111,8 +111,7 @@ class SyncServer(PypeModule, ITrayModule):
Sets 'enabled' according to global settings for the module.
Shouldnt be doing any initialization, thats a job for 'tray_init'
"""
sync_server_settings = module_settings[self.name]
self.enabled = sync_server_settings["enabled"]
self.enabled = module_settings[self.name]["enabled"]
if asyncio is None:
raise AssertionError(
"SyncServer module requires Python 3.5 or higher."
@ -404,6 +403,14 @@ class SyncServer(PypeModule, ITrayModule):
""" End of Public API """
def get_local_file_path(self, collection, file_path):
"""
Externalized for app
"""
local_file_path, _ = self._resolve_paths(file_path, collection)
return local_file_path
def _get_remote_sites_from_settings(self, sync_settings):
if not self.enabled or not sync_settings['enabled']:
return []
@ -529,7 +536,7 @@ class SyncServer(PypeModule, ITrayModule):
For performance
"""
sync_project_presets = {}
sync_project_settings = {}
if not self.connection:
self.connection = AvalonMongoDB()
self.connection.install()
@ -537,12 +544,12 @@ class SyncServer(PypeModule, ITrayModule):
for collection in self.connection.database.collection_names(False):
sync_settings = self.get_sync_project_setting(collection)
if sync_settings:
sync_project_presets[collection] = sync_settings
sync_project_settings[collection] = sync_settings
if not sync_project_presets:
if not sync_project_settings:
log.info("No enabled and configured projects for sync.")
self.sync_project_settings = sync_project_presets
self.sync_project_settings = sync_project_settings
def get_sync_project_settings(self, refresh=False):
"""
@ -767,7 +774,7 @@ class SyncServer(PypeModule, ITrayModule):
return SyncStatus.DO_NOTHING
async def upload(self, collection, file, representation, provider_name,
site_name, tree=None, preset=None):
remote_site_name, tree=None, preset=None):
"""
Upload single 'file' of a 'representation' to 'provider'.
Source url is taken from 'file' portion, where {root} placeholder
@ -797,42 +804,40 @@ class SyncServer(PypeModule, ITrayModule):
# this part modifies structure on 'remote_site', only single
# thread can do that at a time, upload/download to prepared
# structure should be run in parallel
handler = lib.factory.get_provider(provider_name, site_name,
tree=tree, presets=preset)
remote_handler = lib.factory.get_provider(provider_name,
remote_site_name,
tree=tree,
presets=preset)
root_configs = self._get_roots_config(self.sync_project_settings,
collection,
site_name)
remote_file = self._get_remote_file_path(file, root_configs)
file_path = file.get("path", "")
local_file_path, remote_file_path = self._resolve_paths(
file_path, collection, remote_site_name, remote_handler
)
local_file = self.get_local_file_path(collection,
file.get("path", ""))
target_folder = os.path.dirname(remote_file)
folder_id = handler.create_folder(target_folder)
target_folder = os.path.dirname(remote_file_path)
folder_id = remote_handler.create_folder(target_folder)
if not folder_id:
err = "Folder {} wasn't created. Check permissions.".\
format(target_folder)
raise NotADirectoryError(err)
remote_site = self.get_remote_site(collection)
loop = asyncio.get_running_loop()
file_id = await loop.run_in_executor(None,
handler.upload_file,
local_file,
remote_file,
remote_handler.upload_file,
local_file_path,
remote_file_path,
self,
collection,
file,
representation,
remote_site,
remote_site_name,
True
)
return file_id
async def download(self, collection, file, representation, provider_name,
site_name, tree=None, preset=None):
remote_site_name, tree=None, preset=None):
"""
Downloads file to local folder denoted in representation.Context.
@ -850,16 +855,16 @@ class SyncServer(PypeModule, ITrayModule):
(string) - 'name' of local file
"""
with self.lock:
handler = lib.factory.get_provider(provider_name, site_name,
tree=tree, presets=preset)
remote_handler = lib.factory.get_provider(provider_name,
remote_site_name,
tree=tree,
presets=preset)
root_configs = self._get_roots_config(self.sync_project_settings,
collection,
site_name)
remote_file_path = self._get_remote_file_path(file, root_configs)
file_path = file.get("path", "")
local_file_path, remote_file_path = self._resolve_paths(
file_path, collection, remote_site_name, remote_handler
)
local_file_path = self.get_local_file_path(collection,
file.get("path", ""))
local_folder = os.path.dirname(local_file_path)
os.makedirs(local_folder, exist_ok=True)
@ -867,7 +872,7 @@ class SyncServer(PypeModule, ITrayModule):
loop = asyncio.get_running_loop()
file_id = await loop.run_in_executor(None,
handler.download_file,
remote_handler.download_file,
remote_file_path,
local_file_path,
self,
@ -1184,7 +1189,7 @@ class SyncServer(PypeModule, ITrayModule):
Returns:
only logs, catches IndexError and OSError
"""
my_local_site = self.get_my_local_site()
my_local_site = get_local_site_id()
if my_local_site != site_name:
self.log.warning("Cannot remove non local file for {}".
format(site_name))
@ -1206,12 +1211,14 @@ class SyncServer(PypeModule, ITrayModule):
return
representation = representation.pop()
local_file_path = ''
for file in representation.get("files"):
local_file_path, _ = self._resolve_paths(file.get("path", ""),
collection
)
try:
self.log.debug("Removing {}".format(file["path"]))
local_file = self.get_local_file_path(collection,
file.get("path", ""))
os.remove(local_file)
self.log.debug("Removing {}".format(local_file_path))
os.remove(local_file_path)
except IndexError:
msg = "No file set for {}".format(representation_id)
self.log.debug(msg)
@ -1222,22 +1229,13 @@ class SyncServer(PypeModule, ITrayModule):
raise ValueError(msg)
try:
folder = os.path.dirname(local_file)
folder = os.path.dirname(local_file_path)
os.rmdir(folder)
except OSError:
msg = "folder {} cannot be removed".format(folder)
self.log.warning(msg)
raise ValueError(msg)
def get_my_local_site(self):
""" TODO remove
Returns name of current user local_site, its Pype wide.
Returns:
(string)
"""
return get_local_site_id()
def get_loop_delay(self, project_name):
"""
Return count of seconds before next synchronization loop starts
@ -1320,59 +1318,35 @@ class SyncServer(PypeModule, ITrayModule):
val = {"files.$[f].sites.$[s].progress": progress}
return val
def get_local_file_path(self, collection, path):
def _resolve_paths(self, file_path, collection,
remote_site_name=None, remote_handler=None):
"""
Auxiliary function for replacing rootless path with real path
Returns tuple of local and remote file paths with {root}
placeholders replaced with proper values from Settings or Anatomy
Works with multi roots.
If root definition is not found in Settings, anatomy is used
Args:
collection (string): project name
path (dictionary): 'path' to file with {root}
Returns:
(string) - absolute path on local system
Args:
file_path(string): path with {root}
collection(string): project name
remote_site_name(string): remote site
remote_handler(AbstractProvider): implementation
Returns:
(string, string) - proper absolute paths
"""
local_active_site = self.get_active_site(collection)
sites = self.get_sync_project_setting(collection)["sites"]
root_config = sites[local_active_site]["root"]
remote_file_path = ''
if remote_handler:
root_configs = self._get_roots_config(self.sync_project_settings,
collection,
remote_site_name)
if not root_config.get("root"):
root_config = {"root": root_config}
remote_file_path = remote_handler.resolve_path(file_path,
root_configs)
try:
path = path.format(**root_config)
except KeyError:
try:
anatomy = self.get_anatomy(collection)
path = anatomy.fill_root(path)
except KeyError:
msg = "Error in resolving local root from anatomy"
self.log.error(msg)
raise ValueError(msg)
local_handler = lib.factory.get_provider(
'local_drive', self.get_active_site(collection))
local_file_path = local_handler.resolve_path(
file_path, None, self.get_anatomy(collection))
return path
def _get_remote_file_path(self, file, root_config):
"""
Auxiliary function for replacing rootless path with real path
Args:
file (dictionary): file info, get 'path' to file with {root}
root_config (dict): value of {root} for remote location
Returns:
(string) - absolute path on remote location
"""
path = file.get("path", "")
if not root_config.get("root"):
root_config = {"root": root_config}
try:
return path.format(**root_config)
except KeyError:
msg = "Error in resolving remote root, unknown key"
self.log.error(msg)
return local_file_path, remote_file_path
def _get_retries_arr(self, project_name):
"""

40
pype/modules/sync_server/tray/app.py
View file

@ -159,7 +159,8 @@ class SyncProjectListWidget(ProjectListWidget):
model.clear()
project_name = None
for project_name in self.sync_server.get_sync_project_settings().keys():
for project_name in self.sync_server.get_sync_project_settings().\
keys():
if self.sync_server.is_paused() or \
self.sync_server.is_project_paused(project_name):
icon = self._get_icon("paused")
@ -203,7 +204,6 @@ class SyncProjectListWidget(ProjectListWidget):
menu = QtWidgets.QMenu()
actions_mapping = {}
action = None
if self.sync_server.is_project_paused(self.project_name):
action = QtWidgets.QAction("Unpause")
actions_mapping[action] = self._unpause
@ -212,7 +212,7 @@ class SyncProjectListWidget(ProjectListWidget):
actions_mapping[action] = self._pause
menu.addAction(action)
if self.local_site == self.sync_server.get_my_local_site():
if self.local_site == get_local_site_id():
action = QtWidgets.QAction("Clear local project")
actions_mapping[action] = self._clear_project
menu.addAction(action)
@ -241,6 +241,7 @@ class SyncProjectListWidget(ProjectListWidget):
self.project_name = None
self.refresh()
class ProjectModel(QtCore.QAbstractListModel):
def __init__(self, *args, projects=None, **kwargs):
super(ProjectModel, self).__init__(*args, **kwargs)
@ -256,6 +257,7 @@ class ProjectModel(QtCore.QAbstractListModel):
def rowCount(self, index):
return len(self.todos)
class SyncRepresentationWidget(QtWidgets.QWidget):
"""
Summary dialog with list of representations that matches current
@ -478,7 +480,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
local_site_name = self.sync_server.get_my_local_site()
try:
self.sync_server.add_site(
self.table_view.model()._project,
project_name,
self.representation_id,
local_site_name
)
@ -802,7 +804,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self._data.append(item)
self._rec_loaded += 1
def canFetchMore(self, index):
"""
Check if there are more records than currently loaded
@ -854,6 +855,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1}
self.query = self.get_default_query()
# import json
# log.debug(json.dumps(self.query, indent=4).replace('False', 'false').\
# replace('True', 'true').replace('None', 'null'))
representations = self.dbcon.aggregate(self.query)
self.refresh(representations)
@ -891,7 +895,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
Returns:
(QModelIndex)
"""
index = None
for i in range(self.rowCount(None)):
index = self.index(i, 0)
value = self.data(index, Qt.UserRole)
@ -1000,7 +1003,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
0]},
'failed_remote_tries': {
'$cond': [{'$size': '$order_remote.tries'},
{'$first': '$order_local.tries'},
{'$first': '$order_remote.tries'},
0]},
'paused_remote': {
'$cond': [{'$size': "$order_remote.paused"},
@ -1027,9 +1030,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
# select last touch of file
'updated_dt_remote': {'$max': "$updated_dt_remote"},
'failed_remote': {'$sum': '$failed_remote'},
'failed_local': {'$sum': '$paused_remote'},
'failed_local_tries': {'$sum': '$failed_local_tries'},
'failed_local': {'$sum': '$failed_local'},
'failed_remote_tries': {'$sum': '$failed_remote_tries'},
'failed_local_tries': {'$sum': '$failed_local_tries'},
'paused_remote': {'$sum': '$paused_remote'},
'paused_local': {'$sum': '$paused_local'},
'updated_dt_local': {'$max': "$updated_dt_local"}
@ -1669,7 +1672,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
Returns:
(QModelIndex)
"""
index = None
for i in range(self.rowCount(None)):
index = self.index(i, 0)
value = self.data(index, Qt.UserRole)
@ -1777,14 +1779,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
"$order_local.error",
[""]]}},
'tries': {'$first': {
'$cond': [{'$size': "$order_local.tries"},
"$order_local.tries",
{'$cond': [
{'$size': "$order_remote.tries"},
"$order_remote.tries",
[]
]}
]}}
'$cond': [
{'$size': "$order_local.tries"},
"$order_local.tries",
{'$cond': [
{'$size': "$order_remote.tries"},
"$order_remote.tries",
[]
]}
]}}
}},
{"$project": self.projection},
{"$sort": self.sort},
@ -2015,6 +2018,7 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate):
value /= 1024.0
return "%.1f%s%s" % (value, 'Yi', suffix)
def _convert_progress(value):
try:
progress = float(value)