Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2026-01-01 08:24:53 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

SyncServer adding functionality to Loader

Browse source 
In one big commit as PR wasnt merged before rebranding and merge exploded
This commit is contained in:
Petr Kalis 2021-04-06 12:27:49 +02:00
parent f2ac34fe2e
commit 399f9bd059
14 changed files with 1986 additions and 1674 deletions
Download patch file Download diff file Expand all files Collapse all files

4
openpype/modules/__init__.py
View file

@ -41,7 +41,7 @@ from .log_viewer import LogViewModule
from .muster import MusterModule
from .deadline import DeadlineModule
from .standalonepublish_action import StandAlonePublishAction
from .sync_server import SyncServer
from .sync_server import SyncServerModule
__all__ = (
@ -82,5 +82,5 @@ __all__ = (
"DeadlineModule",
"StandAlonePublishAction",
"SyncServer"
"SyncServerModule"
)

4
openpype/modules/sync_server/__init__.py
View file

@ -1,5 +1,5 @@
from openpype.modules.sync_server.sync_server import SyncServer
from openpype.modules.sync_server.sync_server_module import SyncServerModule
def tray_init(tray_widget, main_widget):
return SyncServer()
return SyncServerModule()

0
openpype/modules/sync_server/providers/__init__.py Normal file
View file

109
openpype/modules/sync_server/providers/abstract_provider.py
View file

@ -1,16 +1,22 @@
from abc import ABCMeta, abstractmethod
import abc, six
from openpype.api import Anatomy, Logger
log = Logger().get_logger("SyncServer")
class AbstractProvider(metaclass=ABCMeta):
@six.add_metaclass(abc.ABCMeta)
class AbstractProvider:
def __init__(self, site_name, tree=None, presets=None):
def __init__(self, project_name, site_name, tree=None, presets=None):
self.presets = None
self.active = False
self.site_name = site_name
self.presets = presets
@abstractmethod
super(AbstractProvider, self).__init__()
@abc.abstractmethod
def is_active(self):
"""
Returns True if provider is activated, eg. has working credentials.
@ -18,36 +24,54 @@ class AbstractProvider(metaclass=ABCMeta):
(boolean)
"""
@abstractmethod
def upload_file(self, source_path, target_path, overwrite=True):
@abc.abstractmethod
def upload_file(self, source_path, path,
server, collection, file, representation, site,
overwrite=False):
"""
Copy file from 'source_path' to 'target_path' on provider.
Use 'overwrite' boolean to rewrite existing file on provider
Args:
source_path (string): absolute path on local system
target_path (string): absolute path on provider (GDrive etc.)
overwrite (boolean): True if overwite existing
source_path (string):
path (string): absolute path with or without name of the file
overwrite (boolean): replace existing file
arguments for saving progress:
server (SyncServer): server instance to call update_db on
collection (str): name of collection
file (dict): info about uploaded file (matches structure from db)
representation (dict): complete repre containing 'file'
site (str): site name
Returns:
(string) file_id of created file, raises exception
"""
pass
@abstractmethod
def download_file(self, source_path, local_path, overwrite=True):
@abc.abstractmethod
def download_file(self, source_path, local_path,
server, collection, file, representation, site,
overwrite=False):
"""
Download file from provider into local system
Args:
source_path (string): absolute path on provider
local_path (string): absolute path on local
overwrite (bool): default set to True
local_path (string): absolute path with or without name of the file
overwrite (boolean): replace existing file
arguments for saving progress:
server (SyncServer): server instance to call update_db on
collection (str): name of collection
file (dict): info about uploaded file (matches structure from db)
representation (dict): complete repre containing 'file'
site (str): site name
Returns:
None
"""
pass
@abstractmethod
@abc.abstractmethod
def delete_file(self, path):
"""
Deletes file from 'path'. Expects path to specific file.
@ -60,7 +84,7 @@ class AbstractProvider(metaclass=ABCMeta):
"""
pass
@abstractmethod
@abc.abstractmethod
def list_folder(self, folder_path):
"""
List all files and subfolders of particular path non-recursively.
@ -72,7 +96,7 @@ class AbstractProvider(metaclass=ABCMeta):
"""
pass
@abstractmethod
@abc.abstractmethod
def create_folder(self, folder_path):
"""
Create all nonexistent folders and subfolders in 'path'.
@ -85,7 +109,7 @@ class AbstractProvider(metaclass=ABCMeta):
"""
pass
@abstractmethod
@abc.abstractmethod
def get_tree(self):
"""
Creates folder structure for providers which do not provide
@ -94,16 +118,49 @@ class AbstractProvider(metaclass=ABCMeta):
"""
pass
@abstractmethod
def resolve_path(self, path, root_config, anatomy=None):
@abc.abstractmethod
def get_roots_config(self, anatomy=None):
"""
Replaces root placeholders with appropriate real value from
'root_configs' (from Settings or Local Settings) or Anatomy
(mainly for 'studio' site)
Returns root values for path resolving
Args:
path(string): path with '{root[work]}/...'
root_config(dict): from Settings or Local Settings
anatomy (Anatomy): prepared anatomy object for project
Takes value from Anatomy which takes values from Settings
overridden by Local Settings
Returns:
(dict) - {"root": {"root": "/My Drive"}}
OR
{"root": {"root_ONE": "value", "root_TWO":"value}}
Format is importing for usage of python's format ** approach
"""
pass
def resolve_path(self, path, root_config=None, anatomy=None):
"""
Replaces all root placeholders with proper values
Args:
path(string): root[work]/folder...
root_config (dict): {'work': "c:/..."...}
anatomy (Anatomy): object of Anatomy
Returns:
(string): proper url
"""
if root_config and not root_config.get("root"):
root_config = {"root": root_config}
else:
root_config = self.get_roots_config(anatomy)
try:
if not root_config:
raise KeyError
path = path.format(**root_config)
except KeyError:
try:
path = anatomy.fill_root(path)
except KeyError:
msg = "Error in resolving local root from anatomy"
log.error(msg)
raise ValueError(msg)
return path

336
openpype/modules/sync_server/providers/gdrive.py
View file

@ -6,10 +6,11 @@ from googleapiclient import errors
from .abstract_provider import AbstractProvider
from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
from openpype.api import Logger
from openpype.api import get_system_settings
from openpype.api import get_system_settings, Anatomy
from ..utils import time_function
import time
SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly',
'https://www.googleapis.com/auth/drive.file',
'https://www.googleapis.com/auth/drive.readonly'] # for write|delete
@ -45,9 +46,10 @@ class GDriveHandler(AbstractProvider):
MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive
CHUNK_SIZE = 2097152 # must be divisible by 256!
def __init__(self, site_name, tree=None, presets=None):
def __init__(self, project_name, site_name, tree=None, presets=None):
self.presets = None
self.active = False
self.project_name = project_name
self.site_name = site_name
self.presets = presets
@ -65,137 +67,6 @@ class GDriveHandler(AbstractProvider):
self._tree = tree
self.active = True
def _get_gd_service(self):
"""
Authorize client with 'credentials.json', uses service account.
Service account needs to have target folder shared with.
Produces service that communicates with GDrive API.
Returns:
None
"""
creds = service_account.Credentials.from_service_account_file(
self.presets["credentials_url"],
scopes=SCOPES)
service = build('drive', 'v3',
credentials=creds, cache_discovery=False)
return service
def _prepare_root_info(self):
"""
Prepare info about roots and theirs folder ids from 'presets'.
Configuration might be for single or multiroot projects.
Regular My Drive and Shared drives are implemented, their root
folder ids need to be queried in slightly different way.
Returns:
(dicts) of dicts where root folders are keys
"""
roots = {}
for path in self.get_roots_config().values():
if self.MY_DRIVE_STR in path:
roots[self.MY_DRIVE_STR] = self.service.files()\
.get(fileId='root').execute()
else:
shared_drives = []
page_token = None
while True:
response = self.service.drives().list(
pageSize=100,
pageToken=page_token).execute()
shared_drives.extend(response.get('drives', []))
page_token = response.get('nextPageToken', None)
if page_token is None:
break
folders = path.split('/')
if len(folders) < 2:
raise ValueError("Wrong root folder definition {}".
format(path))
for shared_drive in shared_drives:
if folders[1] in shared_drive["name"]:
roots[shared_drive["name"]] = {
"name": shared_drive["name"],
"id": shared_drive["id"]}
if self.MY_DRIVE_STR not in roots: # add My Drive always
roots[self.MY_DRIVE_STR] = self.service.files() \
.get(fileId='root').execute()
return roots
@time_function
def _build_tree(self, folders):
"""
Create in-memory structure resolving paths to folder id as
recursive querying might be slower.
Initialized in the time of class initialization.
Maybe should be persisted
Tree is structure of path to id:
'/ROOT': {'id': '1234567'}
'/ROOT/PROJECT_FOLDER': {'id':'222222'}
'/ROOT/PROJECT_FOLDER/Assets': {'id': '3434545'}
Args:
folders (list): list of dictionaries with folder metadata
Returns:
(dictionary) path as a key, folder id as a value
"""
log.debug("build_tree len {}".format(len(folders)))
root_ids = []
default_root_id = None
tree = {}
ending_by = {}
for root_name, root in self.root.items(): # might be multiple roots
if root["id"] not in root_ids:
tree["/" + root_name] = {"id": root["id"]}
ending_by[root["id"]] = "/" + root_name
root_ids.append(root["id"])
if self.MY_DRIVE_STR == root_name:
default_root_id = root["id"]
no_parents_yet = {}
while folders:
folder = folders.pop(0)
parents = folder.get("parents", [])
# weird cases, shared folders, etc, parent under root
if not parents:
parent = default_root_id
else:
parent = parents[0]
if folder["id"] in root_ids: # do not process root
continue
if parent in ending_by:
path_key = ending_by[parent] + "/" + folder["name"]
ending_by[folder["id"]] = path_key
tree[path_key] = {"id": folder["id"]}
else:
no_parents_yet.setdefault(parent, []).append((folder["id"],
folder["name"]))
loop_cnt = 0
# break if looped more then X times - safety against infinite loop
while no_parents_yet and loop_cnt < 20:
keys = list(no_parents_yet.keys())
for parent in keys:
if parent in ending_by.keys():
subfolders = no_parents_yet.pop(parent)
for folder_id, folder_name in subfolders:
path_key = ending_by[parent] + "/" + folder_name
ending_by[folder_id] = path_key
tree[path_key] = {"id": folder_id}
loop_cnt += 1
if len(no_parents_yet) > 0:
log.debug("Some folders path are not resolved {}".
format(no_parents_yet))
log.debug("Remove deleted folders from trash.")
return tree
def is_active(self):
"""
Returns True if provider is activated, eg. has working credentials.
@ -204,6 +75,21 @@ class GDriveHandler(AbstractProvider):
"""
return self.active
def get_roots_config(self, anatomy=None):
"""
Returns root values for path resolving
Use only Settings as GDrive cannot be modified by Local Settings
Returns:
(dict) - {"root": {"root": "/My Drive"}}
OR
{"root": {"root_ONE": "value", "root_TWO":"value}}
Format is importing for usage of python's format ** approach
"""
# GDrive roots cannot be locally overridden
return self.presets['root']
def get_tree(self):
"""
Building of the folder tree could be potentially expensive,
@ -217,26 +103,6 @@ class GDriveHandler(AbstractProvider):
self._tree = self._build_tree(self.list_folders())
return self._tree
def get_roots_config(self):
"""
Returns value from presets of roots. It calculates with multi
roots. Config should be simple key value, or dictionary.
Examples:
"root": "/My Drive"
OR
"root": {"root_ONE": "value", "root_TWO":"value}
Returns:
(dict) - {"root": {"root": "/My Drive"}}
OR
{"root": {"root_ONE": "value", "root_TWO":"value}}
Format is importing for usage of python's format ** approach
"""
roots = self.presets["root"]
if isinstance(roots, str):
roots = {"root": roots}
return roots
def create_folder(self, path):
"""
Create all nonexistent folders and subfolders in 'path'.
@ -510,20 +376,6 @@ class GDriveHandler(AbstractProvider):
self.service.files().delete(fileId=file["id"],
supportsAllDrives=True).execute()
def _get_folder_metadata(self, path):
"""
Get info about folder with 'path'
Args:
path (string):
Returns:
(dictionary) with metadata or raises ValueError
"""
try:
return self.get_tree()[path]
except Exception:
raise ValueError("Uknown folder id {}".format(id))
def list_folder(self, folder_path):
"""
List all files and subfolders of particular path non-recursively.
@ -678,15 +530,151 @@ class GDriveHandler(AbstractProvider):
return
return provider_presets
def resolve_path(self, path, root_config, anatomy=None):
if not root_config.get("root"):
root_config = {"root": root_config}
def _get_gd_service(self):
"""
Authorize client with 'credentials.json', uses service account.
Service account needs to have target folder shared with.
Produces service that communicates with GDrive API.
Returns:
None
"""
creds = service_account.Credentials.from_service_account_file(
self.presets["credentials_url"],
scopes=SCOPES)
service = build('drive', 'v3',
credentials=creds, cache_discovery=False)
return service
def _prepare_root_info(self):
"""
Prepare info about roots and theirs folder ids from 'presets'.
Configuration might be for single or multiroot projects.
Regular My Drive and Shared drives are implemented, their root
folder ids need to be queried in slightly different way.
Returns:
(dicts) of dicts where root folders are keys
"""
roots = {}
config_roots = self.get_roots_config()
for path in config_roots.values():
if self.MY_DRIVE_STR in path:
roots[self.MY_DRIVE_STR] = self.service.files()\
.get(fileId='root').execute()
else:
shared_drives = []
page_token = None
while True:
response = self.service.drives().list(
pageSize=100,
pageToken=page_token).execute()
shared_drives.extend(response.get('drives', []))
page_token = response.get('nextPageToken', None)
if page_token is None:
break
folders = path.split('/')
if len(folders) < 2:
raise ValueError("Wrong root folder definition {}".
format(path))
for shared_drive in shared_drives:
if folders[1] in shared_drive["name"]:
roots[shared_drive["name"]] = {
"name": shared_drive["name"],
"id": shared_drive["id"]}
if self.MY_DRIVE_STR not in roots: # add My Drive always
roots[self.MY_DRIVE_STR] = self.service.files() \
.get(fileId='root').execute()
return roots
@time_function
def _build_tree(self, folders):
"""
Create in-memory structure resolving paths to folder id as
recursive querying might be slower.
Initialized in the time of class initialization.
Maybe should be persisted
Tree is structure of path to id:
'/ROOT': {'id': '1234567'}
'/ROOT/PROJECT_FOLDER': {'id':'222222'}
'/ROOT/PROJECT_FOLDER/Assets': {'id': '3434545'}
Args:
folders (list): list of dictionaries with folder metadata
Returns:
(dictionary) path as a key, folder id as a value
"""
log.debug("build_tree len {}".format(len(folders)))
root_ids = []
default_root_id = None
tree = {}
ending_by = {}
for root_name, root in self.root.items(): # might be multiple roots
if root["id"] not in root_ids:
tree["/" + root_name] = {"id": root["id"]}
ending_by[root["id"]] = "/" + root_name
root_ids.append(root["id"])
if self.MY_DRIVE_STR == root_name:
default_root_id = root["id"]
no_parents_yet = {}
while folders:
folder = folders.pop(0)
parents = folder.get("parents", [])
# weird cases, shared folders, etc, parent under root
if not parents:
parent = default_root_id
else:
parent = parents[0]
if folder["id"] in root_ids: # do not process root
continue
if parent in ending_by:
path_key = ending_by[parent] + "/" + folder["name"]
ending_by[folder["id"]] = path_key
tree[path_key] = {"id": folder["id"]}
else:
no_parents_yet.setdefault(parent, []).append((folder["id"],
folder["name"]))
loop_cnt = 0
# break if looped more then X times - safety against infinite loop
while no_parents_yet and loop_cnt < 20:
keys = list(no_parents_yet.keys())
for parent in keys:
if parent in ending_by.keys():
subfolders = no_parents_yet.pop(parent)
for folder_id, folder_name in subfolders:
path_key = ending_by[parent] + "/" + folder_name
ending_by[folder_id] = path_key
tree[path_key] = {"id": folder_id}
loop_cnt += 1
if len(no_parents_yet) > 0:
log.debug("Some folders path are not resolved {}".
format(no_parents_yet))
log.debug("Remove deleted folders from trash.")
return tree
def _get_folder_metadata(self, path):
"""
Get info about folder with 'path'
Args:
path (string):
Returns:
(dictionary) with metadata or raises ValueError
"""
try:
return path.format(**root_config)
except KeyError:
msg = "Error in resolving remote root, unknown key"
log.error(msg)
return self.get_tree()[path]
except Exception:
raise ValueError("Uknown folder id {}".format(id))
def _handle_q(self, q, trashed=False):
""" API list call contain trashed and hidden files/folder by default.

8
openpype/modules/sync_server/providers/lib.py
View file

@ -1,4 +1,3 @@
from enum import Enum
from .gdrive import GDriveHandler
from .local_drive import LocalDriveHandler
@ -25,7 +24,8 @@ class ProviderFactory:
"""
self.providers[provider] = (creator, batch_limit)
def get_provider(self, provider, site_name, tree=None, presets=None):
def get_provider(self, provider, project_name, site_name,
tree=None, presets=None):
"""
Returns new instance of provider client for specific site.
One provider could have multiple sites.
@ -37,6 +37,7 @@ class ProviderFactory:
provider (string): 'gdrive','S3'
site_name (string): descriptor of site, different service accounts
must have different site name
project_name (string): different projects could have diff. sites
tree (dictionary): - folder paths to folder id structure
presets (dictionary): config for provider and site (eg.
"credentials_url"..)
@ -44,7 +45,8 @@ class ProviderFactory:
(implementation of AbstractProvider)
"""
creator_info = self._get_creator_info(provider)
site = creator_info[0](site_name, tree, presets) # call init
# call init
site = creator_info[0](project_name, site_name, tree, presets)
return site

60
openpype/modules/sync_server/providers/local_drive.py
View file

@ -4,7 +4,7 @@ import shutil
import threading
import time
from openpype.api import Logger
from openpype.api import Logger, Anatomy
from .abstract_provider import AbstractProvider
log = Logger().get_logger("SyncServer")
@ -12,6 +12,14 @@ log = Logger().get_logger("SyncServer")
class LocalDriveHandler(AbstractProvider):
""" Handles required operations on mounted disks with OS """
def __init__(self, project_name, site_name, tree=None, presets=None):
self.presets = None
self.active = False
self.project_name = project_name
self.site_name = site_name
self.active = self.is_active()
def is_active(self):
return True
@ -82,27 +90,37 @@ class LocalDriveHandler(AbstractProvider):
os.makedirs(folder_path, exist_ok=True)
return folder_path
def get_roots_config(self, anatomy=None):
"""
Returns root values for path resolving
Takes value from Anatomy which takes values from Settings
overridden by Local Settings
Returns:
(dict) - {"root": {"root": "/My Drive"}}
OR
{"root": {"root_ONE": "value", "root_TWO":"value}}
Format is importing for usage of python's format ** approach
"""
if not anatomy:
anatomy = Anatomy(self.project_name,
self._normalize_site_name(self.site_name))
return {'root': anatomy.roots}
def get_tree(self):
return
def resolve_path(self, path, root_config, anatomy=None):
if root_config and not root_config.get("root"):
root_config = {"root": root_config}
def get_configurable_items_for_site(self):
"""
Returns list of items that should be configurable by User
try:
if not root_config:
raise KeyError
path = path.format(**root_config)
except KeyError:
try:
path = anatomy.fill_root(path)
except KeyError:
msg = "Error in resolving local root from anatomy"
log.error(msg)
raise ValueError(msg)
return path
Returns:
(list of dict)
[{key:"root", label:"root", value:"valueFromSettings"}]
"""
pass
def _copy(self, source_path, target_path):
print("copying {}->{}".format(source_path, target_path))
@ -133,3 +151,9 @@ class LocalDriveHandler(AbstractProvider):
)
target_file_size = os.path.getsize(target_path)
time.sleep(0.5)
def _normalize_site_name(self, site_name):
"""Transform user id to 'local' for Local settings"""
if site_name != 'studio':
return 'local'
return site_name

1567
openpype/modules/sync_server/sync_server.py
View file

File diff suppressed because it is too large Load diff

1194
openpype/modules/sync_server/sync_server_module.py Normal file
View file

File diff suppressed because it is too large Load diff

298
openpype/modules/sync_server/tray/app.py
View file

@ -159,7 +159,7 @@ class SyncProjectListWidget(ProjectListWidget):
model.clear()
project_name = None
for project_name in self.sync_server.get_sync_project_settings().\
for project_name in self.sync_server.sync_project_settings.\
keys():
if self.sync_server.is_paused() or \
self.sync_server.is_project_paused(project_name):
@ -169,7 +169,7 @@ class SyncProjectListWidget(ProjectListWidget):
model.appendRow(QtGui.QStandardItem(icon, project_name))
if len(self.sync_server.get_sync_project_settings().keys()) == 0:
if len(self.sync_server.sync_project_settings.keys()) == 0:
model.appendRow(QtGui.QStandardItem(DUMMY_PROJECT))
self.current_project = self.project_list.currentIndex().data(
@ -271,15 +271,29 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
("subset", 190),
("version", 10),
("representation", 90),
("created_dt", 100),
("sync_dt", 100),
("local_site", 60),
("remote_site", 70),
("files_count", 70),
("files_size", 70),
("created_dt", 105),
("sync_dt", 105),
("local_site", 80),
("remote_site", 80),
("files_count", 50),
("files_size", 60),
("priority", 20),
("state", 50)
)
column_labels = (
("asset", "Asset"),
("subset", "Subset"),
("version", "Version"),
("representation", "Representation"),
("created_dt", "Created"),
("sync_dt", "Synced"),
("local_site", "Active site"),
("remote_site", "Remote site"),
("files_count", "Files"),
("files_size", "Size"),
("priority", "Priority"),
("state", "Status")
)
def __init__(self, sync_server, project=None, parent=None):
super(SyncRepresentationWidget, self).__init__(parent)
@ -298,8 +312,10 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
self.table_view = QtWidgets.QTableView()
headers = [item[0] for item in self.default_widths]
header_labels = [item[1] for item in self.column_labels]
model = SyncRepresentationModel(sync_server, headers, project)
model = SyncRepresentationModel(sync_server, headers,
project, header_labels)
self.table_view.setModel(model)
self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self.table_view.setSelectionMode(
@ -376,7 +392,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
"""
_id = self.table_view.model().data(index, Qt.UserRole)
detail_window = SyncServerDetailWindow(
self.sync_server, _id, self.table_view.model()._project)
self.sync_server, _id, self.table_view.model().project)
detail_window.exec()
def _on_context_menu(self, point):
@ -394,15 +410,28 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
menu = QtWidgets.QMenu()
actions_mapping = {}
actions_kwargs_mapping = {}
action = QtWidgets.QAction("Open in explorer")
actions_mapping[action] = self._open_in_explorer
menu.addAction(action)
local_site = self.item.local_site
local_progress = self.item.local_progress
remote_site = self.item.remote_site
remote_progress = self.item.remote_progress
local_site, local_progress = self.item.local_site.split()
remote_site, remote_progress = self.item.remote_site.split()
local_progress = float(local_progress)
remote_progress = float(remote_progress)
for site, progress in {local_site: local_progress,
remote_site: remote_progress}.items():
project = self.table_view.model().project
provider = self.sync_server.get_provider_for_site(project,
site)
if provider == 'local_drive':
if 'studio' in site:
txt = " studio version"
else:
txt = " local version"
action = QtWidgets.QAction("Open in explorer" + txt)
if progress == 1.0:
actions_mapping[action] = self._open_in_explorer
actions_kwargs_mapping[action] = {'site': site}
menu.addAction(action)
# progress smaller then 1.0 --> in progress or queued
if local_progress < 1.0:
@ -452,13 +481,14 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
result = menu.exec_(QtGui.QCursor.pos())
if result:
to_run = actions_mapping[result]
to_run_kwargs = actions_kwargs_mapping.get(result, {})
if to_run:
to_run()
to_run(**to_run_kwargs)
self.table_view.model().refresh()
def _pause(self):
self.sync_server.pause_representation(self.table_view.model()._project,
self.sync_server.pause_representation(self.table_view.model().project,
self.representation_id,
self.site_name)
self.site_name = None
@ -466,7 +496,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
def _unpause(self):
self.sync_server.unpause_representation(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
self.site_name)
self.site_name = None
@ -476,7 +506,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
# temporary here for testing, will be removed TODO
def _add_site(self):
log.info(self.representation_id)
project_name = self.table_view.model()._project
project_name = self.table_view.model().project
local_site_name = self.sync_server.get_my_local_site()
try:
self.sync_server.add_site(
@ -504,7 +534,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
try:
local_site = get_local_site_id()
self.sync_server.remove_site(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
local_site,
True
@ -519,7 +549,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
'local'
)
@ -530,18 +560,20 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
'remote'
)
def _open_in_explorer(self):
def _open_in_explorer(self, site):
if not self.item:
return
fpath = self.item.path
project = self.table_view.model()._project
fpath = self.sync_server.get_local_file_path(project, fpath)
project = self.table_view.model().project
fpath = self.sync_server.get_local_file_path(project,
site,
fpath)
fpath = os.path.normpath(os.path.dirname(fpath))
if os.path.isdir(fpath):
@ -556,6 +588,10 @@ class SyncRepresentationWidget(QtWidgets.QWidget):
raise OSError('unsupported xdg-open call??')
ProviderRole = QtCore.Qt.UserRole + 2
ProgressRole = QtCore.Qt.UserRole + 4
class SyncRepresentationModel(QtCore.QAbstractTableModel):
"""
Model for summary of representations.
@ -612,15 +648,20 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
sync_dt = attr.ib(default=None)
local_site = attr.ib(default=None)
remote_site = attr.ib(default=None)
local_provider = attr.ib(default=None)
remote_provider = attr.ib(default=None)
local_progress = attr.ib(default=None)
remote_progress = attr.ib(default=None)
files_count = attr.ib(default=None)
files_size = attr.ib(default=None)
priority = attr.ib(default=None)
state = attr.ib(default=None)
path = attr.ib(default=None)
def __init__(self, sync_server, header, project=None):
def __init__(self, sync_server, header, project=None, header_labels=None):
super(SyncRepresentationModel, self).__init__()
self._header = header
self._header_labels = header_labels
self._data = []
self._project = project
self._rec_loaded = 0
@ -634,8 +675,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self.sync_server = sync_server
# TODO think about admin mode
# this is for regular user, always only single local and single remote
self.local_site = self.sync_server.get_active_site(self._project)
self.remote_site = self.sync_server.get_remote_site(self._project)
self.local_site = self.sync_server.get_active_site(self.project)
self.remote_site = self.sync_server.get_remote_site(self.project)
self.projection = self.get_default_projection()
@ -659,26 +700,46 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
All queries should go through this (because of collection).
"""
return self.sync_server.connection.database[self._project]
return self.sync_server.connection.database[self.project]
@property
def project(self):
"""Returns project"""
return self._project
def data(self, index, role):
item = self._data[index.row()]
if role == ProviderRole:
if self._header[index.column()] == 'local_site':
return item.local_provider
if self._header[index.column()] == 'remote_site':
return item.remote_provider
if role == ProgressRole:
if self._header[index.column()] == 'local_site':
return item.local_progress
if self._header[index.column()] == 'remote_site':
return item.remote_progress
if role == Qt.DisplayRole:
return attr.asdict(item)[self._header[index.column()]]
if role == Qt.UserRole:
return item._id
def rowCount(self, index):
def rowCount(self, _index):
return len(self._data)
def columnCount(self, index):
def columnCount(self, _index):
return len(self._header)
def headerData(self, section, orientation, role):
if role == Qt.DisplayRole:
if orientation == Qt.Horizontal:
return str(self._header[section])
if self._header_labels:
return str(self._header_labels[section])
else:
return str(self._header[section])
def tick(self):
"""
@ -718,7 +779,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
than single page of records)
"""
if self.sync_server.is_paused() or \
self.sync_server.is_project_paused(self._project):
self.sync_server.is_project_paused(self.project):
return
self.beginResetModel()
@ -751,10 +812,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self._total_records = count
local_provider = _translate_provider_for_icon(self.sync_server,
self._project,
self.project,
local_site)
remote_provider = _translate_provider_for_icon(self.sync_server,
self._project,
self.project,
remote_site)
for repre in result.get("paginatedResults"):
@ -784,7 +845,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
if context.get("version"):
version = "v{:0>3d}".format(context.get("version"))
else:
version = "hero"
version = "master"
item = self.SyncRepresentation(
repre.get("_id"),
@ -794,8 +855,12 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
context.get("representation"),
local_updated,
remote_updated,
'{} {}'.format(local_provider, avg_progress_local),
'{} {}'.format(remote_provider, avg_progress_remote),
local_site,
remote_site,
local_provider,
remote_provider,
avg_progress_local,
avg_progress_remote,
repre.get("files_count", 1),
repre.get("files_size", 0),
1,
@ -806,7 +871,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self._data.append(item)
self._rec_loaded += 1
def canFetchMore(self, index):
def canFetchMore(self, _index):
"""
Check if there are more records than currently loaded
"""
@ -858,7 +923,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1}
self.query = self.get_default_query()
# import json
# log.debug(json.dumps(self.query, indent=4).replace('False', 'false').\
# log.debug(json.dumps(self.query, indent=4).\
# replace('False', 'false').\
# replace('True', 'true').replace('None', 'null'))
representations = self.dbcon.aggregate(self.query)
@ -883,8 +949,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel):
"""
self._project = project
self.sync_server.set_sync_project_settings()
self.local_site = self.sync_server.get_active_site(self._project)
self.remote_site = self.sync_server.get_remote_site(self._project)
self.local_site = self.sync_server.get_active_site(self.project)
self.remote_site = self.sync_server.get_remote_site(self.project)
self.refresh()
def get_index(self, id):
@ -1206,15 +1272,26 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
default_widths = (
("file", 290),
("created_dt", 120),
("sync_dt", 120),
("local_site", 60),
("remote_site", 60),
("created_dt", 105),
("sync_dt", 105),
("local_site", 80),
("remote_site", 80),
("size", 60),
("priority", 20),
("state", 90)
)
column_labels = (
("file", "File name"),
("created_dt", "Created"),
("sync_dt", "Synced"),
("local_site", "Active site"),
("remote_site", "Remote site"),
("files_size", "Size"),
("priority", "Priority"),
("state", "Status")
)
def __init__(self, sync_server, _id=None, project=None, parent=None):
super(SyncRepresentationDetailWidget, self).__init__(parent)
@ -1235,9 +1312,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
self.table_view = QtWidgets.QTableView()
headers = [item[0] for item in self.default_widths]
header_labels = [item[1] for item in self.column_labels]
model = SyncRepresentationDetailModel(sync_server, headers, _id,
project)
project, header_labels)
self.table_view.setModel(model)
self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self.table_view.setSelectionMode(
@ -1330,23 +1408,39 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
menu = QtWidgets.QMenu()
actions_mapping = {}
actions_kwargs_mapping = {}
action = QtWidgets.QAction("Open in explorer")
actions_mapping[action] = self._open_in_explorer
menu.addAction(action)
local_site = self.item.local_site
local_progress = self.item.local_progress
remote_site = self.item.remote_site
remote_progress = self.item.remote_progress
for site, progress in {local_site: local_progress,
remote_site: remote_progress}.items():
project = self.table_view.model().project
provider = self.sync_server.get_provider_for_site(project,
site)
if provider == 'local_drive':
if 'studio' in site:
txt = " studio version"
else:
txt = " local version"
action = QtWidgets.QAction("Open in explorer" + txt)
if progress == 1:
actions_mapping[action] = self._open_in_explorer
actions_kwargs_mapping[action] = {'site': site}
menu.addAction(action)
if self.item.state == STATUS[1]:
action = QtWidgets.QAction("Open error detail")
actions_mapping[action] = self._show_detail
menu.addAction(action)
remote_site, remote_progress = self.item.remote_site.split()
if float(remote_progress) == 1.0:
action = QtWidgets.QAction("Reset local site")
actions_mapping[action] = self._reset_local_site
menu.addAction(action)
local_site, local_progress = self.item.local_site.split()
if float(local_progress) == 1.0:
action = QtWidgets.QAction("Reset remote site")
actions_mapping[action] = self._reset_remote_site
@ -1360,8 +1454,9 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
result = menu.exec_(QtGui.QCursor.pos())
if result:
to_run = actions_mapping[result]
to_run_kwargs = actions_kwargs_mapping.get(result, {})
if to_run:
to_run()
to_run(**to_run_kwargs)
def _reset_local_site(self):
"""
@ -1369,7 +1464,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
'local',
self.item._id)
@ -1381,19 +1476,19 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget):
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.table_view.model().project,
self.representation_id,
'remote',
self.item._id)
self.table_view.model().refresh()
def _open_in_explorer(self):
def _open_in_explorer(self, site):
if not self.item:
return
fpath = self.item.path
project = self.table_view.model()._project
fpath = self.sync_server.get_local_file_path(project, fpath)
project = self.project
fpath = self.sync_server.get_local_file_path(project, site, fpath)
fpath = os.path.normpath(os.path.dirname(fpath))
if os.path.isdir(fpath):
@ -1415,6 +1510,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
Used in detail window accessible after clicking on single repre in the
summary.
TODO refactor - merge with SyncRepresentationModel if possible
Args:
sync_server (SyncServer) - object to call server operations (update
db status, set site status...)
@ -1424,7 +1521,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
a specific collection
"""
PAGE_SIZE = 30
# TODO add filter filename
DEFAULT_SORT = {
"files.path": 1
}
@ -1452,6 +1548,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
sync_dt = attr.ib(default=None)
local_site = attr.ib(default=None)
remote_site = attr.ib(default=None)
local_provider = attr.ib(default=None)
remote_provider = attr.ib(default=None)
local_progress = attr.ib(default=None)
remote_progress = attr.ib(default=None)
size = attr.ib(default=None)
priority = attr.ib(default=None)
state = attr.ib(default=None)
@ -1459,9 +1559,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
error = attr.ib(default=None)
path = attr.ib(default=None)
def __init__(self, sync_server, header, _id, project=None):
def __init__(self, sync_server, header, _id,
project=None, header_labels=None):
super(SyncRepresentationDetailModel, self).__init__()
self._header = header
self._header_labels = header_labels
self._data = []
self._project = project
self._rec_loaded = 0
@ -1473,8 +1575,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
self.sync_server = sync_server
# TODO think about admin mode
# this is for regular user, always only single local and single remote
self.local_site = self.sync_server.get_active_site(self._project)
self.remote_site = self.sync_server.get_remote_site(self._project)
self.local_site = self.sync_server.get_active_site(self.project)
self.remote_site = self.sync_server.get_remote_site(self.project)
self.sort = self.DEFAULT_SORT
@ -1491,9 +1593,26 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
@property
def dbcon(self):
return self.sync_server.connection.database[self._project]
"""
Database object with preselected project (collection) to run DB
operations (find, aggregate).
All queries should go through this (because of collection).
"""
return self.sync_server.connection.database[self.project]
@property
def project(self):
"""Returns project"""
return self.project
def tick(self):
"""
Triggers refresh of model.
Because of pagination, prepared (sorting, filtering) query needs
to be run on DB every X seconds.
"""
self.refresh(representations=None, load_records=self._rec_loaded)
self.timer.start(SyncRepresentationModel.REFRESH_SEC)
@ -1510,21 +1629,37 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
def data(self, index, role):
item = self._data[index.row()]
if role == ProviderRole:
if self._header[index.column()] == 'local_site':
return item.local_provider
if self._header[index.column()] == 'remote_site':
return item.remote_provider
if role == ProgressRole:
if self._header[index.column()] == 'local_site':
return item.local_progress
if self._header[index.column()] == 'remote_site':
return item.remote_progress
if role == Qt.DisplayRole:
return attr.asdict(item)[self._header[index.column()]]
if role == Qt.UserRole:
return item._id
def rowCount(self, index):
def rowCount(self, _index):
return len(self._data)
def columnCount(self, index):
def columnCount(self, _index):
return len(self._header)
def headerData(self, section, orientation, role):
if role == Qt.DisplayRole:
if orientation == Qt.Horizontal:
return str(self._header[section])
if self._header_labels:
return str(self._header_labels[section])
else:
return str(self._header[section])
def refresh(self, representations=None, load_records=0):
if self.sync_server.is_paused():
@ -1561,10 +1696,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
self._total_records = count
local_provider = _translate_provider_for_icon(self.sync_server,
self._project,
self.project,
local_site)
remote_provider = _translate_provider_for_icon(self.sync_server,
self._project,
self.project,
remote_site)
for repre in result.get("paginatedResults"):
@ -1585,9 +1720,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
repre.get('updated_dt_remote').strftime(
"%Y%m%dT%H%M%SZ")
progress_remote = _convert_progress(
remote_progress = _convert_progress(
repre.get('progress_remote', '0'))
progress_local = _convert_progress(
local_progress = _convert_progress(
repre.get('progress_local', '0'))
errors = []
@ -1601,8 +1736,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
os.path.basename(file["path"]),
local_updated,
remote_updated,
'{} {}'.format(local_provider, progress_local),
'{} {}'.format(remote_provider, progress_remote),
local_site,
remote_site,
local_provider,
remote_provider,
local_progress,
remote_progress,
file.get('size', 0),
1,
STATUS[repre.get("status", -1)],
@ -1614,7 +1753,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
self._data.append(item)
self._rec_loaded += 1
def canFetchMore(self, index):
def canFetchMore(self, _index):
"""
Check if there are more records than currently loaded
"""
@ -1918,11 +2057,8 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate):
option.palette.highlight())
painter.setOpacity(1)
d = index.data(QtCore.Qt.DisplayRole)
if d:
provider, value = d.split()
else:
return
provider = index.data(ProviderRole)
value = index.data(ProgressRole)
if not self.icons.get(provider):
resource_path = os.path.dirname(__file__)
@ -2008,7 +2144,7 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate):
def __init__(self, parent=None):
super(SizeDelegate, self).__init__(parent)
def displayText(self, value, locale):
def displayText(self, value, _locale):
if value is None:
# Ignore None value
return

8
openpype/modules/sync_server/utils.py
View file

@ -1,8 +1,14 @@
import time
from openpype.api import Logger
from openpype.api import Logger
log = Logger().get_logger("SyncServer")
class SyncStatus:
DO_NOTHING = 0
DO_UPLOAD = 1
DO_DOWNLOAD = 2
def time_function(method):
""" Decorator to print how much time function took.
For debugging.

33
openpype/plugins/load/add_site.py Normal file
View file

@ -0,0 +1,33 @@
from avalon import api
from openpype.modules import ModulesManager
class AddSyncSite(api.Loader):
"""Add sync site to representation"""
representations = ["*"]
families = ["*"]
label = "Add Sync Site"
order = 2 # lower means better
icon = "download"
color = "#999999"
def load(self, context, name=None, namespace=None, data=None):
self.log.info("Adding {} to representation: {}".format(
data["site_name"], data["_id"]))
self.add_site_to_representation(data["project_name"],
data["_id"],
data["site_name"])
self.log.debug("Site added.")
@staticmethod
def add_site_to_representation(project_name, representation_id, site_name):
"""Adds new site to representation_id, resets if exists"""
manager = ModulesManager()
sync_server = manager.modules_by_name["sync_server"]
sync_server.add_site(project_name, representation_id, site_name,
force=True)
def filepath_from_context(self, context):
"""No real file loading"""
return ""

6
openpype/plugins/load/delete_old_versions.py
View file

@ -15,11 +15,12 @@ from openpype.api import Anatomy
class DeleteOldVersions(api.Loader):
"""Deletes specific number of old version"""
representations = ["*"]
families = ["*"]
label = "Delete Old Versions"
order = 35
icon = "trash"
color = "#d8d8d8"
@ -421,8 +422,9 @@ class DeleteOldVersions(api.Loader):
class CalculateOldVersions(DeleteOldVersions):
"""Calculate file size of old versions"""
label = "Calculate Old Versions"
order = 30
options = [
qargparse.Integer(

33
openpype/plugins/load/remove_site.py Normal file
View file

@ -0,0 +1,33 @@
from avalon import api
from openpype.modules import ModulesManager
class RemoveSyncSite(api.Loader):
"""Remove sync site and its files on representation"""
representations = ["*"]
families = ["*"]
label = "Remove Sync Site"
order = 4
icon = "download"
color = "#999999"
def load(self, context, name=None, namespace=None, data=None):
self.log.info("Removing {} on representation: {}".format(
data["site_name"], data["_id"]))
self.remove_site_on_representation(data["project_name"],
data["_id"],
data["site_name"])
self.log.debug("Site added.")
@staticmethod
def remove_site_on_representation(project_name, representation_id,
site_name):
manager = ModulesManager()
sync_server = manager.modules_by_name["sync_server"]
sync_server.remove_site(project_name, representation_id,
site_name, True)
def filepath_from_context(self, context):
"""No real file loading"""
return ""