ayon-core/pype/modules/sync_server/tray/app.py
2021-02-16 09:29:29 +01:00

1511 lines
53 KiB
Python

from Qt import QtWidgets, QtCore, QtGui
from Qt.QtCore import Qt
import attr
import os
from pype.tools.settings import (
ProjectListWidget,
style
)
from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp
from pype.lib import PypeLogger
import json
log = PypeLogger().get_logger("SyncServer")
STATUS = {
0: 'In Progress',
1: 'Failed',
2: 'Queued',
3: 'Paused',
4: 'Synced OK',
-1: 'Not available'
}
class SyncServerWindow(QtWidgets.QDialog):
"""
Main window that contains list of synchronizable projects and summary
view with all synchronizable representations for first project
"""
def __init__(self, sync_server, parent=None):
super(SyncServerWindow, self).__init__(parent)
self.setWindowFlags(QtCore.Qt.Window)
self.setFocusPolicy(QtCore.Qt.StrongFocus)
self.setStyleSheet(style.load_stylesheet())
self.setWindowIcon(QtGui.QIcon(style.app_icon_path()))
self.resize(1400, 800)
body = QtWidgets.QWidget(self)
footer = QtWidgets.QWidget(self)
footer.setFixedHeight(20)
container = QtWidgets.QWidget()
projects = SyncProjectListWidget(sync_server, self)
projects.refresh() # force selection of default
repres = SyncRepresentationWidget(sync_server,
project=projects.current_project,
parent=self)
container_layout = QtWidgets.QHBoxLayout(container)
container_layout.setContentsMargins(0, 0, 0, 0)
split = QtWidgets.QSplitter()
split.addWidget(projects)
split.addWidget(repres)
split.setSizes([180, 950, 200])
container_layout.addWidget(split)
container.setLayout(container_layout)
body_layout = QtWidgets.QHBoxLayout(body)
body_layout.addWidget(container)
body_layout.setContentsMargins(0, 0, 0, 0)
message = QtWidgets.QLabel(footer)
message.hide()
footer_layout = QtWidgets.QVBoxLayout(footer)
footer_layout.addWidget(message)
footer_layout.setContentsMargins(0, 0, 0, 0)
layout = QtWidgets.QVBoxLayout(self)
layout.addWidget(body)
layout.addWidget(footer)
self.setLayout(body_layout)
self.setWindowTitle("Sync Server")
projects.project_changed.connect(
lambda: repres.table_view.model().set_project(
projects.current_project))
class SyncProjectListWidget(ProjectListWidget):
"""
Lists all projects that are synchronized to choose from
"""
def __init__(self, sync_server, parent):
super(SyncProjectListWidget, self).__init__(parent)
self.sync_server = sync_server
def validate_context_change(self):
return True
def refresh(self):
model = self.project_list.model()
model.clear()
for project_name in self.sync_server.get_synced_presets().keys():
model.appendRow(QtGui.QStandardItem(project_name))
if len(self.sync_server.get_synced_presets().keys()) == 0:
model.appendRow(QtGui.QStandardItem("No project configured"))
self.current_project = self.project_list.currentIndex().data(
QtCore.Qt.DisplayRole
)
if not self.current_project:
self.current_project = self.project_list.model().item(0). \
data(QtCore.Qt.DisplayRole)
class SyncRepresentationWidget(QtWidgets.QWidget):
"""
Summary dialog with list of representations that matches current
settings 'local_site' and 'remote_site'.
"""
active_changed = QtCore.Signal() # active index changed
default_widths = (
("asset", 210),
("subset", 190),
("version", 10),
("representation", 90),
("created_dt", 100),
("sync_dt", 100),
("local_site", 60),
("remote_site", 70),
("files_count", 70),
("files_size", 70),
("priority", 20),
("state", 50)
)
def __init__(self, sync_server, project=None, parent=None):
super(SyncRepresentationWidget, self).__init__(parent)
self.sync_server = sync_server
self._selected_id = None # keep last selected _id
self.filter = QtWidgets.QLineEdit()
self.filter.setPlaceholderText("Filter representations..")
top_bar_layout = QtWidgets.QHBoxLayout()
top_bar_layout.addWidget(self.filter)
self.table_view = QtWidgets.QTableView()
headers = [item[0] for item in self.default_widths]
model = SyncRepresentationModel(sync_server, headers, project)
self.table_view.setModel(model)
self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self.table_view.setSelectionMode(
QtWidgets.QAbstractItemView.SingleSelection)
self.table_view.setSelectionBehavior(
QtWidgets.QAbstractItemView.SelectRows)
self.table_view.horizontalHeader().setSortIndicator(
-1, Qt.AscendingOrder)
self.table_view.setSortingEnabled(True)
self.table_view.setAlternatingRowColors(True)
self.table_view.verticalHeader().hide()
time_delegate = PrettyTimeDelegate(self)
column = self.table_view.model().get_header_index("created_dt")
self.table_view.setItemDelegateForColumn(column, time_delegate)
column = self.table_view.model().get_header_index("sync_dt")
self.table_view.setItemDelegateForColumn(column, time_delegate)
column = self.table_view.model().get_header_index("local_site")
delegate = ImageDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
column = self.table_view.model().get_header_index("remote_site")
delegate = ImageDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
column = self.table_view.model().get_header_index("files_size")
delegate = SizeDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
for column_name, width in self.default_widths:
idx = model.get_header_index(column_name)
self.table_view.setColumnWidth(idx, width)
layout = QtWidgets.QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
layout.addLayout(top_bar_layout)
layout.addWidget(self.table_view)
self.table_view.doubleClicked.connect(self._double_clicked)
self.filter.textChanged.connect(lambda: model.set_filter(
self.filter.text()))
self.table_view.customContextMenuRequested.connect(
self._on_context_menu)
self.table_view.model().modelReset.connect(self._set_selection)
self.selection_model = self.table_view.selectionModel()
self.selection_model.selectionChanged.connect(self._selection_changed)
def _selection_changed(self, new_selection):
index = self.selection_model.currentIndex()
self._selected_id = self.table_view.model().data(index, Qt.UserRole)
def _set_selection(self):
"""
Sets selection to 'self._selected_id' if exists.
Keep selection during model refresh.
"""
if self._selected_id:
index = self.table_view.model().get_index(self._selected_id)
if index and index.isValid():
mode = QtCore.QItemSelectionModel.Select | \
QtCore.QItemSelectionModel.Rows
self.selection_model.setCurrentIndex(index, mode)
else:
self._selected_id = None
def _double_clicked(self, index):
"""
Opens representation dialog with all files after doubleclick
"""
_id = self.table_view.model().data(index, Qt.UserRole)
detail_window = SyncServerDetailWindow(
self.sync_server, _id, self.table_view.model()._project)
detail_window.exec()
def _on_context_menu(self, point):
"""
Shows menu with loader actions on Right-click.
"""
point_index = self.table_view.indexAt(point)
if not point_index.isValid():
return
class SyncRepresentationModel(QtCore.QAbstractTableModel):
PAGE_SIZE = 19
REFRESH_SEC = 5000
DEFAULT_SORT = {
"updated_dt_remote": -1,
"_id": 1
}
SORT_BY_COLUMN = [
"context.asset", # asset
"context.subset", # subset
"context.version", # version
"context.representation", # representation
"updated_dt_local", # local created_dt
"updated_dt_remote", # remote created_dt
"avg_progress_local", # local progress
"avg_progress_remote", # remote progress
"files_count", # count of files
"files_size", # file size of all files
"context.asset", # priority TODO
"status" # state
]
numberPopulated = QtCore.Signal(int)
@attr.s
class SyncRepresentation:
"""
Auxiliary object for easier handling.
Fields must contain all header values (+ any arbitrary values).
"""
_id = attr.ib()
asset = attr.ib()
subset = attr.ib()
version = attr.ib()
representation = attr.ib()
created_dt = attr.ib(default=None)
sync_dt = attr.ib(default=None)
local_site = attr.ib(default=None)
remote_site = attr.ib(default=None)
files_count = attr.ib(default=None)
files_size = attr.ib(default=None)
priority = attr.ib(default=None)
state = attr.ib(default=None)
def __init__(self, sync_server, header, project=None):
super(SyncRepresentationModel, self).__init__()
self._header = header
self._data = []
self._project = project
self._rec_loaded = 0
self._buffer = [] # stash one page worth of records (actually cursor)
self.filter = None
self._initialized = False
self.sync_server = sync_server
# TODO think about admin mode
# this is for regular user, always only single local and single remote
self.local_site, self.remote_site = \
self.sync_server.get_sites_for_project(self._project)
self.projection = self.get_default_projection()
self.sort = self.DEFAULT_SORT
self.query = self.get_default_query()
self.default_query = list(self.get_default_query())
log.debug("!!! init query: {}".format(json.dumps(self.query,
indent=4)))
representations = self.dbcon.aggregate(self.query)
self.refresh(representations)
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.tick)
self.timer.start(self.REFRESH_SEC)
@property
def dbcon(self):
return self.sync_server.connection.database[self._project]
def data(self, index, role):
item = self._data[index.row()]
if role == Qt.DisplayRole:
return attr.asdict(item)[self._header[index.column()]]
if role == Qt.UserRole:
return item._id
def rowCount(self, index):
return len(self._data)
def columnCount(self, index):
return len(self._header)
def headerData(self, section, orientation, role):
if role == Qt.DisplayRole:
if orientation == Qt.Horizontal:
return str(self._header[section])
def tick(self):
self.refresh(representations=None, load_records=self._rec_loaded)
self.timer.start(self.REFRESH_SEC)
def get_header_index(self, value):
"""
Returns index of 'value' in headers
Args:
value (str): header name value
Returns:
(int)
"""
return self._header.index(value)
def refresh(self, representations=None, load_records=0):
self.beginResetModel()
self._data = []
self._rec_loaded = 0
if not representations:
self.query = self.get_default_query(load_records)
representations = self.dbcon.aggregate(self.query)
self._add_page_records(self.local_site, self.remote_site,
representations)
self.endResetModel()
def _add_page_records(self, local_site, remote_site, representations):
for repre in representations:
context = repre.get("context").pop()
files = repre.get("files", [])
if isinstance(files, dict): # aggregate returns dictionary
files = [files]
# representation without files doesnt concern us
if not files:
continue
local_updated = remote_updated = None
if repre.get('updated_dt_local'):
local_updated = \
repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ")
if repre.get('updated_dt_remote'):
remote_updated = \
repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ")
avg_progress_remote = repre.get('avg_progress_remote', '')
avg_progress_local = repre.get('avg_progress_local', '')
item = self.SyncRepresentation(
repre.get("_id"),
context.get("asset"),
context.get("subset"),
"v{:0>3d}".format(context.get("version", 1)),
context.get("representation"),
local_updated,
remote_updated,
'{} {}'.format(local_site, avg_progress_local),
'{} {}'.format(remote_site, avg_progress_remote),
repre.get("files_count", 1),
repre.get("files_size", 0),
1,
STATUS[repre.get("status", -1)]
)
self._data.append(item)
self._rec_loaded += 1
def canFetchMore(self, index):
"""
Check if there are more records than currently loaded
"""
# 'skip' might be suboptimal when representation hits 500k+
self._buffer = list(self.dbcon.aggregate(self.query))
# log.info("!!! canFetchMore _rec_loaded::{} - {}".format(
# self._rec_loaded, len(self._buffer)))
return len(self._buffer) > self._rec_loaded
def fetchMore(self, index):
"""
Add more record to model.
Called when 'canFetchMore' returns true, which means there are
more records in DB than loaded.
'self._buffer' is used to stash cursor to limit requery
"""
log.debug("fetchMore")
# cursor.count() returns always total number, not only skipped + limit
remainder = len(self._buffer) - self._rec_loaded
items_to_fetch = min(self.PAGE_SIZE, remainder)
self.beginInsertRows(index,
self._rec_loaded,
self._rec_loaded + items_to_fetch - 1)
self._add_page_records(self.local_site, self.remote_site, self._buffer)
self.endInsertRows()
self.numberPopulated.emit(items_to_fetch) # ??
def sort(self, index, order):
"""
Summary sort per representation.
Sort is happening on a DB side, model is reset, db queried
again.
Args:
index (int): column index
order (int): 0|
"""
# limit unwanted first re-sorting by view
if index < 0:
return
self._rec_loaded = 0
if order == 0:
order = 1
else:
order = -1
self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1}
self.query = self.get_default_query()
representations = self.dbcon.aggregate(self.query)
self.refresh(representations)
def set_filter(self, filter):
"""
Adds text value filtering
Args:
filter (str): string inputted by user
"""
self.filter = filter
self.refresh()
def set_project(self, project):
"""
Changes project, called after project selection is changed
Args:
project (str): name of project
"""
self._project = project
self.refresh()
def get_index(self, id):
"""
Get index of 'id' value.
Used for keeping selection after refresh.
Args:
id (str): MongoDB _id
Returns:
(QModelIndex)
"""
index = None
for i in range(self.rowCount(None)):
index = self.index(i, 0)
value = self.data(index, Qt.UserRole)
if value == id:
return index
return index
def get_default_query(self, limit=0):
"""
Returns basic aggregate query for main table.
Main table provides summary information about representation,
which could have multiple files. Details are accessible after
double click on representation row.
Columns:
'created_dt' - max of created or updated (when failed) per repr
'sync_dt' - same for remote side
'local_site' - progress of repr on local side, 1 = finished
'remote_site' - progress on remote side, calculates from files
'state' -
0 - in progress
1 - failed
2 - queued
3 - paused (not implemented yet)
4 - finished on both sides
are calculated and must be calculated in DB because of
pagination
Args:
limit (int): how many records should be returned, by default
it 'PAGE_SIZE' for performance.
Should be overridden by value of loaded records for refresh
functionality (got more records by scrolling, refresh
shouldn't reset that)
"""
if limit == 0:
limit = SyncRepresentationModel.PAGE_SIZE
return [
{"$match": self._get_match_part()},
{'$unwind': '$files'},
# merge potentially unwinded records back to single per repre
{'$addFields': {
'order_remote': {
'$filter': {'input': '$files.sites', 'as': 'p',
'cond': {'$eq': ['$$p.name', self.remote_site]}
}},
'order_local': {
'$filter': {'input': '$files.sites', 'as': 'p',
'cond': {'$eq': ['$$p.name', self.local_site]}
}}
}},
{'$addFields': {
# prepare progress per file, presence of 'created_dt' denotes
# successfully finished load/download
'progress_remote': {'$first': {
'$cond': [{'$size': "$order_remote.progress"},
"$order_remote.progress",
{'$cond': [
{'$size': "$order_remote.created_dt"},
[1],
[0]
]}
]}},
'progress_local': {'$first': {
'$cond': [{'$size': "$order_local.progress"},
"$order_local.progress",
{'$cond': [
{'$size': "$order_local.created_dt"},
[1],
[0]
]}
]}},
# file might be successfully created or failed, not both
'updated_dt_remote': {'$first': {
'$cond': [{'$size': "$order_remote.created_dt"},
"$order_remote.created_dt",
{'$cond': [
{'$size': "$order_remote.last_failed_dt"},
"$order_remote.last_failed_dt",
[]
]}
]}},
'updated_dt_local': {'$first': {
'$cond': [{'$size': "$order_local.created_dt"},
"$order_local.created_dt",
{'$cond': [
{'$size': "$order_local.last_failed_dt"},
"$order_local.last_failed_dt",
[]
]}
]}},
'files_size': {'$ifNull': ["$files.size", 0]},
'failed_remote': {
'$cond': [{'$size': "$order_remote.last_failed_dt"},
1,
0]},
'failed_local': {
'$cond': [{'$size': "$order_local.last_failed_dt"},
1,
0]}
}},
{'$group': {
'_id': '$_id',
# pass through context - same for representation
'context': {'$addToSet': '$context'},
# pass through files as a list
'files': {'$addToSet': '$files'},
# count how many files
'files_count': {'$sum': 1},
'files_size': {'$sum': '$files_size'},
# sum avg progress, finished = 1
'avg_progress_remote': {'$avg': "$progress_remote"},
'avg_progress_local': {'$avg': "$progress_local"},
# select last touch of file
'updated_dt_remote': {'$max': "$updated_dt_remote"},
'failed_remote': {'$sum': '$failed_remote'},
'failed_local': {'$sum': '$failed_local'},
'updated_dt_local': {'$max': "$updated_dt_local"}
}},
{"$limit": limit},
{"$skip": self._rec_loaded},
{"$project": self.projection},
{"$sort": self.sort}
]
def _get_match_part(self):
"""
Extend match part with filter if present.
Filter is set by user input. Each model has different fields to be
checked.
If performance issues are found, '$text' and text indexes should
be investigated.
"""
if not self.filter:
return {
"type": "representation",
'files.sites': {
'$elemMatch': {
'$or': [
{'name': self.local_site},
{'name': self.remote_site}
]
}
}
}
else:
regex_str = '.*{}.*'.format(self.filter)
return {
"type": "representation",
'$or': [
{'context.subset': {'$regex': regex_str, '$options': 'i'}},
{'context.asset': {'$regex': regex_str, '$options': 'i'}},
{'context.representation': {'$regex': regex_str,
'$options': 'i'}}],
'files.sites': {
'$elemMatch': {
'$or': [
{'name': self.local_site},
{'name': self.remote_site}
]
}
}
}
def get_default_projection(self):
"""
Projection part for aggregate query.
All fields with '1' will be returned, no others.
Returns:
(dict)
"""
return {
"context.subset": 1,
"context.asset": 1,
"context.version": 1,
"context.representation": 1,
"files": 1,
'files_count': 1,
"files_size": 1,
'avg_progress_remote': 1,
'avg_progress_local': 1,
'updated_dt_remote': 1,
'updated_dt_local': 1,
'status': {
'$switch': {
'branches': [
{
'case': {
'$or': [{'$eq': ['$avg_progress_remote', 0]},
{'$eq': ['$avg_progress_local', 0]}]},
'then': 2 # Queued
},
{
'case': {
'$or': ['$failed_remote', '$failed_local']},
'then': 1 # Failed
},
{
'case': {'$or': [{'$and': [
{'$gt': ['$avg_progress_remote', 0]},
{'$lt': ['$avg_progress_remote', 1]}
]},
{'$and': [
{'$gt': ['$avg_progress_local', 0]},
{'$lt': ['$avg_progress_local', 1]}
]}
]},
'then': 0 # In progress
},
{
'case': {'$eq': ['dummy_placeholder', 'paused']},
'then': 3 # Paused
},
{
'case': {'$and': [
{'$eq': ['$avg_progress_remote', 1]},
{'$eq': ['$avg_progress_local', 1]}
]},
'then': 4 # Synced OK
},
],
'default': -1
}
}
}
class SyncServerDetailWindow(QtWidgets.QDialog):
def __init__(self, sync_server, _id, project, parent=None):
log.debug(
"!!! SyncServerDetailWindow _id:: {}".format(_id))
super(SyncServerDetailWindow, self).__init__(parent)
self.setWindowFlags(QtCore.Qt.Window)
self.setFocusPolicy(QtCore.Qt.StrongFocus)
self.setStyleSheet(style.load_stylesheet())
self.setWindowIcon(QtGui.QIcon(style.app_icon_path()))
self.resize(1000, 400)
body = QtWidgets.QWidget()
footer = QtWidgets.QWidget()
footer.setFixedHeight(20)
container = SyncRepresentationDetailWidget(sync_server, _id, project,
parent=self)
body_layout = QtWidgets.QHBoxLayout(body)
body_layout.addWidget(container)
body_layout.setContentsMargins(0, 0, 0, 0)
message = QtWidgets.QLabel()
message.hide()
footer_layout = QtWidgets.QVBoxLayout(footer)
footer_layout.addWidget(message)
footer_layout.setContentsMargins(0, 0, 0, 0)
layout = QtWidgets.QVBoxLayout(self)
layout.addWidget(body)
layout.addWidget(footer)
self.setLayout(body_layout)
self.setWindowTitle("Sync Representation Detail")
class SyncRepresentationDetailWidget(QtWidgets.QWidget):
"""
Widget to display list of synchronizable files for single repre.
Args:
_id (str): representation _id
project (str): name of project with repre
parent (QDialog): SyncServerDetailWindow
"""
active_changed = QtCore.Signal() # active index changed
default_widths = (
("file", 290),
("created_dt", 120),
("sync_dt", 120),
("local_site", 60),
("remote_site", 60),
("size", 60),
("priority", 20),
("state", 90)
)
def __init__(self, sync_server, _id=None, project=None, parent=None):
super(SyncRepresentationDetailWidget, self).__init__(parent)
self.representation_id = _id
self.item = None # set to item that mouse was clicked over
self.sync_server = sync_server
self._selected_id = None
self.filter = QtWidgets.QLineEdit()
self.filter.setPlaceholderText("Filter representation..")
top_bar_layout = QtWidgets.QHBoxLayout()
top_bar_layout.addWidget(self.filter)
self.table_view = QtWidgets.QTableView()
headers = [item[0] for item in self.default_widths]
model = SyncRepresentationDetailModel(sync_server, headers, _id,
project)
self.table_view.setModel(model)
self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self.table_view.setSelectionMode(
QtWidgets.QAbstractItemView.SingleSelection)
self.table_view.setSelectionBehavior(
QtWidgets.QTableView.SelectRows)
self.table_view.horizontalHeader().setSortIndicator(-1,
Qt.AscendingOrder)
self.table_view.setSortingEnabled(True)
self.table_view.setAlternatingRowColors(True)
self.table_view.verticalHeader().hide()
time_delegate = PrettyTimeDelegate(self)
column = self.table_view.model().get_header_index("created_dt")
self.table_view.setItemDelegateForColumn(column, time_delegate)
column = self.table_view.model().get_header_index("sync_dt")
self.table_view.setItemDelegateForColumn(column, time_delegate)
column = self.table_view.model().get_header_index("local_site")
delegate = ImageDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
column = self.table_view.model().get_header_index("remote_site")
delegate = ImageDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
column = self.table_view.model().get_header_index("size")
delegate = SizeDelegate(self)
self.table_view.setItemDelegateForColumn(column, delegate)
for column_name, width in self.default_widths:
idx = model.get_header_index(column_name)
self.table_view.setColumnWidth(idx, width)
layout = QtWidgets.QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
layout.addLayout(top_bar_layout)
layout.addWidget(self.table_view)
self.filter.textChanged.connect(lambda: model.set_filter(
self.filter.text()))
self.table_view.customContextMenuRequested.connect(
self._on_context_menu)
self.table_view.model().modelReset.connect(self._set_selection)
self.selection_model = self.table_view.selectionModel()
self.selection_model.selectionChanged.connect(self._selection_changed)
def _selection_changed(self):
index = self.selection_model.currentIndex()
self._selected_id = self.table_view.model().data(index, Qt.UserRole)
def _set_selection(self):
"""
Sets selection to 'self._selected_id' if exists.
Keep selection during model refresh.
"""
if self._selected_id:
index = self.table_view.model().get_index(self._selected_id)
if index.isValid():
mode = QtCore.QItemSelectionModel.Select | \
QtCore.QItemSelectionModel.Rows
self.selection_model.setCurrentIndex(index, mode)
else:
self._selected_id = None
def _show_detail(self):
"""
Shows windows with error message for failed sync of a file.
"""
dt = max(self.item.created_dt, self.item.sync_dt)
detail_window = SyncRepresentationErrorWindow(self.item._id,
self.project,
dt,
self.item.tries,
self.item.error)
detail_window.exec()
def _on_context_menu(self, point):
"""
Shows menu with loader actions on Right-click.
"""
point_index = self.table_view.indexAt(point)
if not point_index.isValid():
return
self.item = self.table_view.model()._data[point_index.row()]
menu = QtWidgets.QMenu()
actions_mapping = {}
if self.item.state == STATUS[1]:
action = QtWidgets.QAction("Open error detail")
actions_mapping[action] = self._show_detail
menu.addAction(action)
remote_site, remote_progress = self.item.remote_site.split()
if remote_progress == '1':
action = QtWidgets.QAction("Reset local site")
actions_mapping[action] = self._reset_local_site
menu.addAction(action)
local_site, local_progress = self.item.local_site.split()
if local_progress == '1':
action = QtWidgets.QAction("Reset remote site")
actions_mapping[action] = self._reset_remote_site
menu.addAction(action)
if not actions_mapping:
action = QtWidgets.QAction("< No action >")
actions_mapping[action] = None
menu.addAction(action)
result = menu.exec_(QtGui.QCursor.pos())
if result:
to_run = actions_mapping[result]
if to_run:
to_run()
def _reset_local_site(self):
"""
Removes errors or success metadata for particular file >> forces
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.representation_id,
self.item._id,
'local')
def _reset_remote_site(self):
"""
Removes errors or success metadata for particular file >> forces
redo of upload/download
"""
self.sync_server.reset_provider_for_file(
self.table_view.model()._project,
self.representation_id,
self.item._id,
'remote')
class SyncRepresentationDetailModel(QtCore.QAbstractTableModel):
"""
List of all syncronizable files per single representation.
"""
PAGE_SIZE = 30
# TODO add filter filename
DEFAULT_SORT = {
"files.path": 1
}
SORT_BY_COLUMN = [
"files.path",
"updated_dt_local", # local created_dt
"updated_dt_remote", # remote created_dt
"progress_local", # local progress
"progress_remote", # remote progress
"size", # remote progress
"context.asset", # priority TODO
"status" # state
]
@attr.s
class SyncRepresentationDetail:
"""
Auxiliary object for easier handling.
Fields must contain all header values (+ any arbitrary values).
"""
_id = attr.ib()
file = attr.ib()
created_dt = attr.ib(default=None)
sync_dt = attr.ib(default=None)
local_site = attr.ib(default=None)
remote_site = attr.ib(default=None)
size = attr.ib(default=None)
priority = attr.ib(default=None)
state = attr.ib(default=None)
tries = attr.ib(default=None)
error = attr.ib(default=None)
def __init__(self, sync_server, header, _id, project=None):
super(SyncRepresentationDetailModel, self).__init__()
self._header = header
self._data = []
self._project = project
self._rec_loaded = 0
self.filter = None
self._buffer = [] # stash one page worth of records (actually cursor)
self._id = _id
self._initialized = False
self.sync_server = sync_server
# TODO think about admin mode
# this is for regular user, always only single local and single remote
self.local_site, self.remote_site = \
self.sync_server.get_sites_for_project(self._project)
self.sort = self.DEFAULT_SORT
# in case we would like to hide/show some columns
self.projection = self.get_default_projection()
self.query = self.get_default_query()
representations = self.dbcon.aggregate(self.query)
self.refresh(representations)
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.tick)
self.timer.start(SyncRepresentationModel.REFRESH_SEC)
@property
def dbcon(self):
return self.sync_server.connection.database[self._project]
def tick(self):
self.refresh(representations=None, load_records=self._rec_loaded)
self.timer.start(SyncRepresentationModel.REFRESH_SEC)
def get_header_index(self, value):
"""
Returns index of 'value' in headers
Args:
value (str): header name value
Returns:
(int)
"""
return self._header.index(value)
def data(self, index, role):
item = self._data[index.row()]
if role == Qt.DisplayRole:
return attr.asdict(item)[self._header[index.column()]]
if role == Qt.UserRole:
return item._id
def rowCount(self, index):
return len(self._data)
def columnCount(self, index):
return len(self._header)
def headerData(self, section, orientation, role):
if role == Qt.DisplayRole:
if orientation == Qt.Horizontal:
return str(self._header[section])
def refresh(self, representations=None, load_records=0):
self.beginResetModel()
self._data = []
self._rec_loaded = 0
if not representations:
self.query = self.get_default_query(load_records)
representations = self.dbcon.aggregate(self.query)
self._add_page_records(self.local_site, self.remote_site,
representations)
self.endResetModel()
def _add_page_records(self, local_site, remote_site, representations):
"""
Process all records from 'representation' and add them to storage.
Args:
local_site (str): name of local site (mine)
remote_site (str): name of cloud provider (theirs)
representations (Mongo Cursor)
"""
for repre in representations:
# log.info("!!! repre:: {}".format(repre))
files = repre.get("files", [])
if isinstance(files, dict): # aggregate returns dictionary
files = [files]
for file in files:
local_updated = remote_updated = None
if repre.get('updated_dt_local'):
local_updated = \
repre.get('updated_dt_local').strftime(
"%Y%m%dT%H%M%SZ")
if repre.get('updated_dt_remote'):
remote_updated = \
repre.get('updated_dt_remote').strftime(
"%Y%m%dT%H%M%SZ")
progress_remote = repre.get('progress_remote', '')
progress_local = repre.get('progress_local', '')
errors = []
if repre.get('failed_remote_error'):
errors.append(repre.get('failed_remote_error'))
if repre.get('failed_local_error'):
errors.append(repre.get('failed_local_error'))
item = self.SyncRepresentationDetail(
file.get("_id"),
os.path.basename(file["path"]),
local_updated,
remote_updated,
'{} {}'.format(local_site, progress_local),
'{} {}'.format(remote_site, progress_remote),
file.get('size', 0),
1,
STATUS[repre.get("status", -1)],
repre.get("tries"),
'\n'.join(errors)
)
self._data.append(item)
self._rec_loaded += 1
def canFetchMore(self, index):
"""
Check if there are more records than currently loaded
"""
# 'skip' might be suboptimal when representation hits 500k+
self._buffer = list(self.dbcon.aggregate(self.query))
return len(self._buffer) > self._rec_loaded
def fetchMore(self, index):
"""
Add more record to model.
Called when 'canFetchMore' returns true, which means there are
more records in DB than loaded.
'self._buffer' is used to stash cursor to limit requery
"""
log.debug("fetchMore")
# cursor.count() returns always total number, not only skipped + limit
remainder = len(self._buffer) - self._rec_loaded
items_to_fetch = min(self.PAGE_SIZE, remainder)
self.beginInsertRows(index,
self._rec_loaded,
self._rec_loaded + items_to_fetch - 1)
self._add_page_records(self.local_site, self.remote_site, self._buffer)
self.endInsertRows()
def sort(self, index, order):
# limit unwanted first re-sorting by view
if index < 0:
return
self._rec_loaded = 0 # change sort - reset from start
if order == 0:
order = 1
else:
order = -1
self.sort = {self.SORT_BY_COLUMN[index]: order}
self.query = self.get_default_query()
representations = self.dbcon.aggregate(self.query)
self.refresh(representations)
def set_filter(self, filter):
self.filter = filter
self.refresh()
def get_index(self, id):
"""
Get index of 'id' value.
Used for keeping selection after refresh.
Args:
id (str): MongoDB _id
Returns:
(QModelIndex)
"""
index = None
for i in range(self.rowCount(None)):
index = self.index(i, 0)
value = self.data(index, Qt.UserRole)
if value == id:
return index
return index
def get_default_query(self, limit=0):
"""
Gets query that gets used when no extra sorting, filtering or
projecting is needed.
Called for basic table view.
Returns:
[(dict)] - list with single dict - appropriate for aggregate
function for MongoDB
"""
if limit == 0:
limit = SyncRepresentationModel.PAGE_SIZE
return [
{"$match": self._get_match_part()},
{"$unwind": "$files"},
{'$addFields': {
'order_remote': {
'$filter': {'input': '$files.sites', 'as': 'p',
'cond': {'$eq': ['$$p.name', self.remote_site]}
}},
'order_local': {
'$filter': {'input': '$files.sites', 'as': 'p',
'cond': {'$eq': ['$$p.name', self.local_site]}
}}
}},
{'$addFields': {
# prepare progress per file, presence of 'created_dt' denotes
# successfully finished load/download
'progress_remote': {'$first': {
'$cond': [{'$size': "$order_remote.progress"},
"$order_remote.progress",
{'$cond': [
{'$size': "$order_remote.created_dt"},
[1],
[0]
]}
]}},
'progress_local': {'$first': {
'$cond': [{'$size': "$order_local.progress"},
"$order_local.progress",
{'$cond': [
{'$size': "$order_local.created_dt"},
[1],
[0]
]}
]}},
# file might be successfully created or failed, not both
'updated_dt_remote': {'$first': {
'$cond': [
{'$size': "$order_remote.created_dt"},
"$order_remote.created_dt",
{
'$cond': [
{'$size': "$order_remote.last_failed_dt"},
"$order_remote.last_failed_dt",
[]
]
}
]
}},
'updated_dt_local': {'$first': {
'$cond': [
{'$size': "$order_local.created_dt"},
"$order_local.created_dt",
{
'$cond': [
{'$size': "$order_local.last_failed_dt"},
"$order_local.last_failed_dt",
[]
]
}
]
}},
'failed_remote': {
'$cond': [{'$size': "$order_remote.last_failed_dt"},
1,
0]},
'failed_local': {
'$cond': [{'$size': "$order_local.last_failed_dt"},
1,
0]},
'failed_remote_error': {'$first': {
'$cond': [{'$size': "$order_remote.error"},
"$order_remote.error",
[""]]}},
'failed_local_error': {'$first': {
'$cond': [{'$size': "$order_local.error"},
"$order_local.error",
[""]]}},
'tries': {'$first': {
'$cond': [{'$size': "$order_local.tries"},
"$order_local.tries",
{'$cond': [
{'$size': "$order_remote.tries"},
"$order_remote.tries",
[]
]}
]}}
}},
{"$limit": limit},
{"$skip": self._rec_loaded},
{"$project": self.projection},
{"$sort": self.sort}
]
def _get_match_part(self):
"""
Returns different content for 'match' portion if filtering by
name is present
Returns:
(dict)
"""
if not self.filter:
return {
"type": "representation",
"_id": self._id
}
else:
regex_str = '.*{}.*'.format(self.filter)
return {
"type": "representation",
"_id": self._id,
'$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}]
}
def get_default_projection(self):
"""
Projection part for aggregate query.
All fields with '1' will be returned, no others.
Returns:
(dict)
"""
return {
"files": 1,
'progress_remote': 1,
'progress_local': 1,
'updated_dt_remote': 1,
'updated_dt_local': 1,
'failed_remote_error': 1,
'failed_local_error': 1,
'tries': 1,
'status': {
'$switch': {
'branches': [
{
'case': {
'$or': [{'$eq': ['$progress_remote', 0]},
{'$eq': ['$progress_local', 0]}]},
'then': 2 # Queued
},
{
'case': {
'$or': ['$failed_remote', '$failed_local']},
'then': 1 # Failed
},
{
'case': {'$or': [{'$and': [
{'$gt': ['$progress_remote', 0]},
{'$lt': ['$progress_remote', 1]}
]},
{'$and': [
{'$gt': ['$progress_local', 0]},
{'$lt': ['$progress_local', 1]}
]}
]},
'then': 0 # In Progress
},
{
'case': {'$eq': ['dummy_placeholder', 'paused']},
'then': 3
},
{
'case': {'$and': [
{'$eq': ['$progress_remote', 1]},
{'$eq': ['$progress_local', 1]}
]},
'then': 4 # Synced OK
},
],
'default': -1
}
}
}
class ImageDelegate(QtWidgets.QStyledItemDelegate):
"""
Prints icon of site and progress of synchronization
"""
def __init__(self, parent=None):
super(ImageDelegate, self).__init__(parent)
self.icons = {}
def paint(self, painter, option, index):
option = QtWidgets.QStyleOptionViewItem(option)
option.showDecorationSelected = True
if (option.showDecorationSelected and
(option.state & QtWidgets.QStyle.State_Selected)):
painter.setOpacity(0.20) # highlight color is a bit off
painter.fillRect(option.rect,
option.palette.highlight())
painter.setOpacity(1)
d = index.data(QtCore.Qt.DisplayRole)
if d:
provider, value = d.split()
else:
return
if not self.icons.get(provider):
resource_path = os.path.dirname(__file__)
resource_path = os.path.join(resource_path, "..",
"providers", "resources")
pix_url = "{}/{}.png".format(resource_path, provider)
pixmap = QtGui.QPixmap(pix_url)
self.icons[provider] = pixmap
else:
pixmap = self.icons[provider]
point = QtCore.QPoint(option.rect.x() +
(option.rect.width() - pixmap.width()) / 2,
option.rect.y() +
(option.rect.height() - pixmap.height()) / 2)
painter.drawPixmap(point, pixmap)
painter.setOpacity(0.5)
overlay_rect = option.rect
overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value)))
painter.fillRect(overlay_rect,
QtGui.QBrush(QtGui.QColor(0, 0, 0, 200)))
painter.setOpacity(1)
class SyncRepresentationErrorWindow(QtWidgets.QDialog):
def __init__(self, _id, project, dt, tries, msg, parent=None):
super(SyncRepresentationErrorWindow, self).__init__(parent)
self.setWindowFlags(QtCore.Qt.Window)
self.setFocusPolicy(QtCore.Qt.StrongFocus)
self.setStyleSheet(style.load_stylesheet())
self.setWindowIcon(QtGui.QIcon(style.app_icon_path()))
self.resize(250, 200)
body = QtWidgets.QWidget()
footer = QtWidgets.QWidget()
footer.setFixedHeight(20)
container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg,
parent=self)
body_layout = QtWidgets.QHBoxLayout(body)
body_layout.addWidget(container)
body_layout.setContentsMargins(0, 0, 0, 0)
message = QtWidgets.QLabel()
message.hide()
footer_layout = QtWidgets.QVBoxLayout(footer)
footer_layout.addWidget(message)
footer_layout.setContentsMargins(0, 0, 0, 0)
layout = QtWidgets.QVBoxLayout(self)
layout.addWidget(body)
layout.addWidget(footer)
self.setLayout(body_layout)
self.setWindowTitle("Sync Representation Error Detail")
class SyncRepresentationErrorWidget(QtWidgets.QWidget):
"""
Dialog to show when sync error happened, prints error message
"""
def __init__(self, _id, project, dt, tries, msg, parent=None):
super(SyncRepresentationErrorWidget, self).__init__(parent)
layout = QtWidgets.QFormLayout(self)
layout.addRow(QtWidgets.QLabel("Last update date"),
QtWidgets.QLabel(pretty_timestamp(dt)))
layout.addRow(QtWidgets.QLabel("Retries"),
QtWidgets.QLabel(str(tries)))
layout.addRow(QtWidgets.QLabel("Error message"),
QtWidgets.QLabel(msg))
class SizeDelegate(QtWidgets.QStyledItemDelegate):
"""
Pretty print for file size
"""
def __init__(self, parent=None):
super(SizeDelegate, self).__init__(parent)
def displayText(self, value, locale):
if value is None:
# Ignore None value
return
return self._pretty_size(value)
def _pretty_size(self, value, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(value) < 1024.0:
return "%3.1f%s%s" % (value, unit, suffix)
value /= 1024.0
return "%.1f%s%s" % (value, 'Yi', suffix)