from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from pype.tools.settings.settings.widgets.base import ProjectListWidget import attr import os from pype.tools.settings.settings import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger import json log = PypeLogger().get_logger("SyncServer") STATUS = { 0: 'In Progress', 1: 'Failed', 2: 'Queued', 3: 'Paused', 4: 'Synced OK', -1: 'Not available' } class SyncServerWindow(QtWidgets.QDialog): """ Main window that contains list of synchronizable projects and summary view with all synchronizable representations for first project """ def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1400, 800) body = QtWidgets.QWidget(self) footer = QtWidgets.QWidget(self) footer.setFixedHeight(20) container = QtWidgets.QWidget() projects = SyncProjectListWidget(sync_server, self) projects.refresh() # force selection of default repres = SyncRepresentationWidget(sync_server, project=projects.current_project, parent=self) container_layout = QtWidgets.QHBoxLayout(container) container_layout.setContentsMargins(0, 0, 0, 0) split = QtWidgets.QSplitter() split.addWidget(projects) split.addWidget(repres) split.setSizes([180, 950, 200]) container_layout.addWidget(split) container.setLayout(container_layout) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) message = QtWidgets.QLabel(footer) message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) footer_layout.addWidget(message) footer_layout.setContentsMargins(0, 0, 0, 0) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(body) layout.addWidget(footer) self.setLayout(body_layout) self.setWindowTitle("Sync Server") projects.project_changed.connect( lambda: repres.table_view.model().set_project( projects.current_project)) class SyncProjectListWidget(ProjectListWidget): """ Lists all projects that are synchronized to choose from """ def __init__(self, sync_server, parent): super(SyncProjectListWidget, self).__init__(parent) self.sync_server = sync_server def validate_context_change(self): return True def refresh(self): model = self.project_list.model() model.clear() for project_name in self.sync_server.get_synced_presets().keys(): model.appendRow(QtGui.QStandardItem(project_name)) if len(self.sync_server.get_synced_presets().keys()) == 0: model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( QtCore.Qt.DisplayRole ) if not self.current_project: self.current_project = self.project_list.model().item(0). \ data(QtCore.Qt.DisplayRole) class SyncRepresentationWidget(QtWidgets.QWidget): """ Summary dialog with list of representations that matches current settings 'local_site' and 'remote_site'. """ active_changed = QtCore.Signal() # active index changed default_widths = ( ("asset", 210), ("subset", 190), ("version", 10), ("representation", 90), ("created_dt", 100), ("sync_dt", 100), ("local_site", 60), ("remote_site", 70), ("files_count", 70), ("files_size", 70), ("priority", 20), ("state", 50) ) def __init__(self, sync_server, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) self.sync_server = sync_server self._selected_id = None # keep last selected _id self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") top_bar_layout = QtWidgets.QHBoxLayout() top_bar_layout.addWidget(self.filter) self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] model = SyncRepresentationModel(sync_server, headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( -1, Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) column = self.table_view.model().get_header_index("files_size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) layout.addWidget(self.table_view) self.table_view.doubleClicked.connect(self._double_clicked) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) self.table_view.model().modelReset.connect(self._set_selection) self.selection_model = self.table_view.selectionModel() self.selection_model.selectionChanged.connect(self._selection_changed) def _selection_changed(self, new_selection): index = self.selection_model.currentIndex() self._selected_id = self.table_view.model().data(index, Qt.UserRole) def _set_selection(self): """ Sets selection to 'self._selected_id' if exists. Keep selection during model refresh. """ if self._selected_id: index = self.table_view.model().get_index(self._selected_id) if index and index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None def _double_clicked(self, index): """ Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) detail_window = SyncServerDetailWindow( self.sync_server, _id, self.table_view.model()._project) detail_window.exec() def _on_context_menu(self, point): """ Shows menu with loader actions on Right-click. """ point_index = self.table_view.indexAt(point) if not point_index.isValid(): return class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 19 REFRESH_SEC = 5000 DEFAULT_SORT = { "updated_dt_remote": -1, "_id": 1 } SORT_BY_COLUMN = [ "context.asset", # asset "context.subset", # subset "context.version", # version "context.representation", # representation "updated_dt_local", # local created_dt "updated_dt_remote", # remote created_dt "avg_progress_local", # local progress "avg_progress_remote", # remote progress "files_count", # count of files "files_size", # file size of all files "context.asset", # priority TODO "status" # state ] numberPopulated = QtCore.Signal(int) @attr.s class SyncRepresentation: """ Auxiliary object for easier handling. Fields must contain all header values (+ any arbitrary values). """ _id = attr.ib() asset = attr.ib() subset = attr.ib() version = attr.ib() representation = attr.ib() created_dt = attr.ib(default=None) sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) files_count = attr.ib(default=None) files_size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) def __init__(self, sync_server, header, project=None): super(SyncRepresentationModel, self).__init__() self._header = header self._data = [] self._project = project self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) self.filter = None self._initialized = False self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ self.sync_server.get_sites_for_project(self._project) self.projection = self.get_default_projection() self.sort = self.DEFAULT_SORT self.query = self.get_default_query() self.default_query = list(self.get_default_query()) log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) self.timer = QtCore.QTimer() self.timer.timeout.connect(self.tick) self.timer.start(self.REFRESH_SEC) @property def dbcon(self): return self.sync_server.connection.database[self._project] def data(self, index, role): item = self._data[index.row()] if role == Qt.DisplayRole: return attr.asdict(item)[self._header[index.column()]] if role == Qt.UserRole: return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return str(self._header[section]) def tick(self): self.refresh(representations=None, load_records=self._rec_loaded) self.timer.start(self.REFRESH_SEC) def get_header_index(self, value): """ Returns index of 'value' in headers Args: value (str): header name value Returns: (int) """ return self._header.index(value) def refresh(self, representations=None, load_records=0): self.beginResetModel() self._data = [] self._rec_loaded = 0 if not representations: self.query = self.get_default_query(load_records) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) self.endResetModel() def _add_page_records(self, local_site, remote_site, representations): for repre in representations: context = repre.get("context").pop() files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] # representation without files doesnt concern us if not files: continue local_updated = remote_updated = None if repre.get('updated_dt_local'): local_updated = \ repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") if repre.get('updated_dt_remote'): remote_updated = \ repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") avg_progress_remote = repre.get('avg_progress_remote', '') avg_progress_local = repre.get('avg_progress_local', '') item = self.SyncRepresentation( repre.get("_id"), context.get("asset"), context.get("subset"), "v{:0>3d}".format(context.get("version", 1)), context.get("representation"), local_updated, remote_updated, '{} {}'.format(local_site, avg_progress_local), '{} {}'.format(remote_site, avg_progress_remote), repre.get("files_count", 1), repre.get("files_size", 0), 1, STATUS[repre.get("status", -1)] ) self._data.append(item) self._rec_loaded += 1 def canFetchMore(self, index): """ Check if there are more records than currently loaded """ # 'skip' might be suboptimal when representation hits 500k+ self._buffer = list(self.dbcon.aggregate(self.query)) # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( # self._rec_loaded, len(self._buffer))) return len(self._buffer) > self._rec_loaded def fetchMore(self, index): """ Add more record to model. Called when 'canFetchMore' returns true, which means there are more records in DB than loaded. 'self._buffer' is used to stash cursor to limit requery """ log.debug("fetchMore") # cursor.count() returns always total number, not only skipped + limit remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) self._add_page_records(self.local_site, self.remote_site, self._buffer) self.endInsertRows() self.numberPopulated.emit(items_to_fetch) # ?? def sort(self, index, order): """ Summary sort per representation. Sort is happening on a DB side, model is reset, db queried again. Args: index (int): column index order (int): 0| """ # limit unwanted first re-sorting by view if index < 0: return self._rec_loaded = 0 if order == 0: order = 1 else: order = -1 self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} self.query = self.get_default_query() representations = self.dbcon.aggregate(self.query) self.refresh(representations) def set_filter(self, filter): """ Adds text value filtering Args: filter (str): string inputted by user """ self.filter = filter self.refresh() def set_project(self, project): """ Changes project, called after project selection is changed Args: project (str): name of project """ self._project = project self.refresh() def get_index(self, id): """ Get index of 'id' value. Used for keeping selection after refresh. Args: id (str): MongoDB _id Returns: (QModelIndex) """ index = None for i in range(self.rowCount(None)): index = self.index(i, 0) value = self.data(index, Qt.UserRole) if value == id: return index return index def get_default_query(self, limit=0): """ Returns basic aggregate query for main table. Main table provides summary information about representation, which could have multiple files. Details are accessible after double click on representation row. Columns: 'created_dt' - max of created or updated (when failed) per repr 'sync_dt' - same for remote side 'local_site' - progress of repr on local side, 1 = finished 'remote_site' - progress on remote side, calculates from files 'state' - 0 - in progress 1 - failed 2 - queued 3 - paused (not implemented yet) 4 - finished on both sides are calculated and must be calculated in DB because of pagination Args: limit (int): how many records should be returned, by default it 'PAGE_SIZE' for performance. Should be overridden by value of loaded records for refresh functionality (got more records by scrolling, refresh shouldn't reset that) """ if limit == 0: limit = SyncRepresentationModel.PAGE_SIZE return [ {"$match": self._get_match_part()}, {'$unwind': '$files'}, # merge potentially unwinded records back to single per repre {'$addFields': { 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} }}, 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} }}, {'$addFields': { # prepare progress per file, presence of 'created_dt' denotes # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, "$order_remote.progress", {'$cond': [ {'$size': "$order_remote.created_dt"}, [1], [0] ]} ]}}, 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, "$order_local.progress", {'$cond': [ {'$size': "$order_local.created_dt"}, [1], [0] ]} ]}}, # file might be successfully created or failed, not both 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, "$order_remote.created_dt", {'$cond': [ {'$size': "$order_remote.last_failed_dt"}, "$order_remote.last_failed_dt", [] ]} ]}}, 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", {'$cond': [ {'$size': "$order_local.last_failed_dt"}, "$order_local.last_failed_dt", [] ]} ]}}, 'files_size': {'$ifNull': ["$files.size", 0]}, 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]}, 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} }}, {'$group': { '_id': '$_id', # pass through context - same for representation 'context': {'$addToSet': '$context'}, # pass through files as a list 'files': {'$addToSet': '$files'}, # count how many files 'files_count': {'$sum': 1}, 'files_size': {'$sum': '$files_size'}, # sum avg progress, finished = 1 'avg_progress_remote': {'$avg': "$progress_remote"}, 'avg_progress_local': {'$avg': "$progress_local"}, # select last touch of file 'updated_dt_remote': {'$max': "$updated_dt_remote"}, 'failed_remote': {'$sum': '$failed_remote'}, 'failed_local': {'$sum': '$failed_local'}, 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, {"$project": self.projection}, {"$sort": self.sort} ] def _get_match_part(self): """ Extend match part with filter if present. Filter is set by user input. Each model has different fields to be checked. If performance issues are found, '$text' and text indexes should be investigated. """ if not self.filter: return { "type": "representation", 'files.sites': { '$elemMatch': { '$or': [ {'name': self.local_site}, {'name': self.remote_site} ] } } } else: regex_str = '.*{}.*'.format(self.filter) return { "type": "representation", '$or': [ {'context.subset': {'$regex': regex_str, '$options': 'i'}}, {'context.asset': {'$regex': regex_str, '$options': 'i'}}, {'context.representation': {'$regex': regex_str, '$options': 'i'}}], 'files.sites': { '$elemMatch': { '$or': [ {'name': self.local_site}, {'name': self.remote_site} ] } } } def get_default_projection(self): """ Projection part for aggregate query. All fields with '1' will be returned, no others. Returns: (dict) """ return { "context.subset": 1, "context.asset": 1, "context.version": 1, "context.representation": 1, "files": 1, 'files_count': 1, "files_size": 1, 'avg_progress_remote': 1, 'avg_progress_local': 1, 'updated_dt_remote': 1, 'updated_dt_local': 1, 'status': { '$switch': { 'branches': [ { 'case': { '$or': [{'$eq': ['$avg_progress_remote', 0]}, {'$eq': ['$avg_progress_local', 0]}]}, 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ {'$gt': ['$avg_progress_remote', 0]}, {'$lt': ['$avg_progress_remote', 1]} ]}, {'$and': [ {'$gt': ['$avg_progress_local', 0]}, {'$lt': ['$avg_progress_local', 1]} ]} ]}, 'then': 0 # In progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, 'then': 3 # Paused }, { 'case': {'$and': [ {'$eq': ['$avg_progress_remote', 1]}, {'$eq': ['$avg_progress_local', 1]} ]}, 'then': 4 # Synced OK }, ], 'default': -1 } } } class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, sync_server, _id, project, parent=None): log.debug( "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1000, 400) body = QtWidgets.QWidget() footer = QtWidgets.QWidget() footer.setFixedHeight(20) container = SyncRepresentationDetailWidget(sync_server, _id, project, parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) message = QtWidgets.QLabel() message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) footer_layout.addWidget(message) footer_layout.setContentsMargins(0, 0, 0, 0) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(body) layout.addWidget(footer) self.setLayout(body_layout) self.setWindowTitle("Sync Representation Detail") class SyncRepresentationDetailWidget(QtWidgets.QWidget): """ Widget to display list of synchronizable files for single repre. Args: _id (str): representation _id project (str): name of project with repre parent (QDialog): SyncServerDetailWindow """ active_changed = QtCore.Signal() # active index changed default_widths = ( ("file", 290), ("created_dt", 120), ("sync_dt", 120), ("local_site", 60), ("remote_site", 60), ("size", 60), ("priority", 20), ("state", 90) ) def __init__(self, sync_server, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) self.representation_id = _id self.item = None # set to item that mouse was clicked over self.sync_server = sync_server self._selected_id = None self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") top_bar_layout = QtWidgets.QHBoxLayout() top_bar_layout.addWidget(self.filter) self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] model = SyncRepresentationDetailModel(sync_server, headers, _id, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QTableView.SelectRows) self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) column = self.table_view.model().get_header_index("size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) layout.addWidget(self.table_view) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) self.table_view.model().modelReset.connect(self._set_selection) self.selection_model = self.table_view.selectionModel() self.selection_model.selectionChanged.connect(self._selection_changed) def _selection_changed(self): index = self.selection_model.currentIndex() self._selected_id = self.table_view.model().data(index, Qt.UserRole) def _set_selection(self): """ Sets selection to 'self._selected_id' if exists. Keep selection during model refresh. """ if self._selected_id: index = self.table_view.model().get_index(self._selected_id) if index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None def _show_detail(self): """ Shows windows with error message for failed sync of a file. """ dt = max(self.item.created_dt, self.item.sync_dt) detail_window = SyncRepresentationErrorWindow(self.item._id, self.project, dt, self.item.tries, self.item.error) detail_window.exec() def _on_context_menu(self, point): """ Shows menu with loader actions on Right-click. """ point_index = self.table_view.indexAt(point) if not point_index.isValid(): return self.item = self.table_view.model()._data[point_index.row()] menu = QtWidgets.QMenu() actions_mapping = {} if self.item.state == STATUS[1]: action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail menu.addAction(action) remote_site, remote_progress = self.item.remote_site.split() if remote_progress == '1': action = QtWidgets.QAction("Reset local site") actions_mapping[action] = self._reset_local_site menu.addAction(action) local_site, local_progress = self.item.local_site.split() if local_progress == '1': action = QtWidgets.QAction("Reset remote site") actions_mapping[action] = self._reset_remote_site menu.addAction(action) if not actions_mapping: action = QtWidgets.QAction("< No action >") actions_mapping[action] = None menu.addAction(action) result = menu.exec_(QtGui.QCursor.pos()) if result: to_run = actions_mapping[result] if to_run: to_run() def _reset_local_site(self): """ Removes errors or success metadata for particular file >> forces redo of upload/download """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, self.item._id, 'local') def _reset_remote_site(self): """ Removes errors or success metadata for particular file >> forces redo of upload/download """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, self.item._id, 'remote') class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ List of all syncronizable files per single representation. """ PAGE_SIZE = 30 # TODO add filter filename DEFAULT_SORT = { "files.path": 1 } SORT_BY_COLUMN = [ "files.path", "updated_dt_local", # local created_dt "updated_dt_remote", # remote created_dt "progress_local", # local progress "progress_remote", # remote progress "size", # remote progress "context.asset", # priority TODO "status" # state ] @attr.s class SyncRepresentationDetail: """ Auxiliary object for easier handling. Fields must contain all header values (+ any arbitrary values). """ _id = attr.ib() file = attr.ib() created_dt = attr.ib(default=None) sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) tries = attr.ib(default=None) error = attr.ib(default=None) def __init__(self, sync_server, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() self._header = header self._data = [] self._project = project self._rec_loaded = 0 self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id self._initialized = False self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ self.sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT # in case we would like to hide/show some columns self.projection = self.get_default_projection() self.query = self.get_default_query() representations = self.dbcon.aggregate(self.query) self.refresh(representations) self.timer = QtCore.QTimer() self.timer.timeout.connect(self.tick) self.timer.start(SyncRepresentationModel.REFRESH_SEC) @property def dbcon(self): return self.sync_server.connection.database[self._project] def tick(self): self.refresh(representations=None, load_records=self._rec_loaded) self.timer.start(SyncRepresentationModel.REFRESH_SEC) def get_header_index(self, value): """ Returns index of 'value' in headers Args: value (str): header name value Returns: (int) """ return self._header.index(value) def data(self, index, role): item = self._data[index.row()] if role == Qt.DisplayRole: return attr.asdict(item)[self._header[index.column()]] if role == Qt.UserRole: return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return str(self._header[section]) def refresh(self, representations=None, load_records=0): self.beginResetModel() self._data = [] self._rec_loaded = 0 if not representations: self.query = self.get_default_query(load_records) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) self.endResetModel() def _add_page_records(self, local_site, remote_site, representations): """ Process all records from 'representation' and add them to storage. Args: local_site (str): name of local site (mine) remote_site (str): name of cloud provider (theirs) representations (Mongo Cursor) """ for repre in representations: # log.info("!!! repre:: {}".format(repre)) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] for file in files: local_updated = remote_updated = None if repre.get('updated_dt_local'): local_updated = \ repre.get('updated_dt_local').strftime( "%Y%m%dT%H%M%SZ") if repre.get('updated_dt_remote'): remote_updated = \ repre.get('updated_dt_remote').strftime( "%Y%m%dT%H%M%SZ") progress_remote = repre.get('progress_remote', '') progress_local = repre.get('progress_local', '') errors = [] if repre.get('failed_remote_error'): errors.append(repre.get('failed_remote_error')) if repre.get('failed_local_error'): errors.append(repre.get('failed_local_error')) item = self.SyncRepresentationDetail( file.get("_id"), os.path.basename(file["path"]), local_updated, remote_updated, '{} {}'.format(local_site, progress_local), '{} {}'.format(remote_site, progress_remote), file.get('size', 0), 1, STATUS[repre.get("status", -1)], repre.get("tries"), '\n'.join(errors) ) self._data.append(item) self._rec_loaded += 1 def canFetchMore(self, index): """ Check if there are more records than currently loaded """ # 'skip' might be suboptimal when representation hits 500k+ self._buffer = list(self.dbcon.aggregate(self.query)) return len(self._buffer) > self._rec_loaded def fetchMore(self, index): """ Add more record to model. Called when 'canFetchMore' returns true, which means there are more records in DB than loaded. 'self._buffer' is used to stash cursor to limit requery """ log.debug("fetchMore") # cursor.count() returns always total number, not only skipped + limit remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) self._add_page_records(self.local_site, self.remote_site, self._buffer) self.endInsertRows() def sort(self, index, order): # limit unwanted first re-sorting by view if index < 0: return self._rec_loaded = 0 # change sort - reset from start if order == 0: order = 1 else: order = -1 self.sort = {self.SORT_BY_COLUMN[index]: order} self.query = self.get_default_query() representations = self.dbcon.aggregate(self.query) self.refresh(representations) def set_filter(self, filter): self.filter = filter self.refresh() def get_index(self, id): """ Get index of 'id' value. Used for keeping selection after refresh. Args: id (str): MongoDB _id Returns: (QModelIndex) """ index = None for i in range(self.rowCount(None)): index = self.index(i, 0) value = self.data(index, Qt.UserRole) if value == id: return index return index def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. Called for basic table view. Returns: [(dict)] - list with single dict - appropriate for aggregate function for MongoDB """ if limit == 0: limit = SyncRepresentationModel.PAGE_SIZE return [ {"$match": self._get_match_part()}, {"$unwind": "$files"}, {'$addFields': { 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} }}, 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} }}, {'$addFields': { # prepare progress per file, presence of 'created_dt' denotes # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, "$order_remote.progress", {'$cond': [ {'$size': "$order_remote.created_dt"}, [1], [0] ]} ]}}, 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, "$order_local.progress", {'$cond': [ {'$size': "$order_local.created_dt"}, [1], [0] ]} ]}}, # file might be successfully created or failed, not both 'updated_dt_remote': {'$first': { '$cond': [ {'$size': "$order_remote.created_dt"}, "$order_remote.created_dt", { '$cond': [ {'$size': "$order_remote.last_failed_dt"}, "$order_remote.last_failed_dt", [] ] } ] }}, 'updated_dt_local': {'$first': { '$cond': [ {'$size': "$order_local.created_dt"}, "$order_local.created_dt", { '$cond': [ {'$size': "$order_local.last_failed_dt"}, "$order_local.last_failed_dt", [] ] } ] }}, 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]}, 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]}, 'failed_remote_error': {'$first': { '$cond': [{'$size': "$order_remote.error"}, "$order_remote.error", [""]]}}, 'failed_local_error': {'$first': { '$cond': [{'$size': "$order_local.error"}, "$order_local.error", [""]]}}, 'tries': {'$first': { '$cond': [{'$size': "$order_local.tries"}, "$order_local.tries", {'$cond': [ {'$size': "$order_remote.tries"}, "$order_remote.tries", [] ]} ]}} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, {"$project": self.projection}, {"$sort": self.sort} ] def _get_match_part(self): """ Returns different content for 'match' portion if filtering by name is present Returns: (dict) """ if not self.filter: return { "type": "representation", "_id": self._id } else: regex_str = '.*{}.*'.format(self.filter) return { "type": "representation", "_id": self._id, '$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}] } def get_default_projection(self): """ Projection part for aggregate query. All fields with '1' will be returned, no others. Returns: (dict) """ return { "files": 1, 'progress_remote': 1, 'progress_local': 1, 'updated_dt_remote': 1, 'updated_dt_local': 1, 'failed_remote_error': 1, 'failed_local_error': 1, 'tries': 1, 'status': { '$switch': { 'branches': [ { 'case': { '$or': [{'$eq': ['$progress_remote', 0]}, {'$eq': ['$progress_local', 0]}]}, 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ {'$gt': ['$progress_remote', 0]}, {'$lt': ['$progress_remote', 1]} ]}, {'$and': [ {'$gt': ['$progress_local', 0]}, {'$lt': ['$progress_local', 1]} ]} ]}, 'then': 0 # In Progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, 'then': 3 }, { 'case': {'$and': [ {'$eq': ['$progress_remote', 1]}, {'$eq': ['$progress_local', 1]} ]}, 'then': 4 # Synced OK }, ], 'default': -1 } } } class ImageDelegate(QtWidgets.QStyledItemDelegate): """ Prints icon of site and progress of synchronization """ def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) self.icons = {} def paint(self, painter, option, index): option = QtWidgets.QStyleOptionViewItem(option) option.showDecorationSelected = True if (option.showDecorationSelected and (option.state & QtWidgets.QStyle.State_Selected)): painter.setOpacity(0.20) # highlight color is a bit off painter.fillRect(option.rect, option.palette.highlight()) painter.setOpacity(1) d = index.data(QtCore.Qt.DisplayRole) if d: provider, value = d.split() else: return if not self.icons.get(provider): resource_path = os.path.dirname(__file__) resource_path = os.path.join(resource_path, "..", "providers", "resources") pix_url = "{}/{}.png".format(resource_path, provider) pixmap = QtGui.QPixmap(pix_url) self.icons[provider] = pixmap else: pixmap = self.icons[provider] point = QtCore.QPoint(option.rect.x() + (option.rect.width() - pixmap.width()) / 2, option.rect.y() + (option.rect.height() - pixmap.height()) / 2) painter.drawPixmap(point, pixmap) painter.setOpacity(0.5) overlay_rect = option.rect overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) painter.setOpacity(1) class SyncRepresentationErrorWindow(QtWidgets.QDialog): def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(250, 200) body = QtWidgets.QWidget() footer = QtWidgets.QWidget() footer.setFixedHeight(20) container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg, parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) message = QtWidgets.QLabel() message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) footer_layout.addWidget(message) footer_layout.setContentsMargins(0, 0, 0, 0) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(body) layout.addWidget(footer) self.setLayout(body_layout) self.setWindowTitle("Sync Representation Error Detail") class SyncRepresentationErrorWidget(QtWidgets.QWidget): """ Dialog to show when sync error happened, prints error message """ def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWidget, self).__init__(parent) layout = QtWidgets.QFormLayout(self) layout.addRow(QtWidgets.QLabel("Last update date"), QtWidgets.QLabel(pretty_timestamp(dt))) layout.addRow(QtWidgets.QLabel("Retries"), QtWidgets.QLabel(str(tries))) layout.addRow(QtWidgets.QLabel("Error message"), QtWidgets.QLabel(msg)) class SizeDelegate(QtWidgets.QStyledItemDelegate): """ Pretty print for file size """ def __init__(self, parent=None): super(SizeDelegate, self).__init__(parent) def displayText(self, value, locale): if value is None: # Ignore None value return return self._pretty_size(value) def _pretty_size(self, value, suffix='B'): for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: if abs(value) < 1024.0: return "%3.1f%s%s" % (value, unit, suffix) value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix)