diff --git a/pype/modules/sync_server/providers/resources/gdrive.png b/pype/modules/sync_server/providers/resources/gdrive.png new file mode 100644 index 0000000000..e6c9131454 Binary files /dev/null and b/pype/modules/sync_server/providers/resources/gdrive.png differ diff --git a/pype/modules/sync_server/providers/resources/studio.png b/pype/modules/sync_server/providers/resources/studio.png new file mode 100644 index 0000000000..e95e9762f8 Binary files /dev/null and b/pype/modules/sync_server/providers/resources/studio.png differ diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 10967a07f9..aa52b06da9 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,3 @@ -import sys - -sys.path.append( - 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') - from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from avalon import style @@ -12,7 +7,7 @@ from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style -from avalon.tools.delegates import PrettyTimeDelegate +from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -29,7 +24,12 @@ STATUS = { -1: 'Not available' } + class SyncServerWindow(QtWidgets.QDialog): + """ + Main window that contains list of synchronizable projects and summary + view with all synchronizable representations for first project + """ def __init__(self, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) @@ -76,42 +76,44 @@ class SyncServerWindow(QtWidgets.QDialog): self.setLayout(body_layout) self.setWindowTitle("Sync Server") + projects.project_changed.connect( + lambda: repres.table_view.model().set_project( + projects.current_project)) + class SyncProjectListWidget(ProjectListWidget): """ - Lists all projects that are syncronized to choose from + Lists all projects that are synchronized to choose from """ def validate_context_change(self): return True def refresh(self): - selected_project = None - for index in self.project_list.selectedIndexes(): - selected_project = index.data(QtCore.Qt.DisplayRole) - break - model = self.project_list.model() model.clear() - items = [] manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] for project_name in sync_server.get_synced_presets().keys(): - items.append(project_name) + model.appendRow(QtGui.QStandardItem(project_name)) - sync_server.log.debug("ld !!!! items:: {}".format(items)) - for item in items: - model.appendRow(QtGui.QStandardItem(item)) - - # self.select_project(selected_project) + if len(sync_server.get_synced_presets().keys()) == 0: + model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( QtCore.Qt.DisplayRole ) + if not self.current_project: + self.current_project = self.project_list.model().item(0).\ + data(QtCore.Qt.DisplayRole) class SyncRepresentationWidget(QtWidgets.QWidget): + """ + Summary dialog with list of representations that matches current + settings 'local_site' and 'remote_site'. + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -139,16 +141,14 @@ class SyncRepresentationWidget(QtWidgets.QWidget): top_bar_layout = QtWidgets.QHBoxLayout() top_bar_layout.addWidget(self.filter) - # TODO ? TreeViewSpinner - self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! headers:: {}".format(headers)) - model = SyncRepresentationModel(headers) + + model = SyncRepresentationModel(headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - # self.table_view.setSelectionMode( - # QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( @@ -158,26 +158,25 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("files_size") + column = self.table_view.model().get_header_index("files_size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -203,17 +202,19 @@ class SyncRepresentationWidget(QtWidgets.QWidget): """ Shows menu with loader actions on Right-click. """ - point_index = self.view.indexAt(point) + point_index = self.table_view.indexAt(point) if not point_index.isValid(): return class SyncRepresentationModel(QtCore.QAbstractTableModel): - PAGE_SIZE = 30 + PAGE_SIZE = 19 + REFRESH_SEC = 5000 DEFAULT_SORT = { "context.asset": 1, "context.subset": 1, "context.version": 1, + "_id": 1 } SORT_BY_COLUMN = [ "context.asset", # asset @@ -229,9 +230,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset", # priority TODO "status" # state ] - DEFAULT_QUERY = { - "type": "representation", - } numberPopulated = QtCore.Signal(int) @@ -269,18 +267,356 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) - self.query = self.DEFAULT_QUERY + self.projection = self.get_default_projection() - self.projection = { + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(json.dumps(self.query, + indent=4))) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(self.REFRESH_SEC) + + def data(self, index, role): + item = self._data[index.row()] + + if role == Qt.DisplayRole: + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._header) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(self.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + + def refresh(self, representations=None, load_records=0): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + + if not representations: + self.query = self.get_default_query(load_records) + representations = self.dbcon.aggregate(self.query) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + for repre in representations: + context = repre.get("context").pop() + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + + # representation without files doesnt concern us + if not files: + continue + + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") + + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") + + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') + + item = self.SyncRepresentation( + repre.get("_id"), + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), + 1, + STATUS[repre.get("status", -1)] + ) + + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( + # self._rec_loaded, len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + """ + Summary sort per representation. + + Sort is happening on a DB side, model is reset, db queried + again. + + Args: + index (int): column index + order (int): 0| + """ + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 + if order == 0: + order = 1 + else: + order = -1 + + self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} + self.query = self.get_default_query() + + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def set_filter(self, filter): + """ + Adds text value filtering + + Args: + filter (str): string inputted by user + """ + self.filter = filter + self.refresh() + + def set_project(self, project): + """ + Changes project, called after project selection is changed + + Args: + project (str): name of project + """ + self._project = project + self.dbcon.Session["AVALON_PROJECT"] = self._project + self.refresh() + + def get_default_query(self, limit=0): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - queued + 1 - failed + 2 - paused (not implemented yet) + 3 - in progress + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + + Args: + limit (int): how many records should be returned, by default + it 'PAGE_SIZE' for performance. + Should be overridden by value of loaded records for refresh + functionality (got more records by scrolling, refresh + shouldn't reset that) + """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + + return [ + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'files_size': {'$ifNull': ["$files.size", 0]} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + }}, + {'$group': { + '_id': '$_id' + # pass through context - same for representation + , 'context': {'$addToSet': '$context'} + # pass through files as a list + , 'files': {'$addToSet': '$files'} + # count how many files + , 'files_count': {'$sum': 1} + , 'files_size': {'$sum': '$files_size'} + # sum avg progress, finished = 1 + , 'avg_progress_remote': {'$avg': "$progress_remote"} + , 'avg_progress_local': {'$avg': "$progress_local"} + # select last touch of file + , 'updated_dt_remote': {'$max': "$updated_dt_remote"} + , 'failed_remote': {'$sum': '$failed_remote'} + , 'failed_local': {'$sum': '$failed_local'} + , 'updated_dt_local': {'$max': "$updated_dt_local"} + }}, + {"$sort": self.sort}, + {"$limit": limit}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [{'context.subset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { "context.subset": 1, "context.asset": 1, "context.version": 1, @@ -335,297 +671,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): } } - self.sort = self.DEFAULT_SORT - - self.query = self.get_default_query() - self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def data(self, index, role): - item = self._data[index.row()] - - if role == Qt.DisplayRole: - return attr.asdict(item)[self._header[index.column()]] - if role == Qt.UserRole: - return item._id - - def rowCount(self, index): - return len(self._data) - - def columnCount(self, index): - return len(self._header) - - def headerData(self, section, orientation, role): - if role == Qt.DisplayRole: - if orientation == Qt.Horizontal: - return str(self._header[section]) - - def refresh(self, representations=None): - self.beginResetModel() - self._data = [] - self._rec_loaded = 0 - log.debug("!!! refresh sort {}".format(self.sort)) - if not representations: - self.query = self.get_default_query() - log.debug( - "!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - - self._add_page_records(self.local_site, self.remote_site, - representations) - self.endResetModel() - - def _add_page_records(self, local_site, remote_site, representations): - log.debug("!!! representations:: {}".format(representations)) - #log.debug("!!! representations:: {}".format(len(representations))) - for repre in representations: - context = repre.get("context").pop() - # log.debug("!!! context:: {}".format(context)) - # log.info("!!! repre:: {}".format(repre)) - # log.debug("!!! repre:: {}".format(type(repre))) - created = {} - # log.debug("!!! files:: {}".format(repre.get("files", []))) - # log.debug("!!! files:: {}".format(type(repre.get("files", [])))) - files = repre.get("files", []) - if isinstance(files, dict): # aggregate returns dictionary - files = [files] - - # representation without files doesnt concern us - if not files: - continue - - local_updated = remote_updated = None - if repre.get('updated_dt_local'): - local_updated = \ - repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") - - if repre.get('updated_dt_remote'): - remote_updated = \ - repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - - avg_progress_remote = repre.get('avg_progress_remote', '') - avg_progress_local = repre.get('avg_progress_local', '') - - item = self.SyncRepresentation( - repre.get("_id"), - context.get("asset"), - context.get("subset"), - "v{:0>3d}".format(context.get("version", 1)), - context.get("representation"), - local_updated, - remote_updated, - '{} {}'.format(local_site, avg_progress_local), - '{} {}'.format(remote_site, avg_progress_remote), - repre.get("files_count", 1), - repre.get("files_size", 0), - 1, - STATUS[repre.get("status", -1)] - ) - - self._data.append(item) - self._rec_loaded += 1 - - def canFetchMore(self, index): - """ - Check if there are more records than currently loaded - """ - log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) - # 'skip' might be suboptimal when representation hits 500k+ - self._buffer = list(self.dbcon.aggregate(self.query)) - # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) - return len(self._buffer) > self._rec_loaded - - def fetchMore(self, index): - """ - Add more record to model. - - Called when 'canFetchMore' returns true, which means there are - more records in DB than loaded. - 'self._buffer' is used to stash cursor to limit requery - """ - log.debug("fetchMore") - # cursor.count() returns always total number, not only skipped + limit - remainder = len(self._buffer) - self._rec_loaded - items_to_fetch = min(self.PAGE_SIZE, remainder) - self.beginInsertRows(index, - self._rec_loaded, - self._rec_loaded + items_to_fetch - 1) - - self._add_page_records(self.local_site, self.remote_site, self._buffer) - - self.endInsertRows() - - self.numberPopulated.emit(items_to_fetch) # ?? - - def sort(self, index, order): - """ - Summary sort per representation - - Args: - index (int): column index - order (int): 0| - """ - # limit unwanted first re-sorting by view - if index < 0: - return - - self._rec_loaded = 0 - if order == 0: - order = 1 - else: - order = -1 - - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - - log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def set_filter(self, filter): - self.filter = filter - self.refresh() - - def get_default_query(self): - """ - Returns basic aggregate query for main table. - - Main table provides summary information about representation, - which could have multiple files. Details are accessible after - double click on representation row. - Columns: - 'created_dt' - max of created or updated (when failed) per repr - 'sync_dt' - same for remote side - 'local_site' - progress of repr on local side, 1 = finished - 'remote_site' - progress on remote side, calculates from files - 'state' - - 0 - queued - 1 - failed - 2 - paused (not implemented yet) - 3 - in progress - 4 - finished on both sides - - are calculated and must be calculated in DB because of - pagination - """ - return [ - {"$match": self._get_match_part()}, - {'$unwind': '$files'}, - # merge potentially unwinded records back to single per repre - {'$addFields': { - 'order_remote': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.local_site]} - }} - }}, - {'$addFields': { - # prepare progress per file, presence of 'created_dt' denotes - # successfully finished load/download - 'progress_remote': {'$first': { - '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} - , 'progress_local': {'$first': { - '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} - # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { - '$cond': [{'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - }]}} - , 'updated_dt_local': {'$first': { - '$cond': [{'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - }]}} - , 'files_size': {'$ifNull': ["$files.size", 0]} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} - }}, - {'$group': { - '_id': '$_id' - # pass through context - same for representation - , 'context': {'$addToSet': '$context'} - # pass through files as a list - , 'files': {'$addToSet': '$files'} - # count how many files - , 'files_count': {'$sum': 1} - , 'files_size': {'$sum': '$files_size'} - # sum avg progress, finished = 1 - , 'avg_progress_remote': {'$avg': "$progress_remote"} - , 'avg_progress_local': {'$avg': "$progress_local"} - # select last touch of file - , 'updated_dt_remote': {'$max': "$updated_dt_remote"} - , 'failed_remote': {'$sum': '$failed_remote'} - , 'failed_local': {'$sum': '$failed_local'} - , 'updated_dt_local': {'$max': "$updated_dt_local"} - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] - - def _get_match_part(self): - """ - Extend match part with filter if present. - - Filter is set by user input. Each model has different fields to be - checked. - If performance issues are found, '$text' and text indexes should - be investigated. - """ - if not self.filter: - return { - "type": "representation", - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - else: - regex_str = '.*{}.*'.format(self.filter) - return { - "type": "representation", - '$or': [{'context.subset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.asset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, _id, project, parent=None): @@ -668,6 +713,14 @@ class SyncServerDetailWindow(QtWidgets.QDialog): class SyncRepresentationDetailWidget(QtWidgets.QWidget): + """ + Widget to display list of synchronizable files for single repre. + + Args: + _id (str): representation _id + project (str): name of project with repre + parent (QDialog): SyncServerDetailWindow + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -683,8 +736,14 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def __init__(self, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) - log.debug( - "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) + + self.representation_id = _id + self.item = None # set to item that mouse was clicked over + self.project = project + + manager = ModulesManager() + self.sync_server = manager.modules_by_name["sync_server"] + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -693,7 +752,6 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) model = SyncRepresentationDetailModel(headers, _id, project) self.table_view.setModel(model) @@ -702,31 +760,32 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QTableView.SelectRows) - self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + self.table_view.horizontalHeader().setSortIndicator(-1, + Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("size") + column = self.table_view.model().get_header_index("size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -740,7 +799,16 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self._on_context_menu) def _show_detail(self): - pass + """ + Shows windows with error message for failed sync of a file. + """ + dt = max(self.item.created_dt, self.item.sync_dt) + detail_window = SyncRepresentationErrorWindow(self.item._id, + self.project, + dt, + self.item.tries, + self.item.error) + detail_window.exec() def _on_context_menu(self, point): """ @@ -750,16 +818,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): if not point_index.isValid(): return - item = self.table_view.model()._data[point_index.row()] - log.info('item:: {}'.format(item)) + self.item = self.table_view.model()._data[point_index.row()] menu = QtWidgets.QMenu() actions_mapping = {} - if item.state == STATUS[1]: - action = QtWidgets.QAction("Open detail") + + if self.item.state == STATUS[1]: + action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail menu.addAction(action) + remote_site, remote_progress = self.item.remote_site.split() + if remote_progress == '1': + action = QtWidgets.QAction("Reset local site") + actions_mapping[action] = self._reset_local_site + menu.addAction(action) + + local_site, local_progress = self.item.local_site.split() + if local_progress == '1': + action = QtWidgets.QAction("Reset remote site") + actions_mapping[action] = self._reset_remote_site + menu.addAction(action) + if not actions_mapping: action = QtWidgets.QAction("< No action >") actions_mapping[action] = None @@ -770,12 +850,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): to_run = actions_mapping[result] if to_run: to_run() - to_run() + + def _reset_local_site(self): + log.info("reset local site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'studio') # TEMP + + def _reset_remote_site(self): + log.info("reset remote site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'gdrive') # TEMP class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + """ + List of all syncronizable files per single representation. + """ PAGE_SIZE = 30 - # TODO add filename to sort + # TODO add filter filename DEFAULT_SORT = { "files.path": 1 } @@ -806,6 +902,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) + tries = attr.ib(default=None) + error = attr.ib(default=None) def __init__(self, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() @@ -816,77 +914,48 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id - log.debug("!!! init _id: {}".format(self._id)) self._initialized = False self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT # in case we would like to hide/show some columns - self.projection = { - "files": 1, - 'progress_remote': 1, - 'progress_local': 1, - 'updated_dt_remote': 1, - 'updated_dt_local': 1, - 'status': { - '$switch': { - 'branches': [ - { - 'case': { - '$or': [{'$eq': ['$progress_remote', 0]}, - {'$eq': ['$progress_local', 0]}]}, - 'then': 0 - }, - { - 'case': { - '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 - }, - { - 'case': {'$or': [{'$and': [ - {'$gt': ['$progress_remote', 0]}, - {'$lt': ['$progress_remote', 1]} - ]}, - {'$and': [ - {'$gt': ['$progress_local', 0]}, - {'$lt': ['$progress_local', 1]} - ]} - ]}, - 'then': 2 - }, - { - 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 - }, - { - 'case': {'$and': [ - {'$eq': ['$progress_remote', 1]}, - {'$eq': ['$progress_local', 1]} - ]}, - 'then': 4 - }, - ], - 'default': -1 - } - } - } + self.projection = self.get_default_projection() self.query = self.get_default_query() log.debug("!!! init query: {}".format(self.query)) representations = self.dbcon.aggregate(self.query) self.refresh(representations) + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + def data(self, index, role): item = self._data[index.row()] if role == Qt.DisplayRole: @@ -905,14 +974,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - def refresh(self, representations=None): + def refresh(self, representations=None, load_records=0): self.beginResetModel() self._data = [] self._rec_loaded = 0 if not representations: - self.query = self.get_default_query() - log.debug("!!! init query: {}".format(self.query)) + self.query = self.get_default_query(load_records) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, @@ -930,18 +998,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ for repre in representations: # log.info("!!! repre:: {}".format(repre)) - - # log.debug("!!! files:: {}".format(repre.get("files", []))) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] for file in files: - created = {} - # log.info("!!! file:: {}".format(file)) - sites = file.get("sites") - # log.debug("!!! sites:: {}".format(sites)) - local_updated = remote_updated = None if repre.get('updated_dt_local'): local_updated = \ @@ -956,6 +1017,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): progress_remote = repre.get('progress_remote', '') progress_local = repre.get('progress_local', '') + errors = [] + if repre.get('failed_remote_error'): + errors.append(repre.get('failed_remote_error')) + if repre.get('failed_local_error'): + errors.append(repre.get('failed_local_error')) + item = self.SyncRepresentationDetail( repre.get("_id"), os.path.basename(file["path"]), @@ -965,13 +1032,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): '{} {}'.format(remote_site, progress_remote), file.get('size', 0), 1, - STATUS[repre.get("status", -1)] + STATUS[repre.get("status", -1)], + repre.get("tries"), + '\n'.join(errors) ) self._data.append(item) self._rec_loaded += 1 - # log.info("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) - def canFetchMore(self, index): """ Check if there are more records than currently loaded @@ -1022,7 +1089,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = filter self.refresh() - def get_default_query(self): + def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. @@ -1033,6 +1100,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): [(dict)] - list with single dict - appropriate for aggregate function for MongoDB """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + return [ {"$match": self._get_match_part()}, {"$unwind": "$files"}, @@ -1052,13 +1122,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_remote.created_dt"}, + [1], + [0]]}]}} , 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_local.created_dt"}, + [1], + [0]]}]}} # file might be successfully created or failed, not both , 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, @@ -1067,7 +1139,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_remote.last_failed_dt"}, "$order_remote.last_failed_dt", []] - }]}} + } + ]}} , 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", @@ -1075,14 +1148,29 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_local.last_failed_dt"}, "$order_local.last_failed_dt", []] - }]}} + } + ]}} , 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} , 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + , 'failed_remote_error': {'$first': { + '$cond': [{'$size': "$order_remote.error"}, + "$order_remote.error", [""]]}} + , 'failed_local_error': {'$first': { + '$cond': [{'$size': "$order_local.error"}, + "$order_local.error", [""]]}} + , 'tries': {'$first': { + '$cond': [{'$size': "$order_local.tries"}, + "$order_local.tries", + {'$cond': [ + {'$size': "$order_remote.tries"}, + "$order_remote.tries", + []] + }]}} }}, {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, + {"$limit": limit}, {"$skip": self._rec_loaded}, {"$project": self.projection} ] @@ -1106,9 +1194,70 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): "type": "representation", "_id": self._id, '$or': [{'files.path': {'$regex': regex_str, - '$options': 'i'}}] + '$options': 'i'}}] } + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'failed_remote_error': 1, + 'failed_local_error': 1, + 'tries': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } + } + class ImageDelegate(QtWidgets.QStyledItemDelegate): """ @@ -1116,6 +1265,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): """ def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) + self.icons = {} def paint(self, painter, option, index): d = index.data(QtCore.Qt.DisplayRole) @@ -1124,9 +1274,15 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): else: return - # log.info("data:: {} - {}".format(provider, value)) - pix_url = "../providers/resources/{}.png".format(provider) - pixmap = QtGui.QPixmap(pix_url) + if not self.icons.get(provider): + resource_path = os.path.dirname(__file__) + resource_path = os.path.join(resource_path, "..", + "providers", "resources") + pix_url = "{}/{}.png".format(resource_path, provider) + pixmap = QtGui.QPixmap(pix_url) + self.icons[provider] = pixmap + else: + pixmap = self.icons[provider] point = QtCore.QPoint(option.rect.x() + (option.rect.width() - pixmap.width()) / 2, @@ -1137,9 +1293,59 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): painter.setOpacity(0.5) overlay_rect = option.rect overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) - #painter.setCompositionMode(QtGui.QPainter.CompositionMode_DestinationOver) - #painter.setBrush(painter.brush(Qt.white)) - painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0,0,0,200))) + painter.fillRect(overlay_rect, + QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) + + +class SyncRepresentationErrorWindow(QtWidgets.QDialog): + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(250, 200) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg, + parent=self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Error Detail") + + +class SyncRepresentationErrorWidget(QtWidgets.QWidget): + """ + Dialog to show when sync error happened, prints error message + """ + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWidget, self).__init__(parent) + + layout = QtWidgets.QFormLayout(self) + layout.addRow(QtWidgets.QLabel("Last update date"), + QtWidgets.QLabel(pretty_timestamp(dt))) + layout.addRow(QtWidgets.QLabel("Retries"), + QtWidgets.QLabel(str(tries))) + layout.addRow(QtWidgets.QLabel("Error message"), + QtWidgets.QLabel(msg)) class SizeDelegate(QtWidgets.QStyledItemDelegate): @@ -1163,39 +1369,3 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) - -# Back up the reference to the exceptionhook -sys._excepthook = sys.excepthook - -def my_exception_hook(exctype, value, traceback): - # Print the error and traceback - print(exctype, value, traceback) - # Call the normal Exception hook after - sys._excepthook(exctype, value, traceback) - sys.exit(1) - -# Set the exception hook to our wrapping function -sys.excepthook = my_exception_hook - -if __name__ == '__main__': - import sys - from time import sleep - app = QtWidgets.QApplication(sys.argv) - #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) - os.environ["PYPE_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_DB"] = "avalon" - os.environ["AVALON_TIMEOUT"] = '3000' - - widget = SyncServerWindow() - widget.show() - - # while True: - # # run some codes that use QAxWidget.dynamicCall() function - # # print some results - # sleep(30) - - try: - sys.exit(app.exec_()) - except: - print("Exiting")