From e6c7382c7ee3d3737b1cfc1e1d6fb7b335246be5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 21 Dec 2020 13:22:54 +0100 Subject: [PATCH 01/16] #817 - Initial Gui implementation Implemented dialog for sync representations Implemented model, pagination, sorting (for most columns) Base of detail dialog --- pype/modules/sync_server/sync_server.py | 24 +- pype/modules/sync_server/tray/app.py | 490 ++++++++++++++++++++++++ 2 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 pype/modules/sync_server/tray/app.py diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 167be665f5..24dd6d4bf5 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -18,7 +18,7 @@ from .utils import time_function import six from pype.lib import PypeLogger -from .. import PypeModule, ITrayService +from .. import PypeModule, ITrayModule if six.PY2: web = asyncio = STATIC_DIR = WebSocketAsync = None @@ -34,7 +34,7 @@ class SyncStatus(Enum): DO_DOWNLOAD = 2 -class SyncServer(PypeModule, ITrayService): +class SyncServer(PypeModule, ITrayModule): """ Synchronization server that is syncing published files from local to any of implemented providers (like GDrive, S3 etc.) @@ -116,6 +116,9 @@ class SyncServer(PypeModule, ITrayService): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread + self.action_show_widget = None + self.connection = AvalonMongoDB() + def connect_with_modules(self, *_a, **kw): return @@ -147,6 +150,9 @@ class SyncServer(PypeModule, ITrayService): "no syncing possible"). format(str(self.presets)), exc_info=True) + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow() + def tray_start(self): """ Triggered when Tray is started. @@ -185,6 +191,16 @@ class SyncServer(PypeModule, ITrayService): exc_info=True ) + def tray_menu(self, parent_menu): + from Qt import QtWidgets + """Add menu or action to Tray(or parent)'s menu""" + action = QtWidgets.QAction("SyncServer", parent_menu) + action.triggered.connect(self.show_widget) + parent_menu.addAction(action) + parent_menu.addSeparator() + + self.action_show_widget = action + @property def is_running(self): return self.sync_server_thread.is_running @@ -641,6 +657,10 @@ class SyncServer(PypeModule, ITrayService): """ return int(self.presets[project_name]["config"]["loop_delay"]) + def show_widget(self): + """Show dialog to enter credentials""" + self.widget.show() + def _get_success_dict(self, file_index, site_index, new_file_id): """ Provide success metadata ("id", "created_dt") to be stored in Db. diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py new file mode 100644 index 0000000000..9cb021c270 --- /dev/null +++ b/pype/modules/sync_server/tray/app.py @@ -0,0 +1,490 @@ +from Qt import QtWidgets, QtCore, QtGui +from Qt.QtCore import Qt +from avalon import style +from avalon.api import AvalonMongoDB +from pype.tools.settings.settings.widgets.base import ProjectListWidget +from pype.modules import ModulesManager + +from pype.lib import PypeLogger +log = PypeLogger().get_logger("SyncServer") + + +class SyncServerWindow(QtWidgets.QDialog): + def __init__(self, parent=None): + super(SyncServerWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.resize(1400, 800) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = QtWidgets.QWidget() + projects = SyncProjectListWidget(self) + repres = SyncRepresentationWidget(self) + + container_layout = QtWidgets.QHBoxLayout(container) + container_layout.setContentsMargins(0, 0, 0, 0) + split = QtWidgets.QSplitter() + split.addWidget(projects) + split.addWidget(repres) + split.setSizes([180, 950, 200]) + container_layout.addWidget(split) + + container.setLayout(container_layout) + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = None + + # Project + self.combo_projects = QtWidgets.QComboBox() + + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Server") + + +class SyncProjectListWidget(ProjectListWidget): + + def validate_context_change(self): + return True + + def refresh(self): + selected_project = None + for index in self.project_list.selectedIndexes(): + selected_project = index.data(QtCore.Qt.DisplayRole) + break + + model = self.project_list.model() + model.clear() + items = [] + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + + for project_name in sync_server.get_synced_presets().keys(): + items.append(project_name) + + print("!!!! items:: {}".format(items)) + sync_server.log.debug("ld !!!! items:: {}".format(items)) + for item in items: + model.appendRow(QtGui.QStandardItem(item)) + + # self.select_project(selected_project) + + self.current_project = self.project_list.currentIndex().data( + QtCore.Qt.DisplayRole + ) + + +class SyncRepresentationWidget(QtWidgets.QWidget): + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("asset", 130), + ("subset", 190), + ("version", 30), + ("representation", 30), + ("created_dt", 120), + ("sync_dt", 85), + ("local_site", 80), + ("remote_site", 60), + ("priority", 55), + ("state", 50) + ) + + def __init__(self, parent=None): + super(SyncRepresentationWidget, self).__init__(parent) + + filter = QtWidgets.QLineEdit() + filter.setPlaceholderText("Filter subsets..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(filter) + + # TODO ? TreeViewSpinner + + table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + log.debug("!!! headers:: {}".format(headers)) + model = SyncRepresentationModel(headers) + table_view.setModel(model) + table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + table_view.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) + table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + table_view.setSortingEnabled(True) + table_view.setAlternatingRowColors(True) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(table_view) + + table_view.doubleClicked.connect(self._doubleClicked) + + def _doubleClicked(self, index): + log.debug("doubleclicked {}:{}".format(index.row(), index.column)) + detail_window = SyncServerDetailWindow(index) + detail_window.open() + + +class SyncRepresentationModel(QtCore.QAbstractTableModel): + PAGE_SIZE = 30 + DEFAULT_SORT = { + "context.asset": 1, + "context.subset": 1, + "context.version": 1, + } + SORT_BY_COLUMN = [ + "context.asset", # asset + "context.subset", # subset + "context.version", # version + "context.representation", # representation + "_id", # local created_dt + "order.created_dt", # remote created_dt + "files.sites.name", # TEMP # local progress + "files.sites.name", # TEMP# remote progress + "context.asset", # priority + "context.asset" # state + ] + DEFAULT_QUERY = { + "type": "representation", + } + + numberPopulated = QtCore.Signal(int) + + def __init__(self, header, project=None): + super(SyncRepresentationModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self._buffer = [] # stash one page worth of records (actually cursor) + + self._initialized = False + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + sync_server.get_sites_for_project('petr_test') + + self.query = self.DEFAULT_QUERY + + self.projection = { + "context.subset": 1, + "context.asset": 1, + "context.version": 1, + "context.representation": 1, + "files": 1 + } + + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def data(self, index, role): + if role == Qt.DisplayRole: + return self._data[index.row()][index.column()] + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._data[0]) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + # if orientation == Qt.Vertical: + # return str(self._data[section]) + + def refresh(self, representations): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + log.debug("!!! refresh sort {}".format(self.sort)) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + log.debug("!!! representations:: {}".format(representations)) + #log.debug("!!! representations:: {}".format(len(representations))) + for repre in representations: + context = repre.get("context") + # log.debug("!!! context:: {}".format(context)) + # log.debug("!!! repre:: {}".format(repre)) + # log.debug("!!! repre:: {}".format(type(repre))) + created = {} + # log.debug("!!! files:: {}".format(repre.get("files", []))) + # log.debug("!!! files:: {}".format(type(repre.get("files", [])))) + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + for file in files: + # log.debug("!!! file:: {}".format(file)) + # log.debug("!!! file:: {}".format(type(file))) + sites = file.get("sites") + # log.debug("!!! sites:: {}".format(sites)) + for site in sites: + # log.debug("!!! site:: {}".format(site)) + # log.debug("!!! site:: {}".format(type(site))) + if not isinstance(site, dict): + # log.debug("Obsolete site {} for {}".format( + # site, repre.get("_id"))) + continue + + if site.get("name") != local_site and \ + site.get("name") != remote_site: + continue + + if not created.get(site.get("name")): + created[site.get("name")] = [] + + created[site.get("name")]. \ + append(site.get("created_dt")) + + # log.debug("!!! created:: {}".format(created)) + # log.debug("!!! remote_site:: {}".format(remote_site)) + local_created = '' + if all(created.get(local_site, [None])): + local_created = min(created[local_site]) + # log.debug("!!! local_created:: {}".format(local_created)) + remote_created = '' + if all(created.get(remote_site, [None])): + remote_created = min(created[remote_site]) + + item = [ + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + str(local_created), + str(remote_created), + local_site, + remote_site, + 1, + 0 + ] + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) + # 'skip' might be suboptimal when representation hits 500k+ + # self._buffer = list(self.dbcon.aggregate(self.query)) + # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) + # return len(self._buffer) > self._rec_loaded + return False + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = self._buffer.count() - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + log.debug("!!! sort {} {}".format(index, order)) + log.debug("!!! orig query {}".format(self.query)) + self._rec_loaded = 0 + # limit unwanted first re-sorting by view + if index < 0: + return + + if order == 0: + order = 1 + else: + order = -1 + + if index < 5: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + elif index == 5: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = [ + {"$match": { + "type": "representation", + "files.sites": { + "$elemMatch": { + "name": self.remote_site, + "created_dt": {"$exists": 1} + }, + } + }}, + {"$unwind": "$files"}, + {"$addFields": { + "order": { + "$filter": { + "input": "$files.sites", + "as": "p", + "cond": {"$eq": ["$$p.name", self.remote_site]} + } + } + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + log.debug("!!! sort {}".format(self.sort)) + log.debug("!!! query {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def get_default_query(self): + return [ + {"$match": { + "type": "representation", + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + + +class SyncServerDetailWindow(QtWidgets.QDialog): + def __init__(self, index, parent=None): + super(SyncServerDetailWindow, self).__init__(parent) + log.debug("SyncServerDetailWindow {}:{}".format(index.row(), index.column)) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.resize(1000, 400) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = None + + container = SyncRepresentationDetailWidget(self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Detail") + + +class SyncRepresentationDetailWidget(QtWidgets.QWidget): + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("file", 230), + ("created_dt", 120), + ("sync_dt", 85), + ("local_site", 80), + ("remote_site", 60), + ("priority", 55), + ("state", 50) + ) + + def __init__(self, parent=None): + super(SyncRepresentationDetailWidget, self).__init__(parent) + + filter = QtWidgets.QLineEdit() + filter.setPlaceholderText("Filter subsets..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(filter) + + table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) + + model = SyncRepresentationModel(headers) + table_view.setModel(model) + table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + table_view.setSelectionMode( + QtWidgets.QAbstractItemView.ExtendedSelection) + table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + table_view.setSortingEnabled(True) + table_view.setAlternatingRowColors(True) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(table_view) + + def data(self, index, role): + if role == Qt.DisplayRole: + return self._data[index.row()][index.column()] + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._data[0]) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + # if orientation == Qt.Vertical: + # return str(self._data[section]) + From ffbf482d124f3a80c045c8032f69243255d0220b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 21 Dec 2020 20:58:51 +0100 Subject: [PATCH 02/16] #817 - Implemented Detail dialog Sorting, pagination --- pype/modules/sync_server/tray/app.py | 376 ++++++++++++++++++++++++--- 1 file changed, 335 insertions(+), 41 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 9cb021c270..5888a2af27 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,9 +1,19 @@ +import sys + +sys.path.append('c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype') +sys.path.append( + 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos') +sys.path.append( + 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') + from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from avalon import style from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager +import attr +import os from pype.lib import PypeLogger log = PypeLogger().get_logger("SyncServer") @@ -23,8 +33,9 @@ class SyncServerWindow(QtWidgets.QDialog): footer.setFixedHeight(20) container = QtWidgets.QWidget() - projects = SyncProjectListWidget(self) - repres = SyncRepresentationWidget(self) + projects = SyncProjectListWidget(parent=self) + repres = SyncRepresentationWidget(project=projects.current_project, + parent=self) container_layout = QtWidgets.QHBoxLayout(container) container_layout.setContentsMargins(0, 0, 0, 0) @@ -36,13 +47,6 @@ class SyncServerWindow(QtWidgets.QDialog): container.setLayout(container_layout) - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = None - - # Project - self.combo_projects = QtWidgets.QComboBox() - body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -63,6 +67,9 @@ class SyncServerWindow(QtWidgets.QDialog): class SyncProjectListWidget(ProjectListWidget): + """ + Lists all projects that are syncronized to choose from + """ def validate_context_change(self): return True @@ -82,7 +89,6 @@ class SyncProjectListWidget(ProjectListWidget): for project_name in sync_server.get_synced_presets().keys(): items.append(project_name) - print("!!!! items:: {}".format(items)) sync_server.log.debug("ld !!!! items:: {}".format(items)) for item in items: model.appendRow(QtGui.QStandardItem(item)) @@ -110,8 +116,9 @@ class SyncRepresentationWidget(QtWidgets.QWidget): ("state", 50) ) - def __init__(self, parent=None): + def __init__(self, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) + self.project = project filter = QtWidgets.QLineEdit() filter.setPlaceholderText("Filter subsets..") @@ -121,28 +128,31 @@ class SyncRepresentationWidget(QtWidgets.QWidget): # TODO ? TreeViewSpinner - table_view = QtWidgets.QTableView() + self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] log.debug("!!! headers:: {}".format(headers)) model = SyncRepresentationModel(headers) - table_view.setModel(model) - table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - table_view.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) - table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) - table_view.setSortingEnabled(True) - table_view.setAlternatingRowColors(True) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.ExtendedSelection) + self.table_view.horizontalHeader().setSortIndicator( + -1, Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) - layout.addWidget(table_view) + layout.addWidget(self.table_view) - table_view.doubleClicked.connect(self._doubleClicked) + self.table_view.doubleClicked.connect(self._doubleClicked) def _doubleClicked(self, index): - log.debug("doubleclicked {}:{}".format(index.row(), index.column)) - detail_window = SyncServerDetailWindow(index) - detail_window.open() + _id = self.table_view.model().data(index, Qt.UserRole) + log.debug("doubleclicked {}".format(_id)) + detail_window = SyncServerDetailWindow(_id, self.project) + detail_window.exec() class SyncRepresentationModel(QtCore.QAbstractTableModel): @@ -170,6 +180,25 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): numberPopulated = QtCore.Signal(int) + @attr.s + class SyncRepresentation: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + asset = attr.ib() + subset = attr.ib() + version = attr.ib() + representation = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + def __init__(self, header, project=None): super(SyncRepresentationModel, self).__init__() self._header = header @@ -210,23 +239,23 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.refresh(representations) def data(self, index, role): + item = self._data[index.row()] if role == Qt.DisplayRole: - return self._data[index.row()][index.column()] + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): - return len(self._data[0]) + return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return str(self._header[section]) - # if orientation == Qt.Vertical: - # return str(self._data[section]) - def refresh(self, representations): self.beginResetModel() self._data = [] @@ -243,7 +272,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): for repre in representations: context = repre.get("context") # log.debug("!!! context:: {}".format(context)) - # log.debug("!!! repre:: {}".format(repre)) + log.debug("!!! repre:: {}".format(repre)) # log.debug("!!! repre:: {}".format(type(repre))) created = {} # log.debug("!!! files:: {}".format(repre.get("files", []))) @@ -284,7 +313,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if all(created.get(remote_site, [None])): remote_created = min(created[remote_site]) - item = [ + item = self.SyncRepresentation( + repre.get("_id"), context.get("asset"), context.get("subset"), "v{:0>3d}".format(context.get("version", 1)), @@ -295,7 +325,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): remote_site, 1, 0 - ] + ) + self._data.append(item) self._rec_loaded += 1 @@ -335,11 +366,11 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): def sort(self, index, order): log.debug("!!! sort {} {}".format(index, order)) log.debug("!!! orig query {}".format(self.query)) - self._rec_loaded = 0 # limit unwanted first re-sorting by view if index < 0: return + self._rec_loaded = 0 if order == 0: order = 1 else: @@ -393,9 +424,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, index, parent=None): + def __init__(self, _id, project, parent=None): + log.debug( + "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) - log.debug("SyncServerDetailWindow {}:{}".format(index.row(), index.column)) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) @@ -410,7 +442,7 @@ class SyncServerDetailWindow(QtWidgets.QDialog): self.dbcon.install() self.dbcon.Session["AVALON_PROJECT"] = None - container = SyncRepresentationDetailWidget(self) + container = SyncRepresentationDetailWidget(_id, project, parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -436,16 +468,17 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): default_widths = ( ("file", 230), ("created_dt", 120), - ("sync_dt", 85), + ("sync_dt", 120), ("local_site", 80), ("remote_site", 60), ("priority", 55), ("state", 50) ) - def __init__(self, parent=None): + def __init__(self, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) - + log.debug( + "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) filter = QtWidgets.QLineEdit() filter.setPlaceholderText("Filter subsets..") @@ -456,7 +489,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): headers = [item[0] for item in self.default_widths] log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) - model = SyncRepresentationModel(headers) + model = SyncRepresentationDetailModel(headers, _id, project) table_view.setModel(model) table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) table_view.setSelectionMode( @@ -470,15 +503,103 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): layout.addLayout(top_bar_layout) layout.addWidget(table_view) + # def data(self, index, role): + # if role == Qt.DisplayRole: + # return self._data[index.row()][index.column()] + # + # def rowCount(self, index): + # return len(self._data) + # + # def columnCount(self, index): + # return len((self._header) + # + # def headerData(self, section, orientation, role): + # if role == Qt.DisplayRole: + # if orientation == Qt.Horizontal: + # return str(self._header[section]) + # + # # if orientation == Qt.Vertical: + # # return str(self._data[section]) + + +class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + PAGE_SIZE = 30 + # TODO add filename to sort + DEFAULT_SORT = { + "files._id": 1 + } + SORT_BY_COLUMN = [ + "files._id" + "_id", # local created_dt + "order.created_dt", # remote created_dt + "files.sites.name", # TEMP # local progress + "files.sites.name", # TEMP# remote progress + "context.asset", # priority + "context.asset" # state + ] + + @attr.s + class SyncRepresentationDetail: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + file = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + + def __init__(self, header, _id, project=None): + super(SyncRepresentationDetailModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self._buffer = [] # stash one page worth of records (actually cursor) + self._id = _id + log.debug("!!! init _id: {}".format(self._id)) + self._initialized = False + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + sync_server.get_sites_for_project('petr_test') + + self.sort = self.DEFAULT_SORT + + # in case we would like to hide/show some columns + self.projection = { + "files": 1 + } + + self.query = self.get_default_query() + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + def data(self, index, role): + item = self._data[index.row()] if role == Qt.DisplayRole: - return self._data[index.row()][index.column()] + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): - return len(self._data[0]) + return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: @@ -488,3 +609,176 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): # if orientation == Qt.Vertical: # return str(self._data[section]) + def refresh(self, representations): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + log.debug("!!! refresh sort {}".format(self.sort)) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + """ + Process all records from 'representation' and add them to storage. + + Args: + local_site (str): name of local site (mine) + remote_site (str): name of cloud provider (theirs) + representations (Mongo Cursor) + """ + for repre in representations: + # log.debug("!!! repre:: {}".format(repre)) + created = {} + # log.debug("!!! files:: {}".format(repre.get("files", []))) + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + for file in files: + log.debug("!!! file:: {}".format(file)) + sites = file.get("sites") + # log.debug("!!! sites:: {}".format(sites)) + for site in sites: + log.debug("!!! site:: {}".format(site)) + # log.debug("!!! site:: {}".format(type(site))) + if not isinstance(site, dict): + # log.debug("Obsolete site {} for {}".format( + # site, repre.get("_id"))) + continue + + if site.get("name") != local_site and \ + site.get("name") != remote_site: + continue + + if not created.get(site.get("name")): + created[site.get("name")] = [] + + created[site.get("name")].append(site.get("created_dt")) + + local_created = created.get(local_site) + remote_created = created.get(remote_site) + + item = self.SyncRepresentationDetail( + repre.get("_id"), + os.path.basename(file["path"]), + str(local_created), + str(remote_created), + local_site, + remote_site, + 1, + 0 + ) + self._data.append(item) + self._rec_loaded += 1 + + log.debug("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) + log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + log.debug("items_to_fetch {}".format(items_to_fetch)) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + def sort(self, index, order): + log.debug("!!! sort {} {}".format(index, order)) + log.debug("!!! orig query {}".format(self.query)) + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 # change sort - reset from start + + if order == 0: + order = 1 + else: + order = -1 + + if index < 2: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + elif index == 2: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = [ + {"$match": { + "type": "representation", + "_id": self._id, + "files.sites": { + "$elemMatch": { + "name": self.remote_site, + "created_dt": {"$exists": 1} + }, + } + }}, + {"$unwind": "$files"}, + {"$addFields": { + "order": { + "$filter": { + "input": "$files.sites", + "as": "p", + "cond": {"$eq": ["$$p.name", self.remote_site]} + } + } + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + log.debug("!!! sort {}".format(self.sort)) + log.debug("!!! query {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def get_default_query(self): + """ + Gets query that gets used when no extra sorting, filtering or + projecting is needed. + + Called for basic table view. + """ + return [ + {"$match": { + "type": "representation", + "_id": self._id + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + +if __name__ == '__main__': + + app = QtWidgets.QApplication(sys.argv) + #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + os.environ["PYPE_MONGO"] = "1" + + widget = SyncServerWindow() + widget.show() + + sys.exit(app.exec_()) From 9c5abace27157f362c1af9375895841978a5105a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Jan 2021 11:38:57 +0100 Subject: [PATCH 03/16] SyncServer GUI - added icons, progress, base of menu Fixed aggregate --- pype/modules/sync_server/tray/app.py | 847 ++++++++++++++++++++------- 1 file changed, 632 insertions(+), 215 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 5888a2af27..10967a07f9 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,5 @@ import sys -sys.path.append('c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype') -sys.path.append( - 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos') sys.path.append( 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') @@ -14,10 +11,23 @@ from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager import attr import os +from pype.tools.settings.settings import style +from avalon.tools.delegates import PrettyTimeDelegate from pype.lib import PypeLogger + +import json + log = PypeLogger().get_logger("SyncServer") +STATUS = { + 0: 'Queued', + 1: 'Failed', + 2: 'In Progress', + 3: 'Paused', + 4: 'Synced OK', + -1: 'Not available' +} class SyncServerWindow(QtWidgets.QDialog): def __init__(self, parent=None): @@ -26,6 +36,7 @@ class SyncServerWindow(QtWidgets.QDialog): self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1400, 800) body = QtWidgets.QWidget() @@ -104,15 +115,17 @@ class SyncRepresentationWidget(QtWidgets.QWidget): active_changed = QtCore.Signal() # active index changed default_widths = ( - ("asset", 130), + ("asset", 210), ("subset", 190), - ("version", 30), - ("representation", 30), - ("created_dt", 120), - ("sync_dt", 85), - ("local_site", 80), - ("remote_site", 60), - ("priority", 55), + ("version", 10), + ("representation", 90), + ("created_dt", 100), + ("sync_dt", 100), + ("local_site", 60), + ("remote_site", 70), + ("files_count", 70), + ("files_size", 70), + ("priority", 20), ("state", 50) ) @@ -120,11 +133,11 @@ class SyncRepresentationWidget(QtWidgets.QWidget): super(SyncRepresentationWidget, self).__init__(parent) self.project = project - filter = QtWidgets.QLineEdit() - filter.setPlaceholderText("Filter subsets..") + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representations..") top_bar_layout = QtWidgets.QHBoxLayout() - top_bar_layout.addWidget(filter) + top_bar_layout.addWidget(self.filter) # TODO ? TreeViewSpinner @@ -134,12 +147,38 @@ class SyncRepresentationWidget(QtWidgets.QWidget): model = SyncRepresentationModel(headers) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.table_view.setSelectionMode( - QtWidgets.QAbstractItemView.ExtendedSelection) + # self.table_view.setSelectionMode( + # QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( -1, Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model()._header.index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model()._header.index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model()._header.index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + + column = self.table_view.model()._header.index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("files_size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model._header.index(column_name) + self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -147,13 +186,27 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addWidget(self.table_view) self.table_view.doubleClicked.connect(self._doubleClicked) + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) def _doubleClicked(self, index): + """ + Opens representation dialog with all files after doubleclick + """ _id = self.table_view.model().data(index, Qt.UserRole) - log.debug("doubleclicked {}".format(_id)) detail_window = SyncServerDetailWindow(_id, self.project) detail_window.exec() + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.view.indexAt(point) + if not point_index.isValid(): + return + class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 30 @@ -167,12 +220,14 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.subset", # subset "context.version", # version "context.representation", # representation - "_id", # local created_dt - "order.created_dt", # remote created_dt - "files.sites.name", # TEMP # local progress - "files.sites.name", # TEMP# remote progress - "context.asset", # priority - "context.asset" # state + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "avg_progress_local", # local progress + "avg_progress_remote", # remote progress + "files_count", # count of files + "files_size", # file size of all files + "context.asset", # priority TODO + "status" # state ] DEFAULT_QUERY = { "type": "representation", @@ -196,6 +251,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) + files_count = attr.ib(default=None) + files_size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) @@ -206,6 +263,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._project = project self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) + self.filter = None self._initialized = False @@ -227,19 +285,67 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset": 1, "context.version": 1, "context.representation": 1, - "files": 1 + "files": 1, + 'files_count': 1, + "files_size": 1, + 'avg_progress_remote': 1, + 'avg_progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$avg_progress_remote', 0]}, + {'$eq': ['$avg_progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$avg_progress_remote', 0]}, + {'$lt': ['$avg_progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$avg_progress_local', 0]}, + {'$lt': ['$avg_progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$avg_progress_remote', 1]}, + {'$eq': ['$avg_progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } } self.sort = self.DEFAULT_SORT self.query = self.get_default_query() self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(self.query)) + log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) def data(self, index, role): item = self._data[index.row()] + if role == Qt.DisplayRole: return attr.asdict(item)[self._header[index.column()]] if role == Qt.UserRole: @@ -256,11 +362,16 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - def refresh(self, representations): + def refresh(self, representations=None): self.beginResetModel() self._data = [] self._rec_loaded = 0 log.debug("!!! refresh sort {}".format(self.sort)) + if not representations: + self.query = self.get_default_query() + log.debug( + "!!! init query: {}".format(json.dumps(self.query, indent=4))) + representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) @@ -270,9 +381,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): log.debug("!!! representations:: {}".format(representations)) #log.debug("!!! representations:: {}".format(len(representations))) for repre in representations: - context = repre.get("context") + context = repre.get("context").pop() # log.debug("!!! context:: {}".format(context)) - log.debug("!!! repre:: {}".format(repre)) + # log.info("!!! repre:: {}".format(repre)) # log.debug("!!! repre:: {}".format(type(repre))) created = {} # log.debug("!!! files:: {}".format(repre.get("files", []))) @@ -280,38 +391,22 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] - for file in files: - # log.debug("!!! file:: {}".format(file)) - # log.debug("!!! file:: {}".format(type(file))) - sites = file.get("sites") - # log.debug("!!! sites:: {}".format(sites)) - for site in sites: - # log.debug("!!! site:: {}".format(site)) - # log.debug("!!! site:: {}".format(type(site))) - if not isinstance(site, dict): - # log.debug("Obsolete site {} for {}".format( - # site, repre.get("_id"))) - continue - if site.get("name") != local_site and \ - site.get("name") != remote_site: - continue + # representation without files doesnt concern us + if not files: + continue - if not created.get(site.get("name")): - created[site.get("name")] = [] + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") - created[site.get("name")]. \ - append(site.get("created_dt")) + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - # log.debug("!!! created:: {}".format(created)) - # log.debug("!!! remote_site:: {}".format(remote_site)) - local_created = '' - if all(created.get(local_site, [None])): - local_created = min(created[local_site]) - # log.debug("!!! local_created:: {}".format(local_created)) - remote_created = '' - if all(created.get(remote_site, [None])): - remote_created = min(created[remote_site]) + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') item = self.SyncRepresentation( repre.get("_id"), @@ -319,12 +414,14 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): context.get("subset"), "v{:0>3d}".format(context.get("version", 1)), context.get("representation"), - str(local_created), - str(remote_created), - local_site, - remote_site, + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), 1, - 0 + STATUS[repre.get("status", -1)] ) self._data.append(item) @@ -336,10 +433,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): """ log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) # 'skip' might be suboptimal when representation hits 500k+ - # self._buffer = list(self.dbcon.aggregate(self.query)) + self._buffer = list(self.dbcon.aggregate(self.query)) # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) - # return len(self._buffer) > self._rec_loaded - return False + return len(self._buffer) > self._rec_loaded def fetchMore(self, index): """ @@ -351,7 +447,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): """ log.debug("fetchMore") # cursor.count() returns always total number, not only skipped + limit - remainder = self._buffer.count() - self._rec_loaded + remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) self.beginInsertRows(index, self._rec_loaded, @@ -364,8 +460,13 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.numberPopulated.emit(items_to_fetch) # ?? def sort(self, index, order): - log.debug("!!! sort {} {}".format(index, order)) - log.debug("!!! orig query {}".format(self.query)) + """ + Summary sort per representation + + Args: + index (int): column index + order (int): 0| + """ # limit unwanted first re-sorting by view if index < 0: return @@ -376,45 +477,107 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): else: order = -1 - if index < 5: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - elif index == 5: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = [ - {"$match": { - "type": "representation", - "files.sites": { - "$elemMatch": { - "name": self.remote_site, - "created_dt": {"$exists": 1} - }, - } - }}, - {"$unwind": "$files"}, - {"$addFields": { - "order": { - "$filter": { - "input": "$files.sites", - "as": "p", - "cond": {"$eq": ["$$p.name", self.remote_site]} - } - } - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(self.query)) + log.debug("!!! query {}".format(json.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) + def set_filter(self, filter): + self.filter = filter + self.refresh() + def get_default_query(self): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - queued + 1 - failed + 2 - paused (not implemented yet) + 3 - in progress + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + """ return [ - {"$match": { - "type": "representation", + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'files_size': {'$ifNull': ["$files.size", 0]} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + }}, + {'$group': { + '_id': '$_id' + # pass through context - same for representation + , 'context': {'$addToSet': '$context'} + # pass through files as a list + , 'files': {'$addToSet': '$files'} + # count how many files + , 'files_count': {'$sum': 1} + , 'files_size': {'$sum': '$files_size'} + # sum avg progress, finished = 1 + , 'avg_progress_remote': {'$avg': "$progress_remote"} + , 'avg_progress_local': {'$avg': "$progress_local"} + # select last touch of file + , 'updated_dt_remote': {'$max': "$updated_dt_remote"} + , 'failed_remote': {'$sum': '$failed_remote'} + , 'failed_local': {'$sum': '$failed_local'} + , 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$sort": self.sort}, {"$limit": self.PAGE_SIZE}, @@ -422,6 +585,47 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): {"$project": self.projection} ] + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [{'context.subset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, _id, project, parent=None): @@ -432,6 +636,7 @@ class SyncServerDetailWindow(QtWidgets.QDialog): self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1000, 400) body = QtWidgets.QWidget() @@ -466,12 +671,13 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): active_changed = QtCore.Signal() # active index changed default_widths = ( - ("file", 230), + ("file", 290), ("created_dt", 120), ("sync_dt", 120), - ("local_site", 80), + ("local_site", 60), ("remote_site", 60), - ("priority", 55), + ("size", 60), + ("priority", 20), ("state", 50) ) @@ -479,63 +685,109 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): super(SyncRepresentationDetailWidget, self).__init__(parent) log.debug( "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) - filter = QtWidgets.QLineEdit() - filter.setPlaceholderText("Filter subsets..") + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representation..") top_bar_layout = QtWidgets.QHBoxLayout() - top_bar_layout.addWidget(filter) + top_bar_layout.addWidget(self.filter) - table_view = QtWidgets.QTableView() + self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) model = SyncRepresentationDetailModel(headers, _id, project) - table_view.setModel(model) - table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - table_view.setSelectionMode( - QtWidgets.QAbstractItemView.ExtendedSelection) - table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) - table_view.setSortingEnabled(True) - table_view.setAlternatingRowColors(True) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QTableView.SelectRows) + self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model()._header.index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model()._header.index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model()._header.index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model._header.index(column_name) + self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) - layout.addWidget(table_view) + layout.addWidget(self.table_view) - # def data(self, index, role): - # if role == Qt.DisplayRole: - # return self._data[index.row()][index.column()] - # - # def rowCount(self, index): - # return len(self._data) - # - # def columnCount(self, index): - # return len((self._header) - # - # def headerData(self, section, orientation, role): - # if role == Qt.DisplayRole: - # if orientation == Qt.Horizontal: - # return str(self._header[section]) - # - # # if orientation == Qt.Vertical: - # # return str(self._data[section]) + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) + + def _show_detail(self): + pass + + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.table_view.indexAt(point) + if not point_index.isValid(): + return + + item = self.table_view.model()._data[point_index.row()] + log.info('item:: {}'.format(item)) + + menu = QtWidgets.QMenu() + actions_mapping = {} + if item.state == STATUS[1]: + action = QtWidgets.QAction("Open detail") + actions_mapping[action] = self._show_detail + menu.addAction(action) + + if not actions_mapping: + action = QtWidgets.QAction("< No action >") + actions_mapping[action] = None + menu.addAction(action) + + result = menu.exec_(QtGui.QCursor.pos()) + if result: + to_run = actions_mapping[result] + if to_run: + to_run() + to_run() class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): PAGE_SIZE = 30 # TODO add filename to sort DEFAULT_SORT = { - "files._id": 1 + "files.path": 1 } SORT_BY_COLUMN = [ - "files._id" - "_id", # local created_dt - "order.created_dt", # remote created_dt - "files.sites.name", # TEMP # local progress - "files.sites.name", # TEMP# remote progress - "context.asset", # priority - "context.asset" # state + "files.path", + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "progress_local", # local progress + "progress_remote", # remote progress + "size", # remote progress + "context.asset", # priority TODO + "status" # state ] @attr.s @@ -551,6 +803,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) + size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) @@ -560,6 +813,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self._data = [] self._project = project self._rec_loaded = 0 + self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id log.debug("!!! init _id: {}".format(self._id)) @@ -580,7 +834,52 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # in case we would like to hide/show some columns self.projection = { - "files": 1 + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } } self.query = self.get_default_query() @@ -606,14 +905,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - # if orientation == Qt.Vertical: - # return str(self._data[section]) - - def refresh(self, representations): + def refresh(self, representations=None): self.beginResetModel() self._data = [] self._rec_loaded = 0 - log.debug("!!! refresh sort {}".format(self.sort)) + + if not representations: + self.query = self.get_default_query() + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) @@ -629,50 +929,48 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): representations (Mongo Cursor) """ for repre in representations: - # log.debug("!!! repre:: {}".format(repre)) - created = {} + # log.info("!!! repre:: {}".format(repre)) + # log.debug("!!! files:: {}".format(repre.get("files", []))) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] + for file in files: - log.debug("!!! file:: {}".format(file)) + created = {} + # log.info("!!! file:: {}".format(file)) sites = file.get("sites") # log.debug("!!! sites:: {}".format(sites)) - for site in sites: - log.debug("!!! site:: {}".format(site)) - # log.debug("!!! site:: {}".format(type(site))) - if not isinstance(site, dict): - # log.debug("Obsolete site {} for {}".format( - # site, repre.get("_id"))) - continue - if site.get("name") != local_site and \ - site.get("name") != remote_site: - continue + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime( + "%Y%m%dT%H%M%SZ") - if not created.get(site.get("name")): - created[site.get("name")] = [] + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime( + "%Y%m%dT%H%M%SZ") - created[site.get("name")].append(site.get("created_dt")) - - local_created = created.get(local_site) - remote_created = created.get(remote_site) + progress_remote = repre.get('progress_remote', '') + progress_local = repre.get('progress_local', '') item = self.SyncRepresentationDetail( repre.get("_id"), os.path.basename(file["path"]), - str(local_created), - str(remote_created), - local_site, - remote_site, + local_updated, + remote_updated, + '{} {}'.format(local_site, progress_local), + '{} {}'.format(remote_site, progress_remote), + file.get('size', 0), 1, - 0 + STATUS[repre.get("status", -1)] ) self._data.append(item) self._rec_loaded += 1 - log.debug("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) + # log.info("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) def canFetchMore(self, index): """ @@ -680,8 +978,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ # 'skip' might be suboptimal when representation hits 500k+ self._buffer = list(self.dbcon.aggregate(self.query)) - log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) - log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) return len(self._buffer) > self._rec_loaded def fetchMore(self, index): @@ -696,7 +992,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # cursor.count() returns always total number, not only skipped + limit remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) - log.debug("items_to_fetch {}".format(items_to_fetch)) + self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) @@ -705,8 +1001,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.endInsertRows() def sort(self, index, order): - log.debug("!!! sort {} {}".format(index, order)) - log.debug("!!! orig query {}".format(self.query)) # limit unwanted first re-sorting by view if index < 0: return @@ -718,53 +1012,74 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): else: order = -1 - if index < 2: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - elif index == 2: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = [ - {"$match": { - "type": "representation", - "_id": self._id, - "files.sites": { - "$elemMatch": { - "name": self.remote_site, - "created_dt": {"$exists": 1} - }, - } - }}, - {"$unwind": "$files"}, - {"$addFields": { - "order": { - "$filter": { - "input": "$files.sites", - "as": "p", - "cond": {"$eq": ["$$p.name", self.remote_site]} - } - } - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] - log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(self.query)) + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + representations = self.dbcon.aggregate(self.query) self.refresh(representations) + def set_filter(self, filter): + self.filter = filter + self.refresh() + def get_default_query(self): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. Called for basic table view. + + Returns: + [(dict)] - list with single dict - appropriate for aggregate + function for MongoDB """ return [ - {"$match": { - "type": "representation", - "_id": self._id + {"$match": self._get_match_part()}, + {"$unwind": "$files"}, + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} }}, {"$sort": self.sort}, {"$limit": self.PAGE_SIZE}, @@ -772,13 +1087,115 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {"$project": self.projection} ] -if __name__ == '__main__': + def _get_match_part(self): + """ + Returns different content for 'match' portion if filtering by + name is present + Returns: + (dict) + """ + if not self.filter: + return { + "type": "representation", + "_id": self._id + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + "_id": self._id, + '$or': [{'files.path': {'$regex': regex_str, + '$options': 'i'}}] + } + + +class ImageDelegate(QtWidgets.QStyledItemDelegate): + """ + Prints icon of site and progress of synchronization + """ + def __init__(self, parent=None): + super(ImageDelegate, self).__init__(parent) + + def paint(self, painter, option, index): + d = index.data(QtCore.Qt.DisplayRole) + if d: + provider, value = d.split() + else: + return + + # log.info("data:: {} - {}".format(provider, value)) + pix_url = "../providers/resources/{}.png".format(provider) + pixmap = QtGui.QPixmap(pix_url) + + point = QtCore.QPoint(option.rect.x() + + (option.rect.width() - pixmap.width()) / 2, + option.rect.y() + + (option.rect.height() - pixmap.height()) / 2) + painter.drawPixmap(point, pixmap) + + painter.setOpacity(0.5) + overlay_rect = option.rect + overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) + #painter.setCompositionMode(QtGui.QPainter.CompositionMode_DestinationOver) + #painter.setBrush(painter.brush(Qt.white)) + painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0,0,0,200))) + + +class SizeDelegate(QtWidgets.QStyledItemDelegate): + """ + Pretty print for file size + """ + def __init__(self, parent=None): + super(SizeDelegate, self).__init__(parent) + + def displayText(self, value, locale): + if value is None: + # Ignore None value + return + + return self._pretty_size(value) + + def _pretty_size(self, value, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(value) < 1024.0: + return "%3.1f%s%s" % (value, unit, suffix) + value /= 1024.0 + return "%.1f%s%s" % (value, 'Yi', suffix) + + +# Back up the reference to the exceptionhook +sys._excepthook = sys.excepthook + +def my_exception_hook(exctype, value, traceback): + # Print the error and traceback + print(exctype, value, traceback) + # Call the normal Exception hook after + sys._excepthook(exctype, value, traceback) + sys.exit(1) + +# Set the exception hook to our wrapping function +sys.excepthook = my_exception_hook + +if __name__ == '__main__': + import sys + from time import sleep app = QtWidgets.QApplication(sys.argv) #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) - os.environ["PYPE_MONGO"] = "1" + os.environ["PYPE_MONGO"] = "mongodb://localhost:27017" + os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" + os.environ["AVALON_DB"] = "avalon" + os.environ["AVALON_TIMEOUT"] = '3000' widget = SyncServerWindow() widget.show() - sys.exit(app.exec_()) + # while True: + # # run some codes that use QAxWidget.dynamicCall() function + # # print some results + # sleep(30) + + try: + sys.exit(app.exec_()) + except: + print("Exiting") From 8f768b2e4a0a203cf2172b6ba8a1f8cda9fc2f60 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Jan 2021 18:44:39 +0100 Subject: [PATCH 04/16] SyncServer GUI - added icons, fixes Added refresh Added menu in detail --- .../providers/resources/gdrive.png | Bin 0 -> 975 bytes .../providers/resources/studio.png | Bin 0 -> 557 bytes pype/modules/sync_server/tray/app.py | 1098 ++++++++++------- 3 files changed, 634 insertions(+), 464 deletions(-) create mode 100644 pype/modules/sync_server/providers/resources/gdrive.png create mode 100644 pype/modules/sync_server/providers/resources/studio.png diff --git a/pype/modules/sync_server/providers/resources/gdrive.png b/pype/modules/sync_server/providers/resources/gdrive.png new file mode 100644 index 0000000000000000000000000000000000000000..e6c913145439ed4f4772035885db281d289e9e45 GIT binary patch literal 975 zcmV;=12FuFP)`J=v zA}Hn}5-qxlWJ3FpzAXg>{{f?`>Y~6XbOtN)%F4PZF*F()Cu^KJXMa3T7yInJkBuo_ z)`pkoWxt!Xp0%F!ZuqZ1y42x{L)$j(nXi#G&B>b}1JA%Rs=a5>{^JSV_38dACRjZ- zvY_4K`vHP~a_#uS21$G&K?e$mR z@C@7+kT-W-k%CoUY=1Z?e3V+M)D53$w?4b!Y+#cq;H+espEuWeJ+rdPCAu5ej(%MS;=j_VTZj zc|TofQwE-tgiG<(j!P2UKl<|OSn-pR07s>P0tnhNxyT{Qt6|p)I@O={udciM;shfj zVUC*ngkpJt=_S4bL~th0j$^G4>JS{9Q>d;I{>EQ0mx=5B8&T3*^hF#&|Y9ij6H z$!1#9o5|bfx)EE_H4rHKSN8;hEDibaH$%f|iLbR}O%ei_3B|A$K#Eyzn{>$wb`B zvIEU1-cqm@y~_h&1T+=@yna>G#agfWzk>EWYo=I#QXsVwu^F(3ls!U~Usa zo!l2yKyzZ~SFE?;c~(WX7*GLFR?WIIAQ+$4+@0b5Wlv4D^5kt)rL*WJ?>lL~FS|tq zu(UoCW!O4ZC1-^%L7MripKsJu09f4U-~r*?n~e zxa;KgfA+su#dP1$aGH4fiUe{IPcQk-rmq=90f-<9;8u05s?*us1EiUa>>F0v{SF*G za^$UV@i6$;i4F7co~9I`pyE!12*r)s2I@IPotFV{_v0UL?7tlq2G;)~;WL(ROULgR x$LBOr1VNqE*;AA{IW<7pX>NMH1pj|Le*-`p;DxPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0m4Z{K~zXf-IXCv z13?glPk}@tk(J1rHpr&eD<^UA52&M(goH{=0S69AgQ~2fvO!fME0MM3dz)FZZSSsk zIC#m+&b*!3*}dDji~b+dG(G5cyFGYak-c`ieQ3+%BT14eyvtaikjjCdwoE<Y z=5FEF1?>1s#wnv7jZXQj3t=G+l?&({;LqSdw#g1?7n%!iPMDjEFbLb-5OcS{T#J*8 z&8=3z`e707oW)Mj&dqhepZNDuseqD{>_ni&;XB$Zyfs<6dd6K>pIZnYL}H3Paw5RK zM0~E0lsm%*kr|zv>CI@7Z6BP2e-_1K&zoG zrK`~x=l(r~xw!~~>=biYH^~8CyZ&N)3>z=Z&BbPrb_~^EBj`VD_lzH={hScHdI1j> z)l2tZPuFCyu9NtV#xGaEI(Y4|Y{EWn3kvjQTi=u`0QJLJYqBB`JEM^Lp;Q6w>V|O$ v{Uh1Mq2(-XnS9vMvliLOqEGo>MHGDlp;tCVl9Dor00000NkvXXu0mjfWd!-+ literal 0 HcmV?d00001 diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 10967a07f9..aa52b06da9 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,3 @@ -import sys - -sys.path.append( - 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') - from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from avalon import style @@ -12,7 +7,7 @@ from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style -from avalon.tools.delegates import PrettyTimeDelegate +from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -29,7 +24,12 @@ STATUS = { -1: 'Not available' } + class SyncServerWindow(QtWidgets.QDialog): + """ + Main window that contains list of synchronizable projects and summary + view with all synchronizable representations for first project + """ def __init__(self, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) @@ -76,42 +76,44 @@ class SyncServerWindow(QtWidgets.QDialog): self.setLayout(body_layout) self.setWindowTitle("Sync Server") + projects.project_changed.connect( + lambda: repres.table_view.model().set_project( + projects.current_project)) + class SyncProjectListWidget(ProjectListWidget): """ - Lists all projects that are syncronized to choose from + Lists all projects that are synchronized to choose from """ def validate_context_change(self): return True def refresh(self): - selected_project = None - for index in self.project_list.selectedIndexes(): - selected_project = index.data(QtCore.Qt.DisplayRole) - break - model = self.project_list.model() model.clear() - items = [] manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] for project_name in sync_server.get_synced_presets().keys(): - items.append(project_name) + model.appendRow(QtGui.QStandardItem(project_name)) - sync_server.log.debug("ld !!!! items:: {}".format(items)) - for item in items: - model.appendRow(QtGui.QStandardItem(item)) - - # self.select_project(selected_project) + if len(sync_server.get_synced_presets().keys()) == 0: + model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( QtCore.Qt.DisplayRole ) + if not self.current_project: + self.current_project = self.project_list.model().item(0).\ + data(QtCore.Qt.DisplayRole) class SyncRepresentationWidget(QtWidgets.QWidget): + """ + Summary dialog with list of representations that matches current + settings 'local_site' and 'remote_site'. + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -139,16 +141,14 @@ class SyncRepresentationWidget(QtWidgets.QWidget): top_bar_layout = QtWidgets.QHBoxLayout() top_bar_layout.addWidget(self.filter) - # TODO ? TreeViewSpinner - self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! headers:: {}".format(headers)) - model = SyncRepresentationModel(headers) + + model = SyncRepresentationModel(headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - # self.table_view.setSelectionMode( - # QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( @@ -158,26 +158,25 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("files_size") + column = self.table_view.model().get_header_index("files_size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -203,17 +202,19 @@ class SyncRepresentationWidget(QtWidgets.QWidget): """ Shows menu with loader actions on Right-click. """ - point_index = self.view.indexAt(point) + point_index = self.table_view.indexAt(point) if not point_index.isValid(): return class SyncRepresentationModel(QtCore.QAbstractTableModel): - PAGE_SIZE = 30 + PAGE_SIZE = 19 + REFRESH_SEC = 5000 DEFAULT_SORT = { "context.asset": 1, "context.subset": 1, "context.version": 1, + "_id": 1 } SORT_BY_COLUMN = [ "context.asset", # asset @@ -229,9 +230,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset", # priority TODO "status" # state ] - DEFAULT_QUERY = { - "type": "representation", - } numberPopulated = QtCore.Signal(int) @@ -269,18 +267,356 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) - self.query = self.DEFAULT_QUERY + self.projection = self.get_default_projection() - self.projection = { + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(json.dumps(self.query, + indent=4))) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(self.REFRESH_SEC) + + def data(self, index, role): + item = self._data[index.row()] + + if role == Qt.DisplayRole: + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._header) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(self.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + + def refresh(self, representations=None, load_records=0): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + + if not representations: + self.query = self.get_default_query(load_records) + representations = self.dbcon.aggregate(self.query) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + for repre in representations: + context = repre.get("context").pop() + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + + # representation without files doesnt concern us + if not files: + continue + + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") + + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") + + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') + + item = self.SyncRepresentation( + repre.get("_id"), + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), + 1, + STATUS[repre.get("status", -1)] + ) + + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( + # self._rec_loaded, len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + """ + Summary sort per representation. + + Sort is happening on a DB side, model is reset, db queried + again. + + Args: + index (int): column index + order (int): 0| + """ + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 + if order == 0: + order = 1 + else: + order = -1 + + self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} + self.query = self.get_default_query() + + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def set_filter(self, filter): + """ + Adds text value filtering + + Args: + filter (str): string inputted by user + """ + self.filter = filter + self.refresh() + + def set_project(self, project): + """ + Changes project, called after project selection is changed + + Args: + project (str): name of project + """ + self._project = project + self.dbcon.Session["AVALON_PROJECT"] = self._project + self.refresh() + + def get_default_query(self, limit=0): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - queued + 1 - failed + 2 - paused (not implemented yet) + 3 - in progress + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + + Args: + limit (int): how many records should be returned, by default + it 'PAGE_SIZE' for performance. + Should be overridden by value of loaded records for refresh + functionality (got more records by scrolling, refresh + shouldn't reset that) + """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + + return [ + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'files_size': {'$ifNull': ["$files.size", 0]} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + }}, + {'$group': { + '_id': '$_id' + # pass through context - same for representation + , 'context': {'$addToSet': '$context'} + # pass through files as a list + , 'files': {'$addToSet': '$files'} + # count how many files + , 'files_count': {'$sum': 1} + , 'files_size': {'$sum': '$files_size'} + # sum avg progress, finished = 1 + , 'avg_progress_remote': {'$avg': "$progress_remote"} + , 'avg_progress_local': {'$avg': "$progress_local"} + # select last touch of file + , 'updated_dt_remote': {'$max': "$updated_dt_remote"} + , 'failed_remote': {'$sum': '$failed_remote'} + , 'failed_local': {'$sum': '$failed_local'} + , 'updated_dt_local': {'$max': "$updated_dt_local"} + }}, + {"$sort": self.sort}, + {"$limit": limit}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [{'context.subset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { "context.subset": 1, "context.asset": 1, "context.version": 1, @@ -335,297 +671,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): } } - self.sort = self.DEFAULT_SORT - - self.query = self.get_default_query() - self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def data(self, index, role): - item = self._data[index.row()] - - if role == Qt.DisplayRole: - return attr.asdict(item)[self._header[index.column()]] - if role == Qt.UserRole: - return item._id - - def rowCount(self, index): - return len(self._data) - - def columnCount(self, index): - return len(self._header) - - def headerData(self, section, orientation, role): - if role == Qt.DisplayRole: - if orientation == Qt.Horizontal: - return str(self._header[section]) - - def refresh(self, representations=None): - self.beginResetModel() - self._data = [] - self._rec_loaded = 0 - log.debug("!!! refresh sort {}".format(self.sort)) - if not representations: - self.query = self.get_default_query() - log.debug( - "!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - - self._add_page_records(self.local_site, self.remote_site, - representations) - self.endResetModel() - - def _add_page_records(self, local_site, remote_site, representations): - log.debug("!!! representations:: {}".format(representations)) - #log.debug("!!! representations:: {}".format(len(representations))) - for repre in representations: - context = repre.get("context").pop() - # log.debug("!!! context:: {}".format(context)) - # log.info("!!! repre:: {}".format(repre)) - # log.debug("!!! repre:: {}".format(type(repre))) - created = {} - # log.debug("!!! files:: {}".format(repre.get("files", []))) - # log.debug("!!! files:: {}".format(type(repre.get("files", [])))) - files = repre.get("files", []) - if isinstance(files, dict): # aggregate returns dictionary - files = [files] - - # representation without files doesnt concern us - if not files: - continue - - local_updated = remote_updated = None - if repre.get('updated_dt_local'): - local_updated = \ - repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") - - if repre.get('updated_dt_remote'): - remote_updated = \ - repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - - avg_progress_remote = repre.get('avg_progress_remote', '') - avg_progress_local = repre.get('avg_progress_local', '') - - item = self.SyncRepresentation( - repre.get("_id"), - context.get("asset"), - context.get("subset"), - "v{:0>3d}".format(context.get("version", 1)), - context.get("representation"), - local_updated, - remote_updated, - '{} {}'.format(local_site, avg_progress_local), - '{} {}'.format(remote_site, avg_progress_remote), - repre.get("files_count", 1), - repre.get("files_size", 0), - 1, - STATUS[repre.get("status", -1)] - ) - - self._data.append(item) - self._rec_loaded += 1 - - def canFetchMore(self, index): - """ - Check if there are more records than currently loaded - """ - log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) - # 'skip' might be suboptimal when representation hits 500k+ - self._buffer = list(self.dbcon.aggregate(self.query)) - # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) - return len(self._buffer) > self._rec_loaded - - def fetchMore(self, index): - """ - Add more record to model. - - Called when 'canFetchMore' returns true, which means there are - more records in DB than loaded. - 'self._buffer' is used to stash cursor to limit requery - """ - log.debug("fetchMore") - # cursor.count() returns always total number, not only skipped + limit - remainder = len(self._buffer) - self._rec_loaded - items_to_fetch = min(self.PAGE_SIZE, remainder) - self.beginInsertRows(index, - self._rec_loaded, - self._rec_loaded + items_to_fetch - 1) - - self._add_page_records(self.local_site, self.remote_site, self._buffer) - - self.endInsertRows() - - self.numberPopulated.emit(items_to_fetch) # ?? - - def sort(self, index, order): - """ - Summary sort per representation - - Args: - index (int): column index - order (int): 0| - """ - # limit unwanted first re-sorting by view - if index < 0: - return - - self._rec_loaded = 0 - if order == 0: - order = 1 - else: - order = -1 - - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - - log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def set_filter(self, filter): - self.filter = filter - self.refresh() - - def get_default_query(self): - """ - Returns basic aggregate query for main table. - - Main table provides summary information about representation, - which could have multiple files. Details are accessible after - double click on representation row. - Columns: - 'created_dt' - max of created or updated (when failed) per repr - 'sync_dt' - same for remote side - 'local_site' - progress of repr on local side, 1 = finished - 'remote_site' - progress on remote side, calculates from files - 'state' - - 0 - queued - 1 - failed - 2 - paused (not implemented yet) - 3 - in progress - 4 - finished on both sides - - are calculated and must be calculated in DB because of - pagination - """ - return [ - {"$match": self._get_match_part()}, - {'$unwind': '$files'}, - # merge potentially unwinded records back to single per repre - {'$addFields': { - 'order_remote': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.local_site]} - }} - }}, - {'$addFields': { - # prepare progress per file, presence of 'created_dt' denotes - # successfully finished load/download - 'progress_remote': {'$first': { - '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} - , 'progress_local': {'$first': { - '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} - # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { - '$cond': [{'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - }]}} - , 'updated_dt_local': {'$first': { - '$cond': [{'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - }]}} - , 'files_size': {'$ifNull': ["$files.size", 0]} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} - }}, - {'$group': { - '_id': '$_id' - # pass through context - same for representation - , 'context': {'$addToSet': '$context'} - # pass through files as a list - , 'files': {'$addToSet': '$files'} - # count how many files - , 'files_count': {'$sum': 1} - , 'files_size': {'$sum': '$files_size'} - # sum avg progress, finished = 1 - , 'avg_progress_remote': {'$avg': "$progress_remote"} - , 'avg_progress_local': {'$avg': "$progress_local"} - # select last touch of file - , 'updated_dt_remote': {'$max': "$updated_dt_remote"} - , 'failed_remote': {'$sum': '$failed_remote'} - , 'failed_local': {'$sum': '$failed_local'} - , 'updated_dt_local': {'$max': "$updated_dt_local"} - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] - - def _get_match_part(self): - """ - Extend match part with filter if present. - - Filter is set by user input. Each model has different fields to be - checked. - If performance issues are found, '$text' and text indexes should - be investigated. - """ - if not self.filter: - return { - "type": "representation", - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - else: - regex_str = '.*{}.*'.format(self.filter) - return { - "type": "representation", - '$or': [{'context.subset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.asset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, _id, project, parent=None): @@ -668,6 +713,14 @@ class SyncServerDetailWindow(QtWidgets.QDialog): class SyncRepresentationDetailWidget(QtWidgets.QWidget): + """ + Widget to display list of synchronizable files for single repre. + + Args: + _id (str): representation _id + project (str): name of project with repre + parent (QDialog): SyncServerDetailWindow + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -683,8 +736,14 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def __init__(self, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) - log.debug( - "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) + + self.representation_id = _id + self.item = None # set to item that mouse was clicked over + self.project = project + + manager = ModulesManager() + self.sync_server = manager.modules_by_name["sync_server"] + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -693,7 +752,6 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) model = SyncRepresentationDetailModel(headers, _id, project) self.table_view.setModel(model) @@ -702,31 +760,32 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QTableView.SelectRows) - self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + self.table_view.horizontalHeader().setSortIndicator(-1, + Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("size") + column = self.table_view.model().get_header_index("size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -740,7 +799,16 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self._on_context_menu) def _show_detail(self): - pass + """ + Shows windows with error message for failed sync of a file. + """ + dt = max(self.item.created_dt, self.item.sync_dt) + detail_window = SyncRepresentationErrorWindow(self.item._id, + self.project, + dt, + self.item.tries, + self.item.error) + detail_window.exec() def _on_context_menu(self, point): """ @@ -750,16 +818,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): if not point_index.isValid(): return - item = self.table_view.model()._data[point_index.row()] - log.info('item:: {}'.format(item)) + self.item = self.table_view.model()._data[point_index.row()] menu = QtWidgets.QMenu() actions_mapping = {} - if item.state == STATUS[1]: - action = QtWidgets.QAction("Open detail") + + if self.item.state == STATUS[1]: + action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail menu.addAction(action) + remote_site, remote_progress = self.item.remote_site.split() + if remote_progress == '1': + action = QtWidgets.QAction("Reset local site") + actions_mapping[action] = self._reset_local_site + menu.addAction(action) + + local_site, local_progress = self.item.local_site.split() + if local_progress == '1': + action = QtWidgets.QAction("Reset remote site") + actions_mapping[action] = self._reset_remote_site + menu.addAction(action) + if not actions_mapping: action = QtWidgets.QAction("< No action >") actions_mapping[action] = None @@ -770,12 +850,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): to_run = actions_mapping[result] if to_run: to_run() - to_run() + + def _reset_local_site(self): + log.info("reset local site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'studio') # TEMP + + def _reset_remote_site(self): + log.info("reset remote site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'gdrive') # TEMP class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + """ + List of all syncronizable files per single representation. + """ PAGE_SIZE = 30 - # TODO add filename to sort + # TODO add filter filename DEFAULT_SORT = { "files.path": 1 } @@ -806,6 +902,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) + tries = attr.ib(default=None) + error = attr.ib(default=None) def __init__(self, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() @@ -816,77 +914,48 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id - log.debug("!!! init _id: {}".format(self._id)) self._initialized = False self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT # in case we would like to hide/show some columns - self.projection = { - "files": 1, - 'progress_remote': 1, - 'progress_local': 1, - 'updated_dt_remote': 1, - 'updated_dt_local': 1, - 'status': { - '$switch': { - 'branches': [ - { - 'case': { - '$or': [{'$eq': ['$progress_remote', 0]}, - {'$eq': ['$progress_local', 0]}]}, - 'then': 0 - }, - { - 'case': { - '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 - }, - { - 'case': {'$or': [{'$and': [ - {'$gt': ['$progress_remote', 0]}, - {'$lt': ['$progress_remote', 1]} - ]}, - {'$and': [ - {'$gt': ['$progress_local', 0]}, - {'$lt': ['$progress_local', 1]} - ]} - ]}, - 'then': 2 - }, - { - 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 - }, - { - 'case': {'$and': [ - {'$eq': ['$progress_remote', 1]}, - {'$eq': ['$progress_local', 1]} - ]}, - 'then': 4 - }, - ], - 'default': -1 - } - } - } + self.projection = self.get_default_projection() self.query = self.get_default_query() log.debug("!!! init query: {}".format(self.query)) representations = self.dbcon.aggregate(self.query) self.refresh(representations) + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + def data(self, index, role): item = self._data[index.row()] if role == Qt.DisplayRole: @@ -905,14 +974,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - def refresh(self, representations=None): + def refresh(self, representations=None, load_records=0): self.beginResetModel() self._data = [] self._rec_loaded = 0 if not representations: - self.query = self.get_default_query() - log.debug("!!! init query: {}".format(self.query)) + self.query = self.get_default_query(load_records) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, @@ -930,18 +998,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ for repre in representations: # log.info("!!! repre:: {}".format(repre)) - - # log.debug("!!! files:: {}".format(repre.get("files", []))) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] for file in files: - created = {} - # log.info("!!! file:: {}".format(file)) - sites = file.get("sites") - # log.debug("!!! sites:: {}".format(sites)) - local_updated = remote_updated = None if repre.get('updated_dt_local'): local_updated = \ @@ -956,6 +1017,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): progress_remote = repre.get('progress_remote', '') progress_local = repre.get('progress_local', '') + errors = [] + if repre.get('failed_remote_error'): + errors.append(repre.get('failed_remote_error')) + if repre.get('failed_local_error'): + errors.append(repre.get('failed_local_error')) + item = self.SyncRepresentationDetail( repre.get("_id"), os.path.basename(file["path"]), @@ -965,13 +1032,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): '{} {}'.format(remote_site, progress_remote), file.get('size', 0), 1, - STATUS[repre.get("status", -1)] + STATUS[repre.get("status", -1)], + repre.get("tries"), + '\n'.join(errors) ) self._data.append(item) self._rec_loaded += 1 - # log.info("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) - def canFetchMore(self, index): """ Check if there are more records than currently loaded @@ -1022,7 +1089,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = filter self.refresh() - def get_default_query(self): + def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. @@ -1033,6 +1100,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): [(dict)] - list with single dict - appropriate for aggregate function for MongoDB """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + return [ {"$match": self._get_match_part()}, {"$unwind": "$files"}, @@ -1052,13 +1122,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_remote.created_dt"}, + [1], + [0]]}]}} , 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_local.created_dt"}, + [1], + [0]]}]}} # file might be successfully created or failed, not both , 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, @@ -1067,7 +1139,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_remote.last_failed_dt"}, "$order_remote.last_failed_dt", []] - }]}} + } + ]}} , 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", @@ -1075,14 +1148,29 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_local.last_failed_dt"}, "$order_local.last_failed_dt", []] - }]}} + } + ]}} , 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} , 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + , 'failed_remote_error': {'$first': { + '$cond': [{'$size': "$order_remote.error"}, + "$order_remote.error", [""]]}} + , 'failed_local_error': {'$first': { + '$cond': [{'$size': "$order_local.error"}, + "$order_local.error", [""]]}} + , 'tries': {'$first': { + '$cond': [{'$size': "$order_local.tries"}, + "$order_local.tries", + {'$cond': [ + {'$size': "$order_remote.tries"}, + "$order_remote.tries", + []] + }]}} }}, {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, + {"$limit": limit}, {"$skip": self._rec_loaded}, {"$project": self.projection} ] @@ -1106,9 +1194,70 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): "type": "representation", "_id": self._id, '$or': [{'files.path': {'$regex': regex_str, - '$options': 'i'}}] + '$options': 'i'}}] } + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'failed_remote_error': 1, + 'failed_local_error': 1, + 'tries': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } + } + class ImageDelegate(QtWidgets.QStyledItemDelegate): """ @@ -1116,6 +1265,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): """ def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) + self.icons = {} def paint(self, painter, option, index): d = index.data(QtCore.Qt.DisplayRole) @@ -1124,9 +1274,15 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): else: return - # log.info("data:: {} - {}".format(provider, value)) - pix_url = "../providers/resources/{}.png".format(provider) - pixmap = QtGui.QPixmap(pix_url) + if not self.icons.get(provider): + resource_path = os.path.dirname(__file__) + resource_path = os.path.join(resource_path, "..", + "providers", "resources") + pix_url = "{}/{}.png".format(resource_path, provider) + pixmap = QtGui.QPixmap(pix_url) + self.icons[provider] = pixmap + else: + pixmap = self.icons[provider] point = QtCore.QPoint(option.rect.x() + (option.rect.width() - pixmap.width()) / 2, @@ -1137,9 +1293,59 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): painter.setOpacity(0.5) overlay_rect = option.rect overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) - #painter.setCompositionMode(QtGui.QPainter.CompositionMode_DestinationOver) - #painter.setBrush(painter.brush(Qt.white)) - painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0,0,0,200))) + painter.fillRect(overlay_rect, + QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) + + +class SyncRepresentationErrorWindow(QtWidgets.QDialog): + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(250, 200) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg, + parent=self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Error Detail") + + +class SyncRepresentationErrorWidget(QtWidgets.QWidget): + """ + Dialog to show when sync error happened, prints error message + """ + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWidget, self).__init__(parent) + + layout = QtWidgets.QFormLayout(self) + layout.addRow(QtWidgets.QLabel("Last update date"), + QtWidgets.QLabel(pretty_timestamp(dt))) + layout.addRow(QtWidgets.QLabel("Retries"), + QtWidgets.QLabel(str(tries))) + layout.addRow(QtWidgets.QLabel("Error message"), + QtWidgets.QLabel(msg)) class SizeDelegate(QtWidgets.QStyledItemDelegate): @@ -1163,39 +1369,3 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) - -# Back up the reference to the exceptionhook -sys._excepthook = sys.excepthook - -def my_exception_hook(exctype, value, traceback): - # Print the error and traceback - print(exctype, value, traceback) - # Call the normal Exception hook after - sys._excepthook(exctype, value, traceback) - sys.exit(1) - -# Set the exception hook to our wrapping function -sys.excepthook = my_exception_hook - -if __name__ == '__main__': - import sys - from time import sleep - app = QtWidgets.QApplication(sys.argv) - #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) - os.environ["PYPE_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_DB"] = "avalon" - os.environ["AVALON_TIMEOUT"] = '3000' - - widget = SyncServerWindow() - widget.show() - - # while True: - # # run some codes that use QAxWidget.dynamicCall() function - # # print some results - # sleep(30) - - try: - sys.exit(app.exec_()) - except: - print("Exiting") From 9938f54da095c1166f878c6864b71ab8695a5b24 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 8 Jan 2021 13:04:16 +0100 Subject: [PATCH 05/16] use settings directly for getting sync sites in integrator --- pype/plugins/global/publish/integrate_new.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 133b4fc6ef..82690ed59d 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -896,7 +896,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): file_info = self.prepare_file_info(path, integrated_file_sizes[dest], - file_hash) + file_hash, + instance=instance) output_resources.append(file_info) return output_resources @@ -916,7 +917,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dest += '.{}'.format(self.TMP_FILE_EXT) return dest - def prepare_file_info(self, path, size=None, file_hash=None, sites=None): + def prepare_file_info(self, path, size=None, file_hash=None, sites=None, instance=None): """ Prepare information for one file (asset or resource) Arguments: @@ -933,15 +934,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): remote_site = None sync_server_presets = None - # manager = ModulesManager() - # sync_server = manager.modules_by_name["sync_server"] - # try: - # if sync_server.enabled: - # local_site, remote_site = sync_server.get_sites_for_project() - # except ValueError: - # log.debug(("There are not set presets for SyncServer." - # " No credentials provided, no synching possible"). - # format(str(sync_server_presets))) + if (instance.context.data["system_settings"] + ["modules"] + ["sync_server"] + ["enabled"]): + sync_server_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_server_presets["enabled"]: + local_site = sync_server_presets["config"].get("active_site", + "studio").strip() + remote_site = sync_server_presets["config"].get("remote_site") rec = { "_id": io.ObjectId(), From 60b2cb46cb8d761eb9d696917596fb2956f1983d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Jan 2021 18:12:12 +0100 Subject: [PATCH 06/16] SyncServer GUI - better handling when no project is configured --- pype/modules/sync_server/sync_server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 24dd6d4bf5..415f1e7d15 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -141,17 +141,19 @@ class SyncServer(PypeModule, ITrayModule): self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow() except ValueError: log.info("No system setting for sync. Not syncing.") + self.enabled = False except KeyError: log.info(( "There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.presets)), exc_info=True) - - from .tray.app import SyncServerWindow - self.widget = SyncServerWindow() + self.enabled = False def tray_start(self): """ @@ -192,6 +194,9 @@ class SyncServer(PypeModule, ITrayModule): ) def tray_menu(self, parent_menu): + if not self.enabled: + return + from Qt import QtWidgets """Add menu or action to Tray(or parent)'s menu""" action = QtWidgets.QAction("SyncServer", parent_menu) From a6e91195fa7866dee709971016d2afc63dfbb3ec Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Jan 2021 19:07:09 +0100 Subject: [PATCH 07/16] SyncServer GUI - safer creation of gui --- pype/modules/sync_server/tray/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index aa52b06da9..8b11445157 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -45,6 +45,7 @@ class SyncServerWindow(QtWidgets.QDialog): container = QtWidgets.QWidget() projects = SyncProjectListWidget(parent=self) + projects.refresh() # force selection of default repres = SyncRepresentationWidget(project=projects.current_project, parent=self) From 8cda04861778f90bda9fc427ac1d33db7800c710 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 15 Jan 2021 20:44:37 +0100 Subject: [PATCH 08/16] SyncServer GUI - fix - project propagates into detail model Removed project from Widget, left project on model only --- pype/modules/sync_server/tray/app.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 8b11445157..59543b121f 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,6 +1,5 @@ from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt -from avalon import style from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager @@ -134,7 +133,6 @@ class SyncRepresentationWidget(QtWidgets.QWidget): def __init__(self, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) - self.project = project self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -196,7 +194,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(_id, self.project) + detail_window = SyncServerDetailWindow(_id, + self.table_view.model()._project) detail_window.exec() def _on_context_menu(self, point): @@ -212,9 +211,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 19 REFRESH_SEC = 5000 DEFAULT_SORT = { - "context.asset": 1, - "context.subset": 1, - "context.version": 1, + "updated_dt_remote": -1, "_id": 1 } SORT_BY_COLUMN = [ @@ -740,7 +737,6 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.representation_id = _id self.item = None # set to item that mouse was clicked over - self.project = project manager = ModulesManager() self.sync_server = manager.modules_by_name["sync_server"] @@ -854,17 +850,19 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def _reset_local_site(self): log.info("reset local site: {}".format(self.item._id)) - self.sync_server.reset_provider_for_file(self.project, - self.representation_id, - self.item._id, - 'studio') # TEMP + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'local') def _reset_remote_site(self): log.info("reset remote site: {}".format(self.item._id)) - self.sync_server.reset_provider_for_file(self.project, - self.representation_id, - self.item._id, - 'gdrive') # TEMP + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'remote') class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): @@ -934,7 +932,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.projection = self.get_default_projection() self.query = self.get_default_query() - log.debug("!!! init query: {}".format(self.query)) + import bson.json_util + # log.debug("detail init query:: {}".format( + # bson.json_util.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) From 2f3a1eb9f0bd6797844be12b6d0be881bbc2207a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 15 Jan 2021 20:47:37 +0100 Subject: [PATCH 09/16] SyncServer GUI - implemented saving progress to DB --- pype/modules/sync_server/providers/gdrive.py | 98 ++++++++++++---- pype/modules/sync_server/sync_server.py | 114 ++++++++++++++----- 2 files changed, 163 insertions(+), 49 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 2207fdf3a3..44810b81d1 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -8,6 +8,7 @@ from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from pype.api import Logger from pype.api import get_system_settings from ..utils import time_function +import time SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly', 'https://www.googleapis.com/auth/drive.file', @@ -42,6 +43,7 @@ class GDriveHandler(AbstractProvider): """ FOLDER_STR = 'application/vnd.google-apps.folder' MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive + CHUNK_SIZE = 2097152 # must be divisible by 256! def __init__(self, site_name, tree=None, presets=None): self.presets = None @@ -277,7 +279,9 @@ class GDriveHandler(AbstractProvider): path = new_path_key return folder_id - def upload_file(self, source_path, path, overwrite=False): + def upload_file(self, source_path, path, + server, collection, file, representation, site, + overwrite=False): """ Uploads single file from 'source_path' to destination 'path'. It creates all folders on the path if are not existing. @@ -287,6 +291,13 @@ class GDriveHandler(AbstractProvider): path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -302,8 +313,8 @@ class GDriveHandler(AbstractProvider): path = os.path.dirname(path) else: target_name = os.path.basename(source_path) - file = self.file_path_exists(path + "/" + target_name) - if file and not overwrite: + target_file = self.file_path_exists(path + "/" + target_name) + if target_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -316,23 +327,45 @@ class GDriveHandler(AbstractProvider): } media = MediaFileUpload(source_path, mimetype='application/octet-stream', + chunksize=self.CHUNK_SIZE, resumable=True) try: - if not file: + if not target_file: # update doesnt like parent file_metadata['parents'] = [folder_id] - file = self.service.files().create(body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().create(body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') else: - file = self.service.files().update(fileId=file["id"], - body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().update(fileId=target_file["id"], + body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') + + media.stream() + log.debug("Start Upload! {}".format(source_path)) + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Uploaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = request.next_chunk() except errors.HttpError as ex: if ex.resp['status'] == '404': @@ -344,13 +377,14 @@ class GDriveHandler(AbstractProvider): log.warning("Forbidden received, hit quota. " "Injecting 60s delay.") - import time time.sleep(60) return False raise - return file["id"] + return response['id'] - def download_file(self, source_path, local_path, overwrite=False): + def download_file(self, source_path, local_path, + server, collection, file, representation, site, + overwrite=False): """ Downloads single file from 'source_path' (remote) to 'local_path'. It creates all folders on the local_path if are not existing. @@ -361,6 +395,13 @@ class GDriveHandler(AbstractProvider): local_path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -378,9 +419,9 @@ class GDriveHandler(AbstractProvider): else: # just folder, get file name from source target_name = os.path.basename(source_path) - file = os.path.isfile(local_path + "/" + target_name) + local_file = os.path.isfile(local_path + "/" + target_name) - if file and not overwrite: + if local_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -389,9 +430,24 @@ class GDriveHandler(AbstractProvider): with open(local_path + "/" + target_name, "wb") as fh: downloader = MediaIoBaseDownload(fh, request) - done = False - while done is False: - status, done = downloader.next_chunk() + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Downloaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = downloader.next_chunk() return target_name diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 415f1e7d15..49a572877c 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -92,6 +92,7 @@ class SyncServer(PypeModule, ITrayModule): # set 0 to no limit REPRESENTATION_LIMIT = 100 DEFAULT_SITE = 'studio' + LOG_PROGRESS_SEC = 5 # how often log progress to DB name = "sync_server" label = "Sync Server" @@ -139,13 +140,11 @@ class SyncServer(PypeModule, ITrayModule): try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) - self.sync_server_thread = SyncServerThread(self) - from .tray.app import SyncServerWindow self.widget = SyncServerWindow() except ValueError: - log.info("No system setting for sync. Not syncing.") + log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False except KeyError: log.info(( @@ -266,7 +265,8 @@ class SyncServer(PypeModule, ITrayModule): settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: - log.info("No project setting for Sync Server, not syncing.") + log.info("No project setting for {}, not syncing.". + format(project_name)) return {} if sync_settings.get("enabled"): return sync_settings @@ -427,8 +427,8 @@ class SyncServer(PypeModule, ITrayModule): return SyncStatus.DO_NOTHING - async def upload(self, file, representation, provider_name, site_name, - tree=None, preset=None): + async def upload(self, collection, file, representation, provider_name, + site_name, tree=None, preset=None): """ Upload single 'file' of a 'representation' to 'provider'. Source url is taken from 'file' portion, where {root} placeholder @@ -439,6 +439,7 @@ class SyncServer(PypeModule, ITrayModule): from GDrive), 'created_dt' - time of upload Args: + collection (str): source collection file (dictionary): of file from representation in Mongo representation (dictionary): of representation provider_name (string): gdrive, gdc etc. @@ -468,21 +469,28 @@ class SyncServer(PypeModule, ITrayModule): err = "Folder {} wasn't created. Check permissions.".\ format(target_folder) raise NotADirectoryError(err) - + _, remote_site = self.get_sites_for_project(collection) loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.upload_file, local_file, remote_file, - True) + self, + collection, + file, + representation, + remote_site, + True + ) return file_id - async def download(self, file, representation, provider_name, + async def download(self, collection, file, representation, provider_name, site_name, tree=None, preset=None): """ Downloads file to local folder denoted in representation.Context. Args: + collection (str): source collection file (dictionary) : info about processed file representation (dictionary): repr that 'file' belongs to provider_name (string): 'gdrive' etc @@ -506,26 +514,37 @@ class SyncServer(PypeModule, ITrayModule): local_folder = os.path.dirname(local_file) os.makedirs(local_folder, exist_ok=True) + local_site, _ = self.get_sites_for_project(collection) + loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.download_file, remote_file, local_file, - False) + False, + self, + collection, + file, + representation, + local_site + ) return file_id - def update_db(self, new_file_id, file, representation, provider_name, - error=None): + def update_db(self, collection, new_file_id, file, representation, + site, error=None, progress=None): """ Update 'provider' portion of records in DB with success (file_id) or error (exception) Args: + collection (string): name of project - force to db connection as + each file might come from different collection new_file_id (string): file (dictionary): info about processed file (pulled from DB) representation (dictionary): parent repr of file (from DB) - provider_name (string): label ('gdrive', 'S3') + site (string): label ('gdrive', 'S3') error (string): exception message + progress (float): 0-1 of progress of upload/download Returns: None @@ -539,26 +558,33 @@ class SyncServer(PypeModule, ITrayModule): file_index, _ = self._get_file_info(representation.get('files', []), file_id) site_index, _ = self._get_provider_rec(file.get('sites', []), - provider_name) + site) update = {} if new_file_id: update["$set"] = self._get_success_dict(file_index, site_index, new_file_id) # reset previous errors if any update["$unset"] = self._get_error_dict(file_index, site_index, - "", "") + "", "", "") + elif progress is not None: + update["$set"] = self._get_progress_dict(file_index, site_index, + progress) else: - tries = self._get_tries_count(file, provider_name) + tries = self._get_tries_count(file, site) tries += 1 update["$set"] = self._get_error_dict(file_index, site_index, error, tries) + self.connection.Session["AVALON_PROJECT"] = collection self.connection.update_one( query, update ) + if progress is not None: + return + status = 'failed' error_str = 'with error {}'.format(error) if new_file_id: @@ -574,7 +600,7 @@ class SyncServer(PypeModule, ITrayModule): def _get_file_info(self, files, _id): """ Return record from list of records which name matches to 'provider' - Could be possibly refactored with '_get_file_info' together. + Could be possibly refactored with '_get_provider_rec' together. Args: files (list): of dictionaries with info about published files @@ -611,7 +637,7 @@ class SyncServer(PypeModule, ITrayModule): return -1, None def reset_provider_for_file(self, collection, representation_id, - file_id, site_name): + file_id, side): """ Reset information about synchronization for particular 'file_id' and provider. @@ -620,7 +646,7 @@ class SyncServer(PypeModule, ITrayModule): collection (string): name of project (eg. collection) in DB representation_id(string): _id of representation file_id (string): file _id in representation - site_name (string): 'gdrive', 'S3' etc + side (string): local or remote side Returns: None """ @@ -634,6 +660,12 @@ class SyncServer(PypeModule, ITrayModule): raise ValueError("Representation {} not found in {}". format(representation_id, collection)) + local_site, remote_site = self.get_active_sites(collection) + if side == 'local': + site_name = local_site + else: + site_name = remote_site + files = representation[0].get('files', []) file_index, _ = self._get_file_info(files, file_id) @@ -685,7 +717,8 @@ class SyncServer(PypeModule, ITrayModule): datetime.utcnow()} return val - def _get_error_dict(self, file_index, site_index, error="", tries=""): + def _get_error_dict(self, file_index, site_index, + error="", tries="", progress=""): """ Provide error metadata to be stored in Db. Used for set (error and tries provided) or unset mode. @@ -700,7 +733,9 @@ class SyncServer(PypeModule, ITrayModule): val = {"files.{}.sites.{}.last_failed_dt". format(file_index, site_index): datetime.utcnow(), "files.{}.sites.{}.error".format(file_index, site_index): error, - "files.{}.sites.{}.tries".format(file_index, site_index): tries + "files.{}.sites.{}.tries".format(file_index, site_index): tries, + "files.{}.sites.{}.progress".format(file_index, site_index): + progress } return val @@ -728,6 +763,22 @@ class SyncServer(PypeModule, ITrayModule): _, rec = self._get_provider_rec(file.get("sites", []), provider) return rec.get("tries", 0) + def _get_progress_dict(self, file_index, site_index, progress): + """ + Provide progress metadata to be stored in Db. + Used during upload/download for GUI to show. + Args: + file_index: (int) - index of modified file + site_index: (int) - index of modified site of modified file + progress: (float) - 0-1 progress of upload/download + Returns: + (dictionary) + """ + val = {"files.{}.sites.{}.progress". + format(file_index, site_index): progress + } + return val + def _get_local_file_path(self, file, local_root): """ Auxiliary function for replacing rootless path with real path @@ -873,23 +924,27 @@ class SyncServerThread(threading.Thread): tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.upload(file, + self.module.upload(collection, + file, sync, provider, site, tree, site_preset)) task_files_to_process.append(task) - # store info for exception handling + # store info for exception handlingy files_processed_info.append((file, sync, - site)) + site, + collection + )) processed_file_path.add(file_path) if status == SyncStatus.DO_DOWNLOAD: tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.download(file, + self.module.download(collection, + file, sync, provider, site, @@ -899,7 +954,9 @@ class SyncServerThread(threading.Thread): files_processed_info.append((file, sync, - local)) + local, + collection + )) processed_file_path.add(file_path) log.debug("Sync tasks count {}". @@ -909,12 +966,13 @@ class SyncServerThread(threading.Thread): return_exceptions=True) for file_id, info in zip(files_created, files_processed_info): - file, representation, site = info + file, representation, site, collection = info error = None if isinstance(file_id, BaseException): error = str(file_id) file_id = None - self.module.update_db(file_id, + self.module.update_db(collection, + file_id, file, representation, site, From c7c3d6b78d54b7f78221899ea88be1b4dba21585 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Jan 2021 14:14:11 +0100 Subject: [PATCH 10/16] SyncServer GUI - fixes for style --- pype/modules/sync_server/tray/app.py | 13 ++++++- pype/tools/settings/settings/style/style.css | 36 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 59543b121f..f83d90fd3b 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -6,6 +6,7 @@ from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style +#from avalon import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -1269,6 +1270,16 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): self.icons = {} def paint(self, painter, option, index): + option = QtWidgets.QStyleOptionViewItem(option) + option.showDecorationSelected = True + + if (option.showDecorationSelected and + (option.state & QtWidgets.QStyle.State_Selected)): + painter.setOpacity(0.20) # highlight color is a bit off + painter.fillRect(option.rect, + option.palette.highlight()) + painter.setOpacity(1) + d = index.data(QtCore.Qt.DisplayRole) if d: provider, value = d.split() @@ -1296,7 +1307,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) - + painter.setOpacity(1) class SyncRepresentationErrorWindow(QtWidgets.QDialog): def __init__(self, _id, project, dt, tries, msg, parent=None): diff --git a/pype/tools/settings/settings/style/style.css b/pype/tools/settings/settings/style/style.css index f3eb3a258e..3ce9837a8b 100644 --- a/pype/tools/settings/settings/style/style.css +++ b/pype/tools/settings/settings/style/style.css @@ -353,3 +353,39 @@ QScrollBar::up-arrow:vertical, QScrollBar::down-arrow:vertical { QScrollBar::add-page:vertical, QScrollBar::sub-page:vertical { background: none; } + +QTableView +{ + border: 1px solid #444; + gridline-color: #6c6c6c; + background-color: #201F1F; + alternate-background-color:#21252B; +} + +QHeaderView +{ + border: 1px transparent; + border-radius: 2px; + margin: 0px; + padding: 0px; +} + +QHeaderView::section { + background-color: #21252B; + /*color: silver;*/ + padding: 4px; + border: 1px solid #6c6c6c; + border-radius: 0px; + text-align: center; + color: #969b9e; + font-weight: bold; +} + +QTableView::item:pressed, QListView::item:pressed, QTreeView::item:pressed { + background: #78879b; + color: #FFFFFF; +} + +QTableView::item:selected:active, QTreeView::item:selected:active, QListView::item:selected:active { + background: #3d8ec9; +} \ No newline at end of file From 21e285c2cc2d72b05186e96de5867ad17cdc56c2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Jan 2021 19:06:57 +0100 Subject: [PATCH 11/16] SyncServer GUI - fix - reset_provider_for_file Removed pulling sync_server from ModulesManager (not a singleton!) to injecting object and passing everywhere --- pype/modules/sync_server/sync_server.py | 14 ++-- pype/modules/sync_server/tray/app.py | 92 +++++++++++++------------ 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 49a572877c..b999518103 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -118,7 +118,6 @@ class SyncServer(PypeModule, ITrayModule): self.sync_server_thread = None # asyncio requires new thread self.action_show_widget = None - self.connection = AvalonMongoDB() def connect_with_modules(self, *_a, **kw): return @@ -136,13 +135,14 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() self.connection = AvalonMongoDB() + self.connection.install() try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) from .tray.app import SyncServerWindow - self.widget = SyncServerWindow() + self.widget = SyncServerWindow(self) except ValueError: log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False @@ -654,13 +654,13 @@ class SyncServer(PypeModule, ITrayModule): query = { "_id": ObjectId(representation_id) } - self.connection.Session["AVALON_PROJECT"] = collection - representation = list(self.connection.find(query)) + + representation = list(self.connection.database[collection].find(query)) if not representation: raise ValueError("Representation {} not found in {}". format(representation_id, collection)) - local_site, remote_site = self.get_active_sites(collection) + local_site, remote_site = self.get_sites_for_project(collection) if side == 'local': site_name = local_site else: @@ -672,7 +672,7 @@ class SyncServer(PypeModule, ITrayModule): site_index, _ = self._get_provider_rec(files[file_index]. get('sites', []), site_name) - if file_index > 0 and site_index > 0: + if file_index >= 0 and site_index >= 0: elem = {"name": site_name} update = { "$set": {"files.{}.sites.{}".format(file_index, site_index): @@ -680,7 +680,7 @@ class SyncServer(PypeModule, ITrayModule): } } - self.connection.update_one( + self.connection.database[collection].update_one( query, update ) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index f83d90fd3b..09f7b9e3af 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,12 +1,9 @@ from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt -from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget -from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style -#from avalon import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -30,7 +27,7 @@ class SyncServerWindow(QtWidgets.QDialog): Main window that contains list of synchronizable projects and summary view with all synchronizable representations for first project """ - def __init__(self, parent=None): + def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) @@ -39,14 +36,15 @@ class SyncServerWindow(QtWidgets.QDialog): self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1400, 800) - body = QtWidgets.QWidget() - footer = QtWidgets.QWidget() + body = QtWidgets.QWidget(self) + footer = QtWidgets.QWidget(self) footer.setFixedHeight(20) container = QtWidgets.QWidget() - projects = SyncProjectListWidget(parent=self) + projects = SyncProjectListWidget(sync_server, self) projects.refresh() # force selection of default - repres = SyncRepresentationWidget(project=projects.current_project, + repres = SyncRepresentationWidget(sync_server, + project=projects.current_project, parent=self) container_layout = QtWidgets.QHBoxLayout(container) @@ -63,7 +61,7 @@ class SyncServerWindow(QtWidgets.QDialog): body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) - message = QtWidgets.QLabel() + message = QtWidgets.QLabel(footer) message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) @@ -86,6 +84,9 @@ class SyncProjectListWidget(ProjectListWidget): """ Lists all projects that are synchronized to choose from """ + def __init__(self, sync_server, parent): + super(SyncProjectListWidget, self).__init__(parent) + self.sync_server = sync_server def validate_context_change(self): return True @@ -93,13 +94,11 @@ class SyncProjectListWidget(ProjectListWidget): def refresh(self): model = self.project_list.model() model.clear() - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] - for project_name in sync_server.get_synced_presets().keys(): + for project_name in self.sync_server.get_synced_presets().keys(): model.appendRow(QtGui.QStandardItem(project_name)) - if len(sync_server.get_synced_presets().keys()) == 0: + if len(self.sync_server.get_synced_presets().keys()) == 0: model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( @@ -132,9 +131,11 @@ class SyncRepresentationWidget(QtWidgets.QWidget): ("state", 50) ) - def __init__(self, project=None, parent=None): + def __init__(self, sync_server, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) + self.sync_server = sync_server + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -144,7 +145,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - model = SyncRepresentationModel(headers, project) + model = SyncRepresentationModel(sync_server, headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( @@ -185,17 +186,23 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addWidget(self.table_view) self.table_view.doubleClicked.connect(self._doubleClicked) + self.table_view.clicked.connect(self._clicked) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) + def _clicked(self, index): + print("clicked") + self.table_view.model()._selected_id = self.table_view.model().\ + data(index, Qt.UserRole) + def _doubleClicked(self, index): """ Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(_id, + detail_window = SyncServerDetailWindow(self.sync_server, _id, self.table_view.model()._project) detail_window.exec() @@ -253,7 +260,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): priority = attr.ib(default=None) state = attr.ib(default=None) - def __init__(self, header, project=None): + def __init__(self, sync_server, header, project=None): super(SyncRepresentationModel, self).__init__() self._header = header self._data = [] @@ -261,19 +268,15 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) self.filter = None + self._selected_id = None self._initialized = False - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project - - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project(self._project) + self.sync_server.get_sites_for_project(self._project) self.projection = self.get_default_projection() @@ -290,6 +293,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.timer.timeout.connect(self.tick) self.timer.start(self.REFRESH_SEC) + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + def data(self, index, role): item = self._data[index.row()] @@ -456,7 +463,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): project (str): name of project """ self._project = project - self.dbcon.Session["AVALON_PROJECT"] = self._project self.refresh() def get_default_query(self, limit=0): @@ -672,7 +678,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, _id, project, parent=None): + def __init__(self, sync_server, _id, project, parent=None): log.debug( "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) @@ -687,11 +693,8 @@ class SyncServerDetailWindow(QtWidgets.QDialog): footer = QtWidgets.QWidget() footer.setFixedHeight(20) - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = None - - container = SyncRepresentationDetailWidget(_id, project, parent=self) + container = SyncRepresentationDetailWidget(sync_server, _id, project, + parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -730,17 +733,16 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): ("remote_site", 60), ("size", 60), ("priority", 20), - ("state", 50) + ("state", 90) ) - def __init__(self, _id=None, project=None, parent=None): + def __init__(self, sync_server, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) self.representation_id = _id self.item = None # set to item that mouse was clicked over - manager = ModulesManager() - self.sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -751,7 +753,8 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - model = SyncRepresentationDetailModel(headers, _id, project) + model = SyncRepresentationDetailModel(sync_server, headers, _id, + project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( @@ -905,7 +908,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): tries = attr.ib(default=None) error = attr.ib(default=None) - def __init__(self, header, _id, project=None): + def __init__(self, sync_server, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() self._header = header self._data = [] @@ -916,16 +919,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self._id = _id self._initialized = False - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project - - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project(self._project) + self.sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT @@ -943,6 +941,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.timer.timeout.connect(self.tick) self.timer.start(SyncRepresentationModel.REFRESH_SEC) + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + def tick(self): self.refresh(representations=None, load_records=self._rec_loaded) self.timer.start(SyncRepresentationModel.REFRESH_SEC) @@ -1026,7 +1028,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): errors.append(repre.get('failed_local_error')) item = self.SyncRepresentationDetail( - repre.get("_id"), + file.get("_id"), os.path.basename(file["path"]), local_updated, remote_updated, From 51ae46bd6d7d41fe4bd52d546ee90a4e9d135c93 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 12:21:08 +0100 Subject: [PATCH 12/16] SyncServer GUI - fix - keep selection during model refresh --- pype/modules/sync_server/sync_server.py | 5 +- pype/modules/sync_server/tray/app.py | 109 +++++++++++++++++++++--- 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index b999518103..984e5e34a6 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -117,6 +117,9 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread + self.connection = AvalonMongoDB() + self.connection.install() + self.action_show_widget = None def connect_with_modules(self, *_a, **kw): @@ -134,8 +137,6 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() - self.connection = AvalonMongoDB() - self.connection.install() try: self.presets = self.get_synced_presets() diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 09f7b9e3af..80680af749 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -136,6 +136,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.sync_server = sync_server + self._selected_id = None # keep last selected _id + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -185,19 +187,37 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addLayout(top_bar_layout) layout.addWidget(self.table_view) - self.table_view.doubleClicked.connect(self._doubleClicked) - self.table_view.clicked.connect(self._clicked) + self.table_view.doubleClicked.connect(self._double_clicked) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) - def _clicked(self, index): - print("clicked") - self.table_view.model()._selected_id = self.table_view.model().\ - data(index, Qt.UserRole) + self.table_view.model().modelReset.connect(self._set_selection) - def _doubleClicked(self, index): + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self, new_selection): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index and index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + + def _double_clicked(self, index): """ Opens representation dialog with all files after doubleclick """ @@ -268,7 +288,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) self.filter = None - self._selected_id = None self._initialized = False @@ -465,6 +484,25 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._project = project self.refresh() + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + def get_default_query(self, limit=0): """ Returns basic aggregate query for main table. @@ -744,6 +782,8 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.sync_server = sync_server + self._selected_id = None + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -799,6 +839,30 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view.customContextMenuRequested.connect( self._on_context_menu) + self.table_view.model().modelReset.connect(self._set_selection) + + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + def _show_detail(self): """ Shows windows with error message for failed sync of a file. @@ -853,7 +917,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): to_run() def _reset_local_site(self): - log.info("reset local site: {}".format(self.item._id)) + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, @@ -861,7 +928,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): 'local') def _reset_remote_site(self): - log.info("reset remote site: {}".format(self.item._id)) + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, @@ -1093,6 +1163,25 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = filter self.refresh() + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or From 2eed46847a6cf4001802919d46deb144697ae7f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 13:09:10 +0100 Subject: [PATCH 13/16] SyncServer GUI - changed order by status Currently processed sorted as first now --- pype/modules/sync_server/tray/app.py | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 80680af749..313cb80c6a 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -13,9 +13,9 @@ import json log = PypeLogger().get_logger("SyncServer") STATUS = { - 0: 'Queued', + 0: 'In Progress', 1: 'Failed', - 2: 'In Progress', + 2: 'Queued', 3: 'Paused', 4: 'Synced OK', -1: 'Not available' @@ -516,10 +516,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'local_site' - progress of repr on local side, 1 = finished 'remote_site' - progress on remote side, calculates from files 'state' - - 0 - queued + 0 - in progress 1 - failed - 2 - paused (not implemented yet) - 3 - in progress + 2 - queued + 3 - paused (not implemented yet) 4 - finished on both sides are calculated and must be calculated in DB because of @@ -603,10 +603,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): , 'failed_local': {'$sum': '$failed_local'} , 'updated_dt_local': {'$max': "$updated_dt_local"} }}, - {"$sort": self.sort}, {"$limit": limit}, {"$skip": self._rec_loaded}, - {"$project": self.projection} + {"$project": self.projection}, + {"$sort": self.sort} ] def _get_match_part(self): @@ -678,12 +678,12 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'case': { '$or': [{'$eq': ['$avg_progress_remote', 0]}, {'$eq': ['$avg_progress_local', 0]}]}, - 'then': 0 + 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 + 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ @@ -695,18 +695,18 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): {'$lt': ['$avg_progress_local', 1]} ]} ]}, - 'then': 2 + 'then': 0 # In progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 + 'then': 3 # Paused }, { 'case': {'$and': [ {'$eq': ['$avg_progress_remote', 1]}, {'$eq': ['$avg_progress_local', 1]} ]}, - 'then': 4 + 'then': 4 # Synced OK }, ], 'default': -1 @@ -1262,10 +1262,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): []] }]}} }}, - {"$sort": self.sort}, {"$limit": limit}, {"$skip": self._rec_loaded}, - {"$project": self.projection} + {"$project": self.projection}, + {"$sort": self.sort} ] def _get_match_part(self): @@ -1315,12 +1315,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'case': { '$or': [{'$eq': ['$progress_remote', 0]}, {'$eq': ['$progress_local', 0]}]}, - 'then': 0 + 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 + 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ @@ -1332,7 +1332,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$lt': ['$progress_local', 1]} ]} ]}, - 'then': 2 + 'then': 0 # In Progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, @@ -1343,7 +1343,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$eq': ['$progress_remote', 1]}, {'$eq': ['$progress_local', 1]} ]}, - 'then': 4 + 'then': 4 # Synced OK }, ], 'default': -1 From 01344854bea40b8acfc7a901b7a7a4b4b9f9b68b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 13:35:10 +0100 Subject: [PATCH 14/16] SyncServer GUI - safer initialization of DB --- pype/modules/sync_server/sync_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 984e5e34a6..84637a1d62 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -117,9 +117,6 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread - self.connection = AvalonMongoDB() - self.connection.install() - self.action_show_widget = None def connect_with_modules(self, *_a, **kw): @@ -138,6 +135,9 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() + self.connection = AvalonMongoDB() + self.connection.install() + try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) From 83a8c6e3a1be6f03ee6390141dc622f6eb0b0e9c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 14:01:51 +0100 Subject: [PATCH 15/16] SyncServer GUI - Hound --- pype/modules/sync_server/providers/gdrive.py | 4 +- pype/modules/sync_server/tray/app.py | 52 +++++++++++--------- pype/plugins/global/publish/integrate_new.py | 16 +++--- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 44810b81d1..5bc6f21b38 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -383,8 +383,8 @@ class GDriveHandler(AbstractProvider): return response['id'] def download_file(self, source_path, local_path, - server, collection, file, representation, site, - overwrite=False): + server, collection, file, representation, site, + overwrite=False): """ Downloads single file from 'source_path' (remote) to 'local_path'. It creates all folders on the local_path if are not existing. diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 313cb80c6a..31dc8744e1 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -212,7 +212,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): index = self.table_view.model().get_index(self._selected_id) if index and index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None @@ -716,7 +716,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, sync_server, _id, project, parent=None): + def __init__(self, sync_server, _id, project, parent=None): log.debug( "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) @@ -1001,9 +1001,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.projection = self.get_default_projection() self.query = self.get_default_query() - import bson.json_util - # log.debug("detail init query:: {}".format( - # bson.json_util.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) @@ -1226,23 +1223,31 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): [0]]}]}} # file might be successfully created or failed, not both , 'updated_dt_remote': {'$first': { - '$cond': [{'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - } - ]}} + '$cond': [ + {'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + { + '$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ] + } + ] + }} , 'updated_dt_local': {'$first': { - '$cond': [{'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - } - ]}} + '$cond': [ + {'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + { + '$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ] + } + ] + }} , 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} , 'failed_local': { @@ -1286,8 +1291,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): return { "type": "representation", "_id": self._id, - '$or': [{'files.path': {'$regex': regex_str, - '$options': 'i'}}] + '$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}] } def get_default_projection(self): @@ -1400,6 +1404,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) painter.setOpacity(1) + class SyncRepresentationErrorWindow(QtWidgets.QDialog): def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWindow, self).__init__(parent) @@ -1471,4 +1476,3 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): return "%3.1f%s%s" % (value, unit, suffix) value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) - diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 2b867e0116..5ba92435fd 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -929,7 +929,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dest += '.{}'.format(self.TMP_FILE_EXT) return dest - def prepare_file_info(self, path, size=None, file_hash=None, sites=None, instance=None): + def prepare_file_info(self, path, size=None, file_hash=None, + sites=None, instance=None): """ Prepare information for one file (asset or resource) Arguments: @@ -939,6 +940,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sites(optional): array of published locations, [ {'name':'studio', 'created_dt':date} by default keys expected ['studio', 'site1', 'gdrive1'] + instance(dict, optional): to get collected settings Returns: rec: dictionary with filled info """ @@ -947,17 +949,17 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sync_server_presets = None if (instance.context.data["system_settings"] - ["modules"] - ["sync_server"] - ["enabled"]): + ["modules"] + ["sync_server"] + ["enabled"]): sync_server_presets = (instance.context.data["project_settings"] ["global"] ["sync_server"]) if sync_server_presets["enabled"]: - local_site = sync_server_presets["config"].get("active_site", - "studio").strip() - remote_site = sync_server_presets["config"].get("remote_site") + local_site = sync_server_presets["config"].\ + get("active_site", "studio").strip() + remote_site = sync_server_presets["config"].get("remote_site") rec = { "_id": io.ObjectId(), From 46f09c8cb2f069d932b4626ed3f2d5e8d313cd34 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 16:46:33 +0100 Subject: [PATCH 16/16] SyncServer GUI - Hound 2. --- pype/modules/sync_server/tray/app.py | 261 +++++++++++++++------------ 1 file changed, 146 insertions(+), 115 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 31dc8744e1..afd103f9d5 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -27,6 +27,7 @@ class SyncServerWindow(QtWidgets.QDialog): Main window that contains list of synchronizable projects and summary view with all synchronizable representations for first project """ + def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) @@ -84,6 +85,7 @@ class SyncProjectListWidget(ProjectListWidget): """ Lists all projects that are synchronized to choose from """ + def __init__(self, sync_server, parent): super(SyncProjectListWidget, self).__init__(parent) self.sync_server = sync_server @@ -105,7 +107,7 @@ class SyncProjectListWidget(ProjectListWidget): QtCore.Qt.DisplayRole ) if not self.current_project: - self.current_project = self.project_list.model().item(0).\ + self.current_project = self.project_list.model().item(0). \ data(QtCore.Qt.DisplayRole) @@ -114,7 +116,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Summary dialog with list of representations that matches current settings 'local_site' and 'remote_site'. """ - active_changed = QtCore.Signal() # active index changed + active_changed = QtCore.Signal() # active index changed default_widths = ( ("asset", 210), @@ -222,8 +224,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(self.sync_server, _id, - self.table_view.model()._project) + detail_window = SyncServerDetailWindow( + self.sync_server, _id, self.table_view.model()._project) detail_window.exec() def _on_context_menu(self, point): @@ -243,18 +245,18 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "_id": 1 } SORT_BY_COLUMN = [ - "context.asset", # asset - "context.subset", # subset - "context.version", # version - "context.representation", # representation - "updated_dt_local", # local created_dt - "updated_dt_remote", # remote created_dt - "avg_progress_local", # local progress - "avg_progress_remote", # remote progress - "files_count", # count of files - "files_size", # file size of all files - "context.asset", # priority TODO - "status" # state + "context.asset", # asset + "context.subset", # subset + "context.version", # version + "context.representation", # representation + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "avg_progress_local", # local progress + "avg_progress_remote", # remote progress + "files_count", # count of files + "files_size", # file size of all files + "context.asset", # priority TODO + "status" # state ] numberPopulated = QtCore.Signal(int) @@ -543,8 +545,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { + }}, + 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} @@ -554,54 +556,68 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} - , 'progress_local': {'$first': { + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { + 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, "$order_remote.created_dt", {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - }]}} - , 'updated_dt_local': {'$first': { + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ]} + ]}}, + 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - }]}} - , 'files_size': {'$ifNull': ["$files.size", 0]} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ]} + ]}}, + 'files_size': {'$ifNull': ["$files.size", 0]}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]} }}, {'$group': { - '_id': '$_id' + '_id': '$_id', # pass through context - same for representation - , 'context': {'$addToSet': '$context'} + 'context': {'$addToSet': '$context'}, # pass through files as a list - , 'files': {'$addToSet': '$files'} + 'files': {'$addToSet': '$files'}, # count how many files - , 'files_count': {'$sum': 1} - , 'files_size': {'$sum': '$files_size'} + 'files_count': {'$sum': 1}, + 'files_size': {'$sum': '$files_size'}, # sum avg progress, finished = 1 - , 'avg_progress_remote': {'$avg': "$progress_remote"} - , 'avg_progress_local': {'$avg': "$progress_local"} + 'avg_progress_remote': {'$avg': "$progress_remote"}, + 'avg_progress_local': {'$avg': "$progress_local"}, # select last touch of file - , 'updated_dt_remote': {'$max': "$updated_dt_remote"} - , 'failed_remote': {'$sum': '$failed_remote'} - , 'failed_local': {'$sum': '$failed_local'} - , 'updated_dt_local': {'$max': "$updated_dt_local"} + 'updated_dt_remote': {'$max': "$updated_dt_remote"}, + 'failed_remote': {'$sum': '$failed_remote'}, + 'failed_local': {'$sum': '$failed_local'}, + 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, @@ -634,12 +650,11 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): regex_str = '.*{}.*'.format(self.filter) return { "type": "representation", - '$or': [{'context.subset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.asset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], + '$or': [ + {'context.subset': {'$regex': regex_str, '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], 'files.sites': { '$elemMatch': { '$or': [ @@ -687,9 +702,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): }, { 'case': {'$or': [{'$and': [ - {'$gt': ['$avg_progress_remote', 0]}, - {'$lt': ['$avg_progress_remote', 1]} - ]}, + {'$gt': ['$avg_progress_remote', 0]}, + {'$lt': ['$avg_progress_remote', 1]} + ]}, {'$and': [ {'$gt': ['$avg_progress_local', 0]}, {'$lt': ['$avg_progress_local', 1]} @@ -761,7 +776,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): project (str): name of project with repre parent (QDialog): SyncServerDetailWindow """ - active_changed = QtCore.Signal() # active index changed + active_changed = QtCore.Signal() # active index changed default_widths = ( ("file", 290), @@ -858,7 +873,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): index = self.table_view.model().get_index(self._selected_id) if index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None @@ -950,13 +965,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): } SORT_BY_COLUMN = [ "files.path", - "updated_dt_local", # local created_dt - "updated_dt_remote", # remote created_dt - "progress_local", # local progress - "progress_remote", # remote progress - "size", # remote progress - "context.asset", # priority TODO - "status" # state + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "progress_local", # local progress + "progress_remote", # remote progress + "size", # remote progress + "context.asset", # priority TODO + "status" # state ] @attr.s @@ -1200,8 +1215,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { + }}, + 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} @@ -1211,61 +1226,74 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, - [1], - [0]]}]}} - , 'progress_local': {'$first': { + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, - [1], - [0]]}]}} + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { + 'updated_dt_remote': {'$first': { '$cond': [ - {'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - { - '$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - [] - ] - } - ] - }} - , 'updated_dt_local': {'$first': { + {'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + { + '$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ] + } + ] + }}, + 'updated_dt_local': {'$first': { '$cond': [ - {'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - { - '$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - [] - ] - } - ] - }} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} - , 'failed_remote_error': {'$first': { + {'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + { + '$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ] + } + ] + }}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]}, + 'failed_remote_error': {'$first': { '$cond': [{'$size': "$order_remote.error"}, - "$order_remote.error", [""]]}} - , 'failed_local_error': {'$first': { + "$order_remote.error", + [""]]}}, + 'failed_local_error': {'$first': { '$cond': [{'$size': "$order_local.error"}, - "$order_local.error", [""]]}} - , 'tries': {'$first': { + "$order_local.error", + [""]]}}, + 'tries': {'$first': { '$cond': [{'$size': "$order_local.tries"}, "$order_local.tries", {'$cond': [ {'$size': "$order_remote.tries"}, "$order_remote.tries", - []] - }]}} + [] + ]} + ]}} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, @@ -1360,6 +1388,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): """ Prints icon of site and progress of synchronization """ + def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) self.icons = {} @@ -1444,6 +1473,7 @@ class SyncRepresentationErrorWidget(QtWidgets.QWidget): """ Dialog to show when sync error happened, prints error message """ + def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWidget, self).__init__(parent) @@ -1460,6 +1490,7 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): """ Pretty print for file size """ + def __init__(self, parent=None): super(SizeDelegate, self).__init__(parent)