From e6c7382c7ee3d3737b1cfc1e1d6fb7b335246be5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 21 Dec 2020 13:22:54 +0100 Subject: [PATCH 01/50] #817 - Initial Gui implementation Implemented dialog for sync representations Implemented model, pagination, sorting (for most columns) Base of detail dialog --- pype/modules/sync_server/sync_server.py | 24 +- pype/modules/sync_server/tray/app.py | 490 ++++++++++++++++++++++++ 2 files changed, 512 insertions(+), 2 deletions(-) create mode 100644 pype/modules/sync_server/tray/app.py diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 167be665f5..24dd6d4bf5 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -18,7 +18,7 @@ from .utils import time_function import six from pype.lib import PypeLogger -from .. import PypeModule, ITrayService +from .. import PypeModule, ITrayModule if six.PY2: web = asyncio = STATIC_DIR = WebSocketAsync = None @@ -34,7 +34,7 @@ class SyncStatus(Enum): DO_DOWNLOAD = 2 -class SyncServer(PypeModule, ITrayService): +class SyncServer(PypeModule, ITrayModule): """ Synchronization server that is syncing published files from local to any of implemented providers (like GDrive, S3 etc.) @@ -116,6 +116,9 @@ class SyncServer(PypeModule, ITrayService): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread + self.action_show_widget = None + self.connection = AvalonMongoDB() + def connect_with_modules(self, *_a, **kw): return @@ -147,6 +150,9 @@ class SyncServer(PypeModule, ITrayService): "no syncing possible"). format(str(self.presets)), exc_info=True) + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow() + def tray_start(self): """ Triggered when Tray is started. @@ -185,6 +191,16 @@ class SyncServer(PypeModule, ITrayService): exc_info=True ) + def tray_menu(self, parent_menu): + from Qt import QtWidgets + """Add menu or action to Tray(or parent)'s menu""" + action = QtWidgets.QAction("SyncServer", parent_menu) + action.triggered.connect(self.show_widget) + parent_menu.addAction(action) + parent_menu.addSeparator() + + self.action_show_widget = action + @property def is_running(self): return self.sync_server_thread.is_running @@ -641,6 +657,10 @@ class SyncServer(PypeModule, ITrayService): """ return int(self.presets[project_name]["config"]["loop_delay"]) + def show_widget(self): + """Show dialog to enter credentials""" + self.widget.show() + def _get_success_dict(self, file_index, site_index, new_file_id): """ Provide success metadata ("id", "created_dt") to be stored in Db. diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py new file mode 100644 index 0000000000..9cb021c270 --- /dev/null +++ b/pype/modules/sync_server/tray/app.py @@ -0,0 +1,490 @@ +from Qt import QtWidgets, QtCore, QtGui +from Qt.QtCore import Qt +from avalon import style +from avalon.api import AvalonMongoDB +from pype.tools.settings.settings.widgets.base import ProjectListWidget +from pype.modules import ModulesManager + +from pype.lib import PypeLogger +log = PypeLogger().get_logger("SyncServer") + + +class SyncServerWindow(QtWidgets.QDialog): + def __init__(self, parent=None): + super(SyncServerWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.resize(1400, 800) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = QtWidgets.QWidget() + projects = SyncProjectListWidget(self) + repres = SyncRepresentationWidget(self) + + container_layout = QtWidgets.QHBoxLayout(container) + container_layout.setContentsMargins(0, 0, 0, 0) + split = QtWidgets.QSplitter() + split.addWidget(projects) + split.addWidget(repres) + split.setSizes([180, 950, 200]) + container_layout.addWidget(split) + + container.setLayout(container_layout) + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = None + + # Project + self.combo_projects = QtWidgets.QComboBox() + + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Server") + + +class SyncProjectListWidget(ProjectListWidget): + + def validate_context_change(self): + return True + + def refresh(self): + selected_project = None + for index in self.project_list.selectedIndexes(): + selected_project = index.data(QtCore.Qt.DisplayRole) + break + + model = self.project_list.model() + model.clear() + items = [] + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + + for project_name in sync_server.get_synced_presets().keys(): + items.append(project_name) + + print("!!!! items:: {}".format(items)) + sync_server.log.debug("ld !!!! items:: {}".format(items)) + for item in items: + model.appendRow(QtGui.QStandardItem(item)) + + # self.select_project(selected_project) + + self.current_project = self.project_list.currentIndex().data( + QtCore.Qt.DisplayRole + ) + + +class SyncRepresentationWidget(QtWidgets.QWidget): + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("asset", 130), + ("subset", 190), + ("version", 30), + ("representation", 30), + ("created_dt", 120), + ("sync_dt", 85), + ("local_site", 80), + ("remote_site", 60), + ("priority", 55), + ("state", 50) + ) + + def __init__(self, parent=None): + super(SyncRepresentationWidget, self).__init__(parent) + + filter = QtWidgets.QLineEdit() + filter.setPlaceholderText("Filter subsets..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(filter) + + # TODO ? TreeViewSpinner + + table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + log.debug("!!! headers:: {}".format(headers)) + model = SyncRepresentationModel(headers) + table_view.setModel(model) + table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + table_view.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) + table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + table_view.setSortingEnabled(True) + table_view.setAlternatingRowColors(True) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(table_view) + + table_view.doubleClicked.connect(self._doubleClicked) + + def _doubleClicked(self, index): + log.debug("doubleclicked {}:{}".format(index.row(), index.column)) + detail_window = SyncServerDetailWindow(index) + detail_window.open() + + +class SyncRepresentationModel(QtCore.QAbstractTableModel): + PAGE_SIZE = 30 + DEFAULT_SORT = { + "context.asset": 1, + "context.subset": 1, + "context.version": 1, + } + SORT_BY_COLUMN = [ + "context.asset", # asset + "context.subset", # subset + "context.version", # version + "context.representation", # representation + "_id", # local created_dt + "order.created_dt", # remote created_dt + "files.sites.name", # TEMP # local progress + "files.sites.name", # TEMP# remote progress + "context.asset", # priority + "context.asset" # state + ] + DEFAULT_QUERY = { + "type": "representation", + } + + numberPopulated = QtCore.Signal(int) + + def __init__(self, header, project=None): + super(SyncRepresentationModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self._buffer = [] # stash one page worth of records (actually cursor) + + self._initialized = False + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + sync_server.get_sites_for_project('petr_test') + + self.query = self.DEFAULT_QUERY + + self.projection = { + "context.subset": 1, + "context.asset": 1, + "context.version": 1, + "context.representation": 1, + "files": 1 + } + + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def data(self, index, role): + if role == Qt.DisplayRole: + return self._data[index.row()][index.column()] + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._data[0]) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + # if orientation == Qt.Vertical: + # return str(self._data[section]) + + def refresh(self, representations): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + log.debug("!!! refresh sort {}".format(self.sort)) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + log.debug("!!! representations:: {}".format(representations)) + #log.debug("!!! representations:: {}".format(len(representations))) + for repre in representations: + context = repre.get("context") + # log.debug("!!! context:: {}".format(context)) + # log.debug("!!! repre:: {}".format(repre)) + # log.debug("!!! repre:: {}".format(type(repre))) + created = {} + # log.debug("!!! files:: {}".format(repre.get("files", []))) + # log.debug("!!! files:: {}".format(type(repre.get("files", [])))) + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + for file in files: + # log.debug("!!! file:: {}".format(file)) + # log.debug("!!! file:: {}".format(type(file))) + sites = file.get("sites") + # log.debug("!!! sites:: {}".format(sites)) + for site in sites: + # log.debug("!!! site:: {}".format(site)) + # log.debug("!!! site:: {}".format(type(site))) + if not isinstance(site, dict): + # log.debug("Obsolete site {} for {}".format( + # site, repre.get("_id"))) + continue + + if site.get("name") != local_site and \ + site.get("name") != remote_site: + continue + + if not created.get(site.get("name")): + created[site.get("name")] = [] + + created[site.get("name")]. \ + append(site.get("created_dt")) + + # log.debug("!!! created:: {}".format(created)) + # log.debug("!!! remote_site:: {}".format(remote_site)) + local_created = '' + if all(created.get(local_site, [None])): + local_created = min(created[local_site]) + # log.debug("!!! local_created:: {}".format(local_created)) + remote_created = '' + if all(created.get(remote_site, [None])): + remote_created = min(created[remote_site]) + + item = [ + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + str(local_created), + str(remote_created), + local_site, + remote_site, + 1, + 0 + ] + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) + # 'skip' might be suboptimal when representation hits 500k+ + # self._buffer = list(self.dbcon.aggregate(self.query)) + # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) + # return len(self._buffer) > self._rec_loaded + return False + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = self._buffer.count() - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + log.debug("!!! sort {} {}".format(index, order)) + log.debug("!!! orig query {}".format(self.query)) + self._rec_loaded = 0 + # limit unwanted first re-sorting by view + if index < 0: + return + + if order == 0: + order = 1 + else: + order = -1 + + if index < 5: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + elif index == 5: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = [ + {"$match": { + "type": "representation", + "files.sites": { + "$elemMatch": { + "name": self.remote_site, + "created_dt": {"$exists": 1} + }, + } + }}, + {"$unwind": "$files"}, + {"$addFields": { + "order": { + "$filter": { + "input": "$files.sites", + "as": "p", + "cond": {"$eq": ["$$p.name", self.remote_site]} + } + } + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + log.debug("!!! sort {}".format(self.sort)) + log.debug("!!! query {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def get_default_query(self): + return [ + {"$match": { + "type": "representation", + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + + +class SyncServerDetailWindow(QtWidgets.QDialog): + def __init__(self, index, parent=None): + super(SyncServerDetailWindow, self).__init__(parent) + log.debug("SyncServerDetailWindow {}:{}".format(index.row(), index.column)) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.resize(1000, 400) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = None + + container = SyncRepresentationDetailWidget(self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Detail") + + +class SyncRepresentationDetailWidget(QtWidgets.QWidget): + active_changed = QtCore.Signal() # active index changed + + default_widths = ( + ("file", 230), + ("created_dt", 120), + ("sync_dt", 85), + ("local_site", 80), + ("remote_site", 60), + ("priority", 55), + ("state", 50) + ) + + def __init__(self, parent=None): + super(SyncRepresentationDetailWidget, self).__init__(parent) + + filter = QtWidgets.QLineEdit() + filter.setPlaceholderText("Filter subsets..") + + top_bar_layout = QtWidgets.QHBoxLayout() + top_bar_layout.addWidget(filter) + + table_view = QtWidgets.QTableView() + headers = [item[0] for item in self.default_widths] + log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) + + model = SyncRepresentationModel(headers) + table_view.setModel(model) + table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + table_view.setSelectionMode( + QtWidgets.QAbstractItemView.ExtendedSelection) + table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + table_view.setSortingEnabled(True) + table_view.setAlternatingRowColors(True) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(top_bar_layout) + layout.addWidget(table_view) + + def data(self, index, role): + if role == Qt.DisplayRole: + return self._data[index.row()][index.column()] + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._data[0]) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + # if orientation == Qt.Vertical: + # return str(self._data[section]) + From 32ee682168e5093f3a70f206b3fc3e99f510e7f2 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 21 Dec 2020 19:46:30 +0100 Subject: [PATCH 02/50] move scripts to tools, make them path independent, clean build dir --- build.ps1 => tools/build.ps1 | 23 +++++--- build.sh => tools/build.sh | 0 tools/create_env.ps1 | 102 +++++++++++++++++++++++++++++++++++ tools/make_docs.ps1 | 51 ++++++++++++++++-- 4 files changed, 166 insertions(+), 10 deletions(-) rename build.ps1 => tools/build.ps1 (79%) rename build.sh => tools/build.sh (100%) create mode 100644 tools/create_env.ps1 diff --git a/build.ps1 b/tools/build.ps1 similarity index 79% rename from build.ps1 rename to tools/build.ps1 index 39ce90e36a..795b9b74ef 100644 --- a/build.ps1 +++ b/tools/build.ps1 @@ -38,7 +38,11 @@ $art = @' Write-Host $art -ForegroundColor DarkGreen -$version_file = Get-Content -Path ".\pype\version.py" +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" $result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') $pype_version = $result[0].Groups['version'].Value if (-not $pype_version) { @@ -47,10 +51,14 @@ if (-not $pype_version) { Exit-WithCode 1 } +Write-Host "--- " -NoNewline -ForegroundColor yellow +Write-Host "Cleaning build directory ..." +Remove-Item -Recurse -Force "$($pype_root)\build\*" + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype [ " -NoNewline -ForegroundColor white Write-host $pype_version -NoNewline -ForegroundColor green -Write-Host " ]..." -ForegroundColor white +Write-Host " ] ..." -ForegroundColor white Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Detecting host Python ... " -NoNewline @@ -78,11 +86,11 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { Write-Host "OK [ $p ]" -ForegroundColor green Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Creating virtual env ..." -& python -m venv venv +& python -m venv "$($pype_root)\venv" Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Entering venv ..." try { - . (".\venv\Scripts\Activate.ps1") + . ("$($pype_root)\venv\Scripts\Activate.ps1") } catch { Write-Host "!!! Failed to activate" -ForegroundColor red @@ -91,15 +99,18 @@ catch { } Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Installing packages to new venv ..." +& pip -m pip install -U pip & pip install -r .\requirements.txt Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline -Get-ChildItem . -Filter "*.pyc" -Force -Recurse | Remove-Item -Force -Get-ChildItem . -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Get-ChildItem $pype_root -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse Write-Host "OK" -ForegroundColor green Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype ..." +Set-Location -Path $pype_root & python setup.py build deactivate +Set-Location -Path $current_dir diff --git a/build.sh b/tools/build.sh similarity index 100% rename from build.sh rename to tools/build.sh diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 new file mode 100644 index 0000000000..ec9f8003b8 --- /dev/null +++ b/tools/create_env.ps1 @@ -0,0 +1,102 @@ +<# +.SYNOPSIS + Helper script create virtual env. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\build.ps1 + +#> + + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} +$current_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $current_dir).parent.FullName + +$art = @' + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +'@ + +Write-Host $art -ForegroundColor DarkGreen + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" +$result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') +$pype_version = $result[0].Groups['version'].Value +if (-not $pype_version) { + Write-Host "!!! " -ForegroundColor yellow -NoNewline + Write-Host "Cannot determine Pype version." + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Detecting host Python ... " -NoNewline +if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + Write-Host "!!! Python not detected" -ForegroundColor red + Exit-WithCode 1 +} +$version_command = @' +import sys +print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) +'@ + +$p = & python -c $version_command +$env:PYTHON_VERSION = $p +$m = $p -match '(\d+)\.(\d+)' +if(-not $m) { + Write-Host "!!! Cannot determine version" -ForegroundColor red + Exit-WithCode 1 +} +# We are supporting python 3.6 and up +if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { + Write-Host "FAILED Version [ $p ] is old and unsupported" -ForegroundColor red + Exit-WithCode 1 +} +Write-Host "OK [ $p ]" -ForegroundColor green + +Write-Host "--- " -NoNewline -ForegroundColor yellow +Write-Host "Cleaning virtual env directory ..." +Remove-Item -Recurse -Force "$($pype_root)/venv/*" + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Creating virtual env ..." +& python -m venv "$($pype_root)/venv" +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 +} +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Installing packages to new venv ..." +& pip install -r "$pype_root\requirements.txt" + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Cleaning cache files ... " -NoNewline +Get-ChildItem "$($pype_root)" -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem "$($pype_root)" -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Write-Host "OK" -ForegroundColor green diff --git a/tools/make_docs.ps1 b/tools/make_docs.ps1 index 475448d05e..30032d41a6 100644 --- a/tools/make_docs.ps1 +++ b/tools/make_docs.ps1 @@ -1,5 +1,48 @@ -& .\venv\Scripts\Activate.ps1 -sphinx-apidoc.exe -M -e -d 10 -o .\docs\source igniter -sphinx-apidoc.exe -M -e -d 10 -o .\docs\source pype vendor, pype\vendor +<# +.SYNOPSIS + Helper script to update Pype Sphinx sources. + +.DESCRIPTION + This script will run apidoc over Pype sources and generate new source rst + files for documentation. Then it will run build_sphinx to create test html + documentation build. + +.EXAMPLE + +PS> .\make_docs.ps1 + +#> + +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +$art = @' + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +'@ + +Write-Host $art -ForegroundColor DarkGreen + + +& "$($pype_root)\venv\Scripts\Activate.ps1" +Write-Host "This will not overwrite existing source rst files, only scan and add new." +Set-Location -Path $pype_root +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Running apidoc ..." +sphinx-apidoc.exe -M -e -d 10 --ext-intersphinx --ext-todo --ext-coverage --ext-viewcode -o "$($pype_root)\docs\source" igniter +sphinx-apidoc.exe -M -e -d 10 --ext-intersphinx --ext-todo --ext-coverage --ext-viewcode -o "$($pype_root)\docs\source" pype vendor, pype\vendor + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Building html ..." python setup.py build_sphinx -deactivate \ No newline at end of file +deactivate +Set-Location -Path $current_dir From 5317aa61318c9902a0c9fc64eb121e71945659d5 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 21 Dec 2020 19:58:08 +0100 Subject: [PATCH 03/50] fixed typo --- tools/build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build.ps1 b/tools/build.ps1 index 795b9b74ef..5aa826a065 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -99,7 +99,7 @@ catch { } Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Installing packages to new venv ..." -& pip -m pip install -U pip +& python -m pip install -U pip & pip install -r .\requirements.txt Write-Host ">>> " -NoNewline -ForegroundColor green From ffbf482d124f3a80c045c8032f69243255d0220b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 21 Dec 2020 20:58:51 +0100 Subject: [PATCH 04/50] #817 - Implemented Detail dialog Sorting, pagination --- pype/modules/sync_server/tray/app.py | 376 ++++++++++++++++++++++++--- 1 file changed, 335 insertions(+), 41 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 9cb021c270..5888a2af27 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,9 +1,19 @@ +import sys + +sys.path.append('c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype') +sys.path.append( + 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos') +sys.path.append( + 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') + from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from avalon import style from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager +import attr +import os from pype.lib import PypeLogger log = PypeLogger().get_logger("SyncServer") @@ -23,8 +33,9 @@ class SyncServerWindow(QtWidgets.QDialog): footer.setFixedHeight(20) container = QtWidgets.QWidget() - projects = SyncProjectListWidget(self) - repres = SyncRepresentationWidget(self) + projects = SyncProjectListWidget(parent=self) + repres = SyncRepresentationWidget(project=projects.current_project, + parent=self) container_layout = QtWidgets.QHBoxLayout(container) container_layout.setContentsMargins(0, 0, 0, 0) @@ -36,13 +47,6 @@ class SyncServerWindow(QtWidgets.QDialog): container.setLayout(container_layout) - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = None - - # Project - self.combo_projects = QtWidgets.QComboBox() - body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -63,6 +67,9 @@ class SyncServerWindow(QtWidgets.QDialog): class SyncProjectListWidget(ProjectListWidget): + """ + Lists all projects that are syncronized to choose from + """ def validate_context_change(self): return True @@ -82,7 +89,6 @@ class SyncProjectListWidget(ProjectListWidget): for project_name in sync_server.get_synced_presets().keys(): items.append(project_name) - print("!!!! items:: {}".format(items)) sync_server.log.debug("ld !!!! items:: {}".format(items)) for item in items: model.appendRow(QtGui.QStandardItem(item)) @@ -110,8 +116,9 @@ class SyncRepresentationWidget(QtWidgets.QWidget): ("state", 50) ) - def __init__(self, parent=None): + def __init__(self, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) + self.project = project filter = QtWidgets.QLineEdit() filter.setPlaceholderText("Filter subsets..") @@ -121,28 +128,31 @@ class SyncRepresentationWidget(QtWidgets.QWidget): # TODO ? TreeViewSpinner - table_view = QtWidgets.QTableView() + self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] log.debug("!!! headers:: {}".format(headers)) model = SyncRepresentationModel(headers) - table_view.setModel(model) - table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - table_view.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) - table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) - table_view.setSortingEnabled(True) - table_view.setAlternatingRowColors(True) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.ExtendedSelection) + self.table_view.horizontalHeader().setSortIndicator( + -1, Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) - layout.addWidget(table_view) + layout.addWidget(self.table_view) - table_view.doubleClicked.connect(self._doubleClicked) + self.table_view.doubleClicked.connect(self._doubleClicked) def _doubleClicked(self, index): - log.debug("doubleclicked {}:{}".format(index.row(), index.column)) - detail_window = SyncServerDetailWindow(index) - detail_window.open() + _id = self.table_view.model().data(index, Qt.UserRole) + log.debug("doubleclicked {}".format(_id)) + detail_window = SyncServerDetailWindow(_id, self.project) + detail_window.exec() class SyncRepresentationModel(QtCore.QAbstractTableModel): @@ -170,6 +180,25 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): numberPopulated = QtCore.Signal(int) + @attr.s + class SyncRepresentation: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + asset = attr.ib() + subset = attr.ib() + version = attr.ib() + representation = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + def __init__(self, header, project=None): super(SyncRepresentationModel, self).__init__() self._header = header @@ -210,23 +239,23 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.refresh(representations) def data(self, index, role): + item = self._data[index.row()] if role == Qt.DisplayRole: - return self._data[index.row()][index.column()] + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): - return len(self._data[0]) + return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: if orientation == Qt.Horizontal: return str(self._header[section]) - # if orientation == Qt.Vertical: - # return str(self._data[section]) - def refresh(self, representations): self.beginResetModel() self._data = [] @@ -243,7 +272,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): for repre in representations: context = repre.get("context") # log.debug("!!! context:: {}".format(context)) - # log.debug("!!! repre:: {}".format(repre)) + log.debug("!!! repre:: {}".format(repre)) # log.debug("!!! repre:: {}".format(type(repre))) created = {} # log.debug("!!! files:: {}".format(repre.get("files", []))) @@ -284,7 +313,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if all(created.get(remote_site, [None])): remote_created = min(created[remote_site]) - item = [ + item = self.SyncRepresentation( + repre.get("_id"), context.get("asset"), context.get("subset"), "v{:0>3d}".format(context.get("version", 1)), @@ -295,7 +325,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): remote_site, 1, 0 - ] + ) + self._data.append(item) self._rec_loaded += 1 @@ -335,11 +366,11 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): def sort(self, index, order): log.debug("!!! sort {} {}".format(index, order)) log.debug("!!! orig query {}".format(self.query)) - self._rec_loaded = 0 # limit unwanted first re-sorting by view if index < 0: return + self._rec_loaded = 0 if order == 0: order = 1 else: @@ -393,9 +424,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, index, parent=None): + def __init__(self, _id, project, parent=None): + log.debug( + "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) - log.debug("SyncServerDetailWindow {}:{}".format(index.row(), index.column)) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) @@ -410,7 +442,7 @@ class SyncServerDetailWindow(QtWidgets.QDialog): self.dbcon.install() self.dbcon.Session["AVALON_PROJECT"] = None - container = SyncRepresentationDetailWidget(self) + container = SyncRepresentationDetailWidget(_id, project, parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -436,16 +468,17 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): default_widths = ( ("file", 230), ("created_dt", 120), - ("sync_dt", 85), + ("sync_dt", 120), ("local_site", 80), ("remote_site", 60), ("priority", 55), ("state", 50) ) - def __init__(self, parent=None): + def __init__(self, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) - + log.debug( + "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) filter = QtWidgets.QLineEdit() filter.setPlaceholderText("Filter subsets..") @@ -456,7 +489,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): headers = [item[0] for item in self.default_widths] log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) - model = SyncRepresentationModel(headers) + model = SyncRepresentationDetailModel(headers, _id, project) table_view.setModel(model) table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) table_view.setSelectionMode( @@ -470,15 +503,103 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): layout.addLayout(top_bar_layout) layout.addWidget(table_view) + # def data(self, index, role): + # if role == Qt.DisplayRole: + # return self._data[index.row()][index.column()] + # + # def rowCount(self, index): + # return len(self._data) + # + # def columnCount(self, index): + # return len((self._header) + # + # def headerData(self, section, orientation, role): + # if role == Qt.DisplayRole: + # if orientation == Qt.Horizontal: + # return str(self._header[section]) + # + # # if orientation == Qt.Vertical: + # # return str(self._data[section]) + + +class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + PAGE_SIZE = 30 + # TODO add filename to sort + DEFAULT_SORT = { + "files._id": 1 + } + SORT_BY_COLUMN = [ + "files._id" + "_id", # local created_dt + "order.created_dt", # remote created_dt + "files.sites.name", # TEMP # local progress + "files.sites.name", # TEMP# remote progress + "context.asset", # priority + "context.asset" # state + ] + + @attr.s + class SyncRepresentationDetail: + """ + Auxiliary object for easier handling. + + Fields must contain all header values (+ any arbitrary values). + """ + _id = attr.ib() + file = attr.ib() + created_dt = attr.ib(default=None) + sync_dt = attr.ib(default=None) + local_site = attr.ib(default=None) + remote_site = attr.ib(default=None) + priority = attr.ib(default=None) + state = attr.ib(default=None) + + def __init__(self, header, _id, project=None): + super(SyncRepresentationDetailModel, self).__init__() + self._header = header + self._data = [] + self._project = project + self._rec_loaded = 0 + self._buffer = [] # stash one page worth of records (actually cursor) + self._id = _id + log.debug("!!! init _id: {}".format(self._id)) + self._initialized = False + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + # TODO think about admin mode + # this is for regular user, always only single local and single remote + self.local_site, self.remote_site = \ + sync_server.get_sites_for_project('petr_test') + + self.sort = self.DEFAULT_SORT + + # in case we would like to hide/show some columns + self.projection = { + "files": 1 + } + + self.query = self.get_default_query() + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + def data(self, index, role): + item = self._data[index.row()] if role == Qt.DisplayRole: - return self._data[index.row()][index.column()] + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id def rowCount(self, index): return len(self._data) def columnCount(self, index): - return len(self._data[0]) + return len(self._header) def headerData(self, section, orientation, role): if role == Qt.DisplayRole: @@ -488,3 +609,176 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): # if orientation == Qt.Vertical: # return str(self._data[section]) + def refresh(self, representations): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + log.debug("!!! refresh sort {}".format(self.sort)) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + """ + Process all records from 'representation' and add them to storage. + + Args: + local_site (str): name of local site (mine) + remote_site (str): name of cloud provider (theirs) + representations (Mongo Cursor) + """ + for repre in representations: + # log.debug("!!! repre:: {}".format(repre)) + created = {} + # log.debug("!!! files:: {}".format(repre.get("files", []))) + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + for file in files: + log.debug("!!! file:: {}".format(file)) + sites = file.get("sites") + # log.debug("!!! sites:: {}".format(sites)) + for site in sites: + log.debug("!!! site:: {}".format(site)) + # log.debug("!!! site:: {}".format(type(site))) + if not isinstance(site, dict): + # log.debug("Obsolete site {} for {}".format( + # site, repre.get("_id"))) + continue + + if site.get("name") != local_site and \ + site.get("name") != remote_site: + continue + + if not created.get(site.get("name")): + created[site.get("name")] = [] + + created[site.get("name")].append(site.get("created_dt")) + + local_created = created.get(local_site) + remote_created = created.get(remote_site) + + item = self.SyncRepresentationDetail( + repre.get("_id"), + os.path.basename(file["path"]), + str(local_created), + str(remote_created), + local_site, + remote_site, + 1, + 0 + ) + self._data.append(item) + self._rec_loaded += 1 + + log.debug("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) + log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + log.debug("items_to_fetch {}".format(items_to_fetch)) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + def sort(self, index, order): + log.debug("!!! sort {} {}".format(index, order)) + log.debug("!!! orig query {}".format(self.query)) + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 # change sort - reset from start + + if order == 0: + order = 1 + else: + order = -1 + + if index < 2: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + elif index == 2: + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = [ + {"$match": { + "type": "representation", + "_id": self._id, + "files.sites": { + "$elemMatch": { + "name": self.remote_site, + "created_dt": {"$exists": 1} + }, + } + }}, + {"$unwind": "$files"}, + {"$addFields": { + "order": { + "$filter": { + "input": "$files.sites", + "as": "p", + "cond": {"$eq": ["$$p.name", self.remote_site]} + } + } + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + log.debug("!!! sort {}".format(self.sort)) + log.debug("!!! query {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def get_default_query(self): + """ + Gets query that gets used when no extra sorting, filtering or + projecting is needed. + + Called for basic table view. + """ + return [ + {"$match": { + "type": "representation", + "_id": self._id + }}, + {"$sort": self.sort}, + {"$limit": self.PAGE_SIZE}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + +if __name__ == '__main__': + + app = QtWidgets.QApplication(sys.argv) + #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + os.environ["PYPE_MONGO"] = "1" + + widget = SyncServerWindow() + widget.show() + + sys.exit(app.exec_()) From 9c5abace27157f362c1af9375895841978a5105a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Jan 2021 11:38:57 +0100 Subject: [PATCH 05/50] SyncServer GUI - added icons, progress, base of menu Fixed aggregate --- pype/modules/sync_server/tray/app.py | 847 ++++++++++++++++++++------- 1 file changed, 632 insertions(+), 215 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 5888a2af27..10967a07f9 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,5 @@ import sys -sys.path.append('c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype') -sys.path.append( - 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos') sys.path.append( 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') @@ -14,10 +11,23 @@ from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager import attr import os +from pype.tools.settings.settings import style +from avalon.tools.delegates import PrettyTimeDelegate from pype.lib import PypeLogger + +import json + log = PypeLogger().get_logger("SyncServer") +STATUS = { + 0: 'Queued', + 1: 'Failed', + 2: 'In Progress', + 3: 'Paused', + 4: 'Synced OK', + -1: 'Not available' +} class SyncServerWindow(QtWidgets.QDialog): def __init__(self, parent=None): @@ -26,6 +36,7 @@ class SyncServerWindow(QtWidgets.QDialog): self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1400, 800) body = QtWidgets.QWidget() @@ -104,15 +115,17 @@ class SyncRepresentationWidget(QtWidgets.QWidget): active_changed = QtCore.Signal() # active index changed default_widths = ( - ("asset", 130), + ("asset", 210), ("subset", 190), - ("version", 30), - ("representation", 30), - ("created_dt", 120), - ("sync_dt", 85), - ("local_site", 80), - ("remote_site", 60), - ("priority", 55), + ("version", 10), + ("representation", 90), + ("created_dt", 100), + ("sync_dt", 100), + ("local_site", 60), + ("remote_site", 70), + ("files_count", 70), + ("files_size", 70), + ("priority", 20), ("state", 50) ) @@ -120,11 +133,11 @@ class SyncRepresentationWidget(QtWidgets.QWidget): super(SyncRepresentationWidget, self).__init__(parent) self.project = project - filter = QtWidgets.QLineEdit() - filter.setPlaceholderText("Filter subsets..") + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representations..") top_bar_layout = QtWidgets.QHBoxLayout() - top_bar_layout.addWidget(filter) + top_bar_layout.addWidget(self.filter) # TODO ? TreeViewSpinner @@ -134,12 +147,38 @@ class SyncRepresentationWidget(QtWidgets.QWidget): model = SyncRepresentationModel(headers) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.table_view.setSelectionMode( - QtWidgets.QAbstractItemView.ExtendedSelection) + # self.table_view.setSelectionMode( + # QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( -1, Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model()._header.index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model()._header.index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model()._header.index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + + column = self.table_view.model()._header.index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("files_size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model._header.index(column_name) + self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -147,13 +186,27 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addWidget(self.table_view) self.table_view.doubleClicked.connect(self._doubleClicked) + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) def _doubleClicked(self, index): + """ + Opens representation dialog with all files after doubleclick + """ _id = self.table_view.model().data(index, Qt.UserRole) - log.debug("doubleclicked {}".format(_id)) detail_window = SyncServerDetailWindow(_id, self.project) detail_window.exec() + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.view.indexAt(point) + if not point_index.isValid(): + return + class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 30 @@ -167,12 +220,14 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.subset", # subset "context.version", # version "context.representation", # representation - "_id", # local created_dt - "order.created_dt", # remote created_dt - "files.sites.name", # TEMP # local progress - "files.sites.name", # TEMP# remote progress - "context.asset", # priority - "context.asset" # state + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "avg_progress_local", # local progress + "avg_progress_remote", # remote progress + "files_count", # count of files + "files_size", # file size of all files + "context.asset", # priority TODO + "status" # state ] DEFAULT_QUERY = { "type": "representation", @@ -196,6 +251,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) + files_count = attr.ib(default=None) + files_size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) @@ -206,6 +263,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._project = project self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) + self.filter = None self._initialized = False @@ -227,19 +285,67 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset": 1, "context.version": 1, "context.representation": 1, - "files": 1 + "files": 1, + 'files_count': 1, + "files_size": 1, + 'avg_progress_remote': 1, + 'avg_progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$avg_progress_remote', 0]}, + {'$eq': ['$avg_progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$avg_progress_remote', 0]}, + {'$lt': ['$avg_progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$avg_progress_local', 0]}, + {'$lt': ['$avg_progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$avg_progress_remote', 1]}, + {'$eq': ['$avg_progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } } self.sort = self.DEFAULT_SORT self.query = self.get_default_query() self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(self.query)) + log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) def data(self, index, role): item = self._data[index.row()] + if role == Qt.DisplayRole: return attr.asdict(item)[self._header[index.column()]] if role == Qt.UserRole: @@ -256,11 +362,16 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - def refresh(self, representations): + def refresh(self, representations=None): self.beginResetModel() self._data = [] self._rec_loaded = 0 log.debug("!!! refresh sort {}".format(self.sort)) + if not representations: + self.query = self.get_default_query() + log.debug( + "!!! init query: {}".format(json.dumps(self.query, indent=4))) + representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) @@ -270,9 +381,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): log.debug("!!! representations:: {}".format(representations)) #log.debug("!!! representations:: {}".format(len(representations))) for repre in representations: - context = repre.get("context") + context = repre.get("context").pop() # log.debug("!!! context:: {}".format(context)) - log.debug("!!! repre:: {}".format(repre)) + # log.info("!!! repre:: {}".format(repre)) # log.debug("!!! repre:: {}".format(type(repre))) created = {} # log.debug("!!! files:: {}".format(repre.get("files", []))) @@ -280,38 +391,22 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] - for file in files: - # log.debug("!!! file:: {}".format(file)) - # log.debug("!!! file:: {}".format(type(file))) - sites = file.get("sites") - # log.debug("!!! sites:: {}".format(sites)) - for site in sites: - # log.debug("!!! site:: {}".format(site)) - # log.debug("!!! site:: {}".format(type(site))) - if not isinstance(site, dict): - # log.debug("Obsolete site {} for {}".format( - # site, repre.get("_id"))) - continue - if site.get("name") != local_site and \ - site.get("name") != remote_site: - continue + # representation without files doesnt concern us + if not files: + continue - if not created.get(site.get("name")): - created[site.get("name")] = [] + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") - created[site.get("name")]. \ - append(site.get("created_dt")) + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - # log.debug("!!! created:: {}".format(created)) - # log.debug("!!! remote_site:: {}".format(remote_site)) - local_created = '' - if all(created.get(local_site, [None])): - local_created = min(created[local_site]) - # log.debug("!!! local_created:: {}".format(local_created)) - remote_created = '' - if all(created.get(remote_site, [None])): - remote_created = min(created[remote_site]) + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') item = self.SyncRepresentation( repre.get("_id"), @@ -319,12 +414,14 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): context.get("subset"), "v{:0>3d}".format(context.get("version", 1)), context.get("representation"), - str(local_created), - str(remote_created), - local_site, - remote_site, + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), 1, - 0 + STATUS[repre.get("status", -1)] ) self._data.append(item) @@ -336,10 +433,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): """ log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) # 'skip' might be suboptimal when representation hits 500k+ - # self._buffer = list(self.dbcon.aggregate(self.query)) + self._buffer = list(self.dbcon.aggregate(self.query)) # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) - # return len(self._buffer) > self._rec_loaded - return False + return len(self._buffer) > self._rec_loaded def fetchMore(self, index): """ @@ -351,7 +447,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): """ log.debug("fetchMore") # cursor.count() returns always total number, not only skipped + limit - remainder = self._buffer.count() - self._rec_loaded + remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) self.beginInsertRows(index, self._rec_loaded, @@ -364,8 +460,13 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.numberPopulated.emit(items_to_fetch) # ?? def sort(self, index, order): - log.debug("!!! sort {} {}".format(index, order)) - log.debug("!!! orig query {}".format(self.query)) + """ + Summary sort per representation + + Args: + index (int): column index + order (int): 0| + """ # limit unwanted first re-sorting by view if index < 0: return @@ -376,45 +477,107 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): else: order = -1 - if index < 5: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - elif index == 5: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = [ - {"$match": { - "type": "representation", - "files.sites": { - "$elemMatch": { - "name": self.remote_site, - "created_dt": {"$exists": 1} - }, - } - }}, - {"$unwind": "$files"}, - {"$addFields": { - "order": { - "$filter": { - "input": "$files.sites", - "as": "p", - "cond": {"$eq": ["$$p.name", self.remote_site]} - } - } - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(self.query)) + log.debug("!!! query {}".format(json.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) + def set_filter(self, filter): + self.filter = filter + self.refresh() + def get_default_query(self): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - queued + 1 - failed + 2 - paused (not implemented yet) + 3 - in progress + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + """ return [ - {"$match": { - "type": "representation", + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'files_size': {'$ifNull': ["$files.size", 0]} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + }}, + {'$group': { + '_id': '$_id' + # pass through context - same for representation + , 'context': {'$addToSet': '$context'} + # pass through files as a list + , 'files': {'$addToSet': '$files'} + # count how many files + , 'files_count': {'$sum': 1} + , 'files_size': {'$sum': '$files_size'} + # sum avg progress, finished = 1 + , 'avg_progress_remote': {'$avg': "$progress_remote"} + , 'avg_progress_local': {'$avg': "$progress_local"} + # select last touch of file + , 'updated_dt_remote': {'$max': "$updated_dt_remote"} + , 'failed_remote': {'$sum': '$failed_remote'} + , 'failed_local': {'$sum': '$failed_local'} + , 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$sort": self.sort}, {"$limit": self.PAGE_SIZE}, @@ -422,6 +585,47 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): {"$project": self.projection} ] + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [{'context.subset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, _id, project, parent=None): @@ -432,6 +636,7 @@ class SyncServerDetailWindow(QtWidgets.QDialog): self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1000, 400) body = QtWidgets.QWidget() @@ -466,12 +671,13 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): active_changed = QtCore.Signal() # active index changed default_widths = ( - ("file", 230), + ("file", 290), ("created_dt", 120), ("sync_dt", 120), - ("local_site", 80), + ("local_site", 60), ("remote_site", 60), - ("priority", 55), + ("size", 60), + ("priority", 20), ("state", 50) ) @@ -479,63 +685,109 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): super(SyncRepresentationDetailWidget, self).__init__(parent) log.debug( "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) - filter = QtWidgets.QLineEdit() - filter.setPlaceholderText("Filter subsets..") + self.filter = QtWidgets.QLineEdit() + self.filter.setPlaceholderText("Filter representation..") top_bar_layout = QtWidgets.QHBoxLayout() - top_bar_layout.addWidget(filter) + top_bar_layout.addWidget(self.filter) - table_view = QtWidgets.QTableView() + self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) model = SyncRepresentationDetailModel(headers, _id, project) - table_view.setModel(model) - table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - table_view.setSelectionMode( - QtWidgets.QAbstractItemView.ExtendedSelection) - table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) - table_view.setSortingEnabled(True) - table_view.setAlternatingRowColors(True) + self.table_view.setModel(model) + self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionBehavior( + QtWidgets.QTableView.SelectRows) + self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + self.table_view.setSortingEnabled(True) + self.table_view.setAlternatingRowColors(True) + self.table_view.verticalHeader().hide() + + time_delegate = PrettyTimeDelegate(self) + column = self.table_view.model()._header.index("created_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + column = self.table_view.model()._header.index("sync_dt") + self.table_view.setItemDelegateForColumn(column, time_delegate) + + column = self.table_view.model()._header.index("local_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("remote_site") + delegate = ImageDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + column = self.table_view.model()._header.index("size") + delegate = SizeDelegate(self) + self.table_view.setItemDelegateForColumn(column, delegate) + + for column_name, width in self.default_widths: + idx = model._header.index(column_name) + self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) layout.addLayout(top_bar_layout) - layout.addWidget(table_view) + layout.addWidget(self.table_view) - # def data(self, index, role): - # if role == Qt.DisplayRole: - # return self._data[index.row()][index.column()] - # - # def rowCount(self, index): - # return len(self._data) - # - # def columnCount(self, index): - # return len((self._header) - # - # def headerData(self, section, orientation, role): - # if role == Qt.DisplayRole: - # if orientation == Qt.Horizontal: - # return str(self._header[section]) - # - # # if orientation == Qt.Vertical: - # # return str(self._data[section]) + self.filter.textChanged.connect(lambda: model.set_filter( + self.filter.text())) + self.table_view.customContextMenuRequested.connect( + self._on_context_menu) + + def _show_detail(self): + pass + + def _on_context_menu(self, point): + """ + Shows menu with loader actions on Right-click. + """ + point_index = self.table_view.indexAt(point) + if not point_index.isValid(): + return + + item = self.table_view.model()._data[point_index.row()] + log.info('item:: {}'.format(item)) + + menu = QtWidgets.QMenu() + actions_mapping = {} + if item.state == STATUS[1]: + action = QtWidgets.QAction("Open detail") + actions_mapping[action] = self._show_detail + menu.addAction(action) + + if not actions_mapping: + action = QtWidgets.QAction("< No action >") + actions_mapping[action] = None + menu.addAction(action) + + result = menu.exec_(QtGui.QCursor.pos()) + if result: + to_run = actions_mapping[result] + if to_run: + to_run() + to_run() class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): PAGE_SIZE = 30 # TODO add filename to sort DEFAULT_SORT = { - "files._id": 1 + "files.path": 1 } SORT_BY_COLUMN = [ - "files._id" - "_id", # local created_dt - "order.created_dt", # remote created_dt - "files.sites.name", # TEMP # local progress - "files.sites.name", # TEMP# remote progress - "context.asset", # priority - "context.asset" # state + "files.path", + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "progress_local", # local progress + "progress_remote", # remote progress + "size", # remote progress + "context.asset", # priority TODO + "status" # state ] @attr.s @@ -551,6 +803,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): sync_dt = attr.ib(default=None) local_site = attr.ib(default=None) remote_site = attr.ib(default=None) + size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) @@ -560,6 +813,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self._data = [] self._project = project self._rec_loaded = 0 + self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id log.debug("!!! init _id: {}".format(self._id)) @@ -580,7 +834,52 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # in case we would like to hide/show some columns self.projection = { - "files": 1 + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } } self.query = self.get_default_query() @@ -606,14 +905,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - # if orientation == Qt.Vertical: - # return str(self._data[section]) - - def refresh(self, representations): + def refresh(self, representations=None): self.beginResetModel() self._data = [] self._rec_loaded = 0 - log.debug("!!! refresh sort {}".format(self.sort)) + + if not representations: + self.query = self.get_default_query() + log.debug("!!! init query: {}".format(self.query)) + representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, representations) @@ -629,50 +929,48 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): representations (Mongo Cursor) """ for repre in representations: - # log.debug("!!! repre:: {}".format(repre)) - created = {} + # log.info("!!! repre:: {}".format(repre)) + # log.debug("!!! files:: {}".format(repre.get("files", []))) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] + for file in files: - log.debug("!!! file:: {}".format(file)) + created = {} + # log.info("!!! file:: {}".format(file)) sites = file.get("sites") # log.debug("!!! sites:: {}".format(sites)) - for site in sites: - log.debug("!!! site:: {}".format(site)) - # log.debug("!!! site:: {}".format(type(site))) - if not isinstance(site, dict): - # log.debug("Obsolete site {} for {}".format( - # site, repre.get("_id"))) - continue - if site.get("name") != local_site and \ - site.get("name") != remote_site: - continue + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime( + "%Y%m%dT%H%M%SZ") - if not created.get(site.get("name")): - created[site.get("name")] = [] + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime( + "%Y%m%dT%H%M%SZ") - created[site.get("name")].append(site.get("created_dt")) - - local_created = created.get(local_site) - remote_created = created.get(remote_site) + progress_remote = repre.get('progress_remote', '') + progress_local = repre.get('progress_local', '') item = self.SyncRepresentationDetail( repre.get("_id"), os.path.basename(file["path"]), - str(local_created), - str(remote_created), - local_site, - remote_site, + local_updated, + remote_updated, + '{} {}'.format(local_site, progress_local), + '{} {}'.format(remote_site, progress_remote), + file.get('size', 0), 1, - 0 + STATUS[repre.get("status", -1)] ) self._data.append(item) self._rec_loaded += 1 - log.debug("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) + # log.info("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) def canFetchMore(self, index): """ @@ -680,8 +978,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ # 'skip' might be suboptimal when representation hits 500k+ self._buffer = list(self.dbcon.aggregate(self.query)) - log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) - log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) return len(self._buffer) > self._rec_loaded def fetchMore(self, index): @@ -696,7 +992,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # cursor.count() returns always total number, not only skipped + limit remainder = len(self._buffer) - self._rec_loaded items_to_fetch = min(self.PAGE_SIZE, remainder) - log.debug("items_to_fetch {}".format(items_to_fetch)) + self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) @@ -705,8 +1001,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.endInsertRows() def sort(self, index, order): - log.debug("!!! sort {} {}".format(index, order)) - log.debug("!!! orig query {}".format(self.query)) # limit unwanted first re-sorting by view if index < 0: return @@ -718,53 +1012,74 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): else: order = -1 - if index < 2: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - elif index == 2: - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = [ - {"$match": { - "type": "representation", - "_id": self._id, - "files.sites": { - "$elemMatch": { - "name": self.remote_site, - "created_dt": {"$exists": 1} - }, - } - }}, - {"$unwind": "$files"}, - {"$addFields": { - "order": { - "$filter": { - "input": "$files.sites", - "as": "p", - "cond": {"$eq": ["$$p.name", self.remote_site]} - } - } - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] - log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(self.query)) + self.sort = {self.SORT_BY_COLUMN[index]: order} + self.query = self.get_default_query() + representations = self.dbcon.aggregate(self.query) self.refresh(representations) + def set_filter(self, filter): + self.filter = filter + self.refresh() + def get_default_query(self): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. Called for basic table view. + + Returns: + [(dict)] - list with single dict - appropriate for aggregate + function for MongoDB """ return [ - {"$match": { - "type": "representation", - "_id": self._id + {"$match": self._get_match_part()}, + {"$unwind": "$files"}, + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} }}, {"$sort": self.sort}, {"$limit": self.PAGE_SIZE}, @@ -772,13 +1087,115 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {"$project": self.projection} ] -if __name__ == '__main__': + def _get_match_part(self): + """ + Returns different content for 'match' portion if filtering by + name is present + Returns: + (dict) + """ + if not self.filter: + return { + "type": "representation", + "_id": self._id + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + "_id": self._id, + '$or': [{'files.path': {'$regex': regex_str, + '$options': 'i'}}] + } + + +class ImageDelegate(QtWidgets.QStyledItemDelegate): + """ + Prints icon of site and progress of synchronization + """ + def __init__(self, parent=None): + super(ImageDelegate, self).__init__(parent) + + def paint(self, painter, option, index): + d = index.data(QtCore.Qt.DisplayRole) + if d: + provider, value = d.split() + else: + return + + # log.info("data:: {} - {}".format(provider, value)) + pix_url = "../providers/resources/{}.png".format(provider) + pixmap = QtGui.QPixmap(pix_url) + + point = QtCore.QPoint(option.rect.x() + + (option.rect.width() - pixmap.width()) / 2, + option.rect.y() + + (option.rect.height() - pixmap.height()) / 2) + painter.drawPixmap(point, pixmap) + + painter.setOpacity(0.5) + overlay_rect = option.rect + overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) + #painter.setCompositionMode(QtGui.QPainter.CompositionMode_DestinationOver) + #painter.setBrush(painter.brush(Qt.white)) + painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0,0,0,200))) + + +class SizeDelegate(QtWidgets.QStyledItemDelegate): + """ + Pretty print for file size + """ + def __init__(self, parent=None): + super(SizeDelegate, self).__init__(parent) + + def displayText(self, value, locale): + if value is None: + # Ignore None value + return + + return self._pretty_size(value) + + def _pretty_size(self, value, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: + if abs(value) < 1024.0: + return "%3.1f%s%s" % (value, unit, suffix) + value /= 1024.0 + return "%.1f%s%s" % (value, 'Yi', suffix) + + +# Back up the reference to the exceptionhook +sys._excepthook = sys.excepthook + +def my_exception_hook(exctype, value, traceback): + # Print the error and traceback + print(exctype, value, traceback) + # Call the normal Exception hook after + sys._excepthook(exctype, value, traceback) + sys.exit(1) + +# Set the exception hook to our wrapping function +sys.excepthook = my_exception_hook + +if __name__ == '__main__': + import sys + from time import sleep app = QtWidgets.QApplication(sys.argv) #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) - os.environ["PYPE_MONGO"] = "1" + os.environ["PYPE_MONGO"] = "mongodb://localhost:27017" + os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" + os.environ["AVALON_DB"] = "avalon" + os.environ["AVALON_TIMEOUT"] = '3000' widget = SyncServerWindow() widget.show() - sys.exit(app.exec_()) + # while True: + # # run some codes that use QAxWidget.dynamicCall() function + # # print some results + # sleep(30) + + try: + sys.exit(app.exec_()) + except: + print("Exiting") From 8f768b2e4a0a203cf2172b6ba8a1f8cda9fc2f60 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Jan 2021 18:44:39 +0100 Subject: [PATCH 06/50] SyncServer GUI - added icons, fixes Added refresh Added menu in detail --- .../providers/resources/gdrive.png | Bin 0 -> 975 bytes .../providers/resources/studio.png | Bin 0 -> 557 bytes pype/modules/sync_server/tray/app.py | 1098 ++++++++++------- 3 files changed, 634 insertions(+), 464 deletions(-) create mode 100644 pype/modules/sync_server/providers/resources/gdrive.png create mode 100644 pype/modules/sync_server/providers/resources/studio.png diff --git a/pype/modules/sync_server/providers/resources/gdrive.png b/pype/modules/sync_server/providers/resources/gdrive.png new file mode 100644 index 0000000000000000000000000000000000000000..e6c913145439ed4f4772035885db281d289e9e45 GIT binary patch literal 975 zcmV;=12FuFP)`J=v zA}Hn}5-qxlWJ3FpzAXg>{{f?`>Y~6XbOtN)%F4PZF*F()Cu^KJXMa3T7yInJkBuo_ z)`pkoWxt!Xp0%F!ZuqZ1y42x{L)$j(nXi#G&B>b}1JA%Rs=a5>{^JSV_38dACRjZ- zvY_4K`vHP~a_#uS21$G&K?e$mR z@C@7+kT-W-k%CoUY=1Z?e3V+M)D53$w?4b!Y+#cq;H+espEuWeJ+rdPCAu5ej(%MS;=j_VTZj zc|TofQwE-tgiG<(j!P2UKl<|OSn-pR07s>P0tnhNxyT{Qt6|p)I@O={udciM;shfj zVUC*ngkpJt=_S4bL~th0j$^G4>JS{9Q>d;I{>EQ0mx=5B8&T3*^hF#&|Y9ij6H z$!1#9o5|bfx)EE_H4rHKSN8;hEDibaH$%f|iLbR}O%ei_3B|A$K#Eyzn{>$wb`B zvIEU1-cqm@y~_h&1T+=@yna>G#agfWzk>EWYo=I#QXsVwu^F(3ls!U~Usa zo!l2yKyzZ~SFE?;c~(WX7*GLFR?WIIAQ+$4+@0b5Wlv4D^5kt)rL*WJ?>lL~FS|tq zu(UoCW!O4ZC1-^%L7MripKsJu09f4U-~r*?n~e zxa;KgfA+su#dP1$aGH4fiUe{IPcQk-rmq=90f-<9;8u05s?*us1EiUa>>F0v{SF*G za^$UV@i6$;i4F7co~9I`pyE!12*r)s2I@IPotFV{_v0UL?7tlq2G;)~;WL(ROULgR x$LBOr1VNqE*;AA{IW<7pX>NMH1pj|Le*-`p;DxPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D0m4Z{K~zXf-IXCv z13?glPk}@tk(J1rHpr&eD<^UA52&M(goH{=0S69AgQ~2fvO!fME0MM3dz)FZZSSsk zIC#m+&b*!3*}dDji~b+dG(G5cyFGYak-c`ieQ3+%BT14eyvtaikjjCdwoE<Y z=5FEF1?>1s#wnv7jZXQj3t=G+l?&({;LqSdw#g1?7n%!iPMDjEFbLb-5OcS{T#J*8 z&8=3z`e707oW)Mj&dqhepZNDuseqD{>_ni&;XB$Zyfs<6dd6K>pIZnYL}H3Paw5RK zM0~E0lsm%*kr|zv>CI@7Z6BP2e-_1K&zoG zrK`~x=l(r~xw!~~>=biYH^~8CyZ&N)3>z=Z&BbPrb_~^EBj`VD_lzH={hScHdI1j> z)l2tZPuFCyu9NtV#xGaEI(Y4|Y{EWn3kvjQTi=u`0QJLJYqBB`JEM^Lp;Q6w>V|O$ v{Uh1Mq2(-XnS9vMvliLOqEGo>MHGDlp;tCVl9Dor00000NkvXXu0mjfWd!-+ literal 0 HcmV?d00001 diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 10967a07f9..aa52b06da9 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,3 @@ -import sys - -sys.path.append( - 'c:\\Users\\petrk\\PycharmProjects\\Pype3.0\\pype\\repos\\pyblish-base') - from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt from avalon import style @@ -12,7 +7,7 @@ from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style -from avalon.tools.delegates import PrettyTimeDelegate +from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -29,7 +24,12 @@ STATUS = { -1: 'Not available' } + class SyncServerWindow(QtWidgets.QDialog): + """ + Main window that contains list of synchronizable projects and summary + view with all synchronizable representations for first project + """ def __init__(self, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) @@ -76,42 +76,44 @@ class SyncServerWindow(QtWidgets.QDialog): self.setLayout(body_layout) self.setWindowTitle("Sync Server") + projects.project_changed.connect( + lambda: repres.table_view.model().set_project( + projects.current_project)) + class SyncProjectListWidget(ProjectListWidget): """ - Lists all projects that are syncronized to choose from + Lists all projects that are synchronized to choose from """ def validate_context_change(self): return True def refresh(self): - selected_project = None - for index in self.project_list.selectedIndexes(): - selected_project = index.data(QtCore.Qt.DisplayRole) - break - model = self.project_list.model() model.clear() - items = [] manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] for project_name in sync_server.get_synced_presets().keys(): - items.append(project_name) + model.appendRow(QtGui.QStandardItem(project_name)) - sync_server.log.debug("ld !!!! items:: {}".format(items)) - for item in items: - model.appendRow(QtGui.QStandardItem(item)) - - # self.select_project(selected_project) + if len(sync_server.get_synced_presets().keys()) == 0: + model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( QtCore.Qt.DisplayRole ) + if not self.current_project: + self.current_project = self.project_list.model().item(0).\ + data(QtCore.Qt.DisplayRole) class SyncRepresentationWidget(QtWidgets.QWidget): + """ + Summary dialog with list of representations that matches current + settings 'local_site' and 'remote_site'. + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -139,16 +141,14 @@ class SyncRepresentationWidget(QtWidgets.QWidget): top_bar_layout = QtWidgets.QHBoxLayout() top_bar_layout.addWidget(self.filter) - # TODO ? TreeViewSpinner - self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! headers:: {}".format(headers)) - model = SyncRepresentationModel(headers) + + model = SyncRepresentationModel(headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - # self.table_view.setSelectionMode( - # QtWidgets.QAbstractItemView.SingleSelection) + self.table_view.setSelectionMode( + QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QAbstractItemView.SelectRows) self.table_view.horizontalHeader().setSortIndicator( @@ -158,26 +158,25 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("files_size") + column = self.table_view.model().get_header_index("files_size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -203,17 +202,19 @@ class SyncRepresentationWidget(QtWidgets.QWidget): """ Shows menu with loader actions on Right-click. """ - point_index = self.view.indexAt(point) + point_index = self.table_view.indexAt(point) if not point_index.isValid(): return class SyncRepresentationModel(QtCore.QAbstractTableModel): - PAGE_SIZE = 30 + PAGE_SIZE = 19 + REFRESH_SEC = 5000 DEFAULT_SORT = { "context.asset": 1, "context.subset": 1, "context.version": 1, + "_id": 1 } SORT_BY_COLUMN = [ "context.asset", # asset @@ -229,9 +230,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset", # priority TODO "status" # state ] - DEFAULT_QUERY = { - "type": "representation", - } numberPopulated = QtCore.Signal(int) @@ -269,18 +267,356 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) - self.query = self.DEFAULT_QUERY + self.projection = self.get_default_projection() - self.projection = { + self.sort = self.DEFAULT_SORT + + self.query = self.get_default_query() + self.default_query = list(self.get_default_query()) + log.debug("!!! init query: {}".format(json.dumps(self.query, + indent=4))) + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(self.REFRESH_SEC) + + def data(self, index, role): + item = self._data[index.row()] + + if role == Qt.DisplayRole: + return attr.asdict(item)[self._header[index.column()]] + if role == Qt.UserRole: + return item._id + + def rowCount(self, index): + return len(self._data) + + def columnCount(self, index): + return len(self._header) + + def headerData(self, section, orientation, role): + if role == Qt.DisplayRole: + if orientation == Qt.Horizontal: + return str(self._header[section]) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(self.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + + def refresh(self, representations=None, load_records=0): + self.beginResetModel() + self._data = [] + self._rec_loaded = 0 + + if not representations: + self.query = self.get_default_query(load_records) + representations = self.dbcon.aggregate(self.query) + + self._add_page_records(self.local_site, self.remote_site, + representations) + self.endResetModel() + + def _add_page_records(self, local_site, remote_site, representations): + for repre in representations: + context = repre.get("context").pop() + files = repre.get("files", []) + if isinstance(files, dict): # aggregate returns dictionary + files = [files] + + # representation without files doesnt concern us + if not files: + continue + + local_updated = remote_updated = None + if repre.get('updated_dt_local'): + local_updated = \ + repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") + + if repre.get('updated_dt_remote'): + remote_updated = \ + repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") + + avg_progress_remote = repre.get('avg_progress_remote', '') + avg_progress_local = repre.get('avg_progress_local', '') + + item = self.SyncRepresentation( + repre.get("_id"), + context.get("asset"), + context.get("subset"), + "v{:0>3d}".format(context.get("version", 1)), + context.get("representation"), + local_updated, + remote_updated, + '{} {}'.format(local_site, avg_progress_local), + '{} {}'.format(remote_site, avg_progress_remote), + repre.get("files_count", 1), + repre.get("files_size", 0), + 1, + STATUS[repre.get("status", -1)] + ) + + self._data.append(item) + self._rec_loaded += 1 + + def canFetchMore(self, index): + """ + Check if there are more records than currently loaded + """ + # 'skip' might be suboptimal when representation hits 500k+ + self._buffer = list(self.dbcon.aggregate(self.query)) + # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( + # self._rec_loaded, len(self._buffer))) + return len(self._buffer) > self._rec_loaded + + def fetchMore(self, index): + """ + Add more record to model. + + Called when 'canFetchMore' returns true, which means there are + more records in DB than loaded. + 'self._buffer' is used to stash cursor to limit requery + """ + log.debug("fetchMore") + # cursor.count() returns always total number, not only skipped + limit + remainder = len(self._buffer) - self._rec_loaded + items_to_fetch = min(self.PAGE_SIZE, remainder) + self.beginInsertRows(index, + self._rec_loaded, + self._rec_loaded + items_to_fetch - 1) + + self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self.endInsertRows() + + self.numberPopulated.emit(items_to_fetch) # ?? + + def sort(self, index, order): + """ + Summary sort per representation. + + Sort is happening on a DB side, model is reset, db queried + again. + + Args: + index (int): column index + order (int): 0| + """ + # limit unwanted first re-sorting by view + if index < 0: + return + + self._rec_loaded = 0 + if order == 0: + order = 1 + else: + order = -1 + + self.sort = {self.SORT_BY_COLUMN[index]: order, '_id': 1} + self.query = self.get_default_query() + + representations = self.dbcon.aggregate(self.query) + self.refresh(representations) + + def set_filter(self, filter): + """ + Adds text value filtering + + Args: + filter (str): string inputted by user + """ + self.filter = filter + self.refresh() + + def set_project(self, project): + """ + Changes project, called after project selection is changed + + Args: + project (str): name of project + """ + self._project = project + self.dbcon.Session["AVALON_PROJECT"] = self._project + self.refresh() + + def get_default_query(self, limit=0): + """ + Returns basic aggregate query for main table. + + Main table provides summary information about representation, + which could have multiple files. Details are accessible after + double click on representation row. + Columns: + 'created_dt' - max of created or updated (when failed) per repr + 'sync_dt' - same for remote side + 'local_site' - progress of repr on local side, 1 = finished + 'remote_site' - progress on remote side, calculates from files + 'state' - + 0 - queued + 1 - failed + 2 - paused (not implemented yet) + 3 - in progress + 4 - finished on both sides + + are calculated and must be calculated in DB because of + pagination + + Args: + limit (int): how many records should be returned, by default + it 'PAGE_SIZE' for performance. + Should be overridden by value of loaded records for refresh + functionality (got more records by scrolling, refresh + shouldn't reset that) + """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + + return [ + {"$match": self._get_match_part()}, + {'$unwind': '$files'}, + # merge potentially unwinded records back to single per repre + {'$addFields': { + 'order_remote': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.remote_site]} + }} + , 'order_local': { + '$filter': {'input': '$files.sites', 'as': 'p', + 'cond': {'$eq': ['$$p.name', self.local_site]} + }} + }}, + {'$addFields': { + # prepare progress per file, presence of 'created_dt' denotes + # successfully finished load/download + 'progress_remote': {'$first': { + '$cond': [{'$size': "$order_remote.progress"}, + "$order_remote.progress", {'$cond': [ + {'$size': "$order_remote.created_dt"}, [1], + [0]]}]}} + , 'progress_local': {'$first': { + '$cond': [{'$size': "$order_local.progress"}, + "$order_local.progress", {'$cond': [ + {'$size': "$order_local.created_dt"}, [1], + [0]]}]}} + # file might be successfully created or failed, not both + , 'updated_dt_remote': {'$first': { + '$cond': [{'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + {'$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + []] + }]}} + , 'updated_dt_local': {'$first': { + '$cond': [{'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + {'$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + []] + }]}} + , 'files_size': {'$ifNull': ["$files.size", 0]} + , 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} + , 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + }}, + {'$group': { + '_id': '$_id' + # pass through context - same for representation + , 'context': {'$addToSet': '$context'} + # pass through files as a list + , 'files': {'$addToSet': '$files'} + # count how many files + , 'files_count': {'$sum': 1} + , 'files_size': {'$sum': '$files_size'} + # sum avg progress, finished = 1 + , 'avg_progress_remote': {'$avg': "$progress_remote"} + , 'avg_progress_local': {'$avg': "$progress_local"} + # select last touch of file + , 'updated_dt_remote': {'$max': "$updated_dt_remote"} + , 'failed_remote': {'$sum': '$failed_remote'} + , 'failed_local': {'$sum': '$failed_local'} + , 'updated_dt_local': {'$max': "$updated_dt_local"} + }}, + {"$sort": self.sort}, + {"$limit": limit}, + {"$skip": self._rec_loaded}, + {"$project": self.projection} + ] + + def _get_match_part(self): + """ + Extend match part with filter if present. + + Filter is set by user input. Each model has different fields to be + checked. + If performance issues are found, '$text' and text indexes should + be investigated. + """ + if not self.filter: + return { + "type": "representation", + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + else: + regex_str = '.*{}.*'.format(self.filter) + return { + "type": "representation", + '$or': [{'context.subset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, + '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], + 'files.sites': { + '$elemMatch': { + '$or': [ + {'name': self.local_site}, + {'name': self.remote_site} + ] + } + } + } + + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { "context.subset": 1, "context.asset": 1, "context.version": 1, @@ -335,297 +671,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): } } - self.sort = self.DEFAULT_SORT - - self.query = self.get_default_query() - self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def data(self, index, role): - item = self._data[index.row()] - - if role == Qt.DisplayRole: - return attr.asdict(item)[self._header[index.column()]] - if role == Qt.UserRole: - return item._id - - def rowCount(self, index): - return len(self._data) - - def columnCount(self, index): - return len(self._header) - - def headerData(self, section, orientation, role): - if role == Qt.DisplayRole: - if orientation == Qt.Horizontal: - return str(self._header[section]) - - def refresh(self, representations=None): - self.beginResetModel() - self._data = [] - self._rec_loaded = 0 - log.debug("!!! refresh sort {}".format(self.sort)) - if not representations: - self.query = self.get_default_query() - log.debug( - "!!! init query: {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - - self._add_page_records(self.local_site, self.remote_site, - representations) - self.endResetModel() - - def _add_page_records(self, local_site, remote_site, representations): - log.debug("!!! representations:: {}".format(representations)) - #log.debug("!!! representations:: {}".format(len(representations))) - for repre in representations: - context = repre.get("context").pop() - # log.debug("!!! context:: {}".format(context)) - # log.info("!!! repre:: {}".format(repre)) - # log.debug("!!! repre:: {}".format(type(repre))) - created = {} - # log.debug("!!! files:: {}".format(repre.get("files", []))) - # log.debug("!!! files:: {}".format(type(repre.get("files", [])))) - files = repre.get("files", []) - if isinstance(files, dict): # aggregate returns dictionary - files = [files] - - # representation without files doesnt concern us - if not files: - continue - - local_updated = remote_updated = None - if repre.get('updated_dt_local'): - local_updated = \ - repre.get('updated_dt_local').strftime("%Y%m%dT%H%M%SZ") - - if repre.get('updated_dt_remote'): - remote_updated = \ - repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - - avg_progress_remote = repre.get('avg_progress_remote', '') - avg_progress_local = repre.get('avg_progress_local', '') - - item = self.SyncRepresentation( - repre.get("_id"), - context.get("asset"), - context.get("subset"), - "v{:0>3d}".format(context.get("version", 1)), - context.get("representation"), - local_updated, - remote_updated, - '{} {}'.format(local_site, avg_progress_local), - '{} {}'.format(remote_site, avg_progress_remote), - repre.get("files_count", 1), - repre.get("files_size", 0), - 1, - STATUS[repre.get("status", -1)] - ) - - self._data.append(item) - self._rec_loaded += 1 - - def canFetchMore(self, index): - """ - Check if there are more records than currently loaded - """ - log.debug("!!! canFetchMore _rec_loaded:: {}".format(self._rec_loaded)) - # 'skip' might be suboptimal when representation hits 500k+ - self._buffer = list(self.dbcon.aggregate(self.query)) - # log.debug("!!! self._buffer.count():: {}".format(len(self._buffer))) - return len(self._buffer) > self._rec_loaded - - def fetchMore(self, index): - """ - Add more record to model. - - Called when 'canFetchMore' returns true, which means there are - more records in DB than loaded. - 'self._buffer' is used to stash cursor to limit requery - """ - log.debug("fetchMore") - # cursor.count() returns always total number, not only skipped + limit - remainder = len(self._buffer) - self._rec_loaded - items_to_fetch = min(self.PAGE_SIZE, remainder) - self.beginInsertRows(index, - self._rec_loaded, - self._rec_loaded + items_to_fetch - 1) - - self._add_page_records(self.local_site, self.remote_site, self._buffer) - - self.endInsertRows() - - self.numberPopulated.emit(items_to_fetch) # ?? - - def sort(self, index, order): - """ - Summary sort per representation - - Args: - index (int): column index - order (int): 0| - """ - # limit unwanted first re-sorting by view - if index < 0: - return - - self._rec_loaded = 0 - if order == 0: - order = 1 - else: - order = -1 - - self.sort = {self.SORT_BY_COLUMN[index]: order} - self.query = self.get_default_query() - - log.debug("!!! sort {}".format(self.sort)) - log.debug("!!! query {}".format(json.dumps(self.query, indent=4))) - representations = self.dbcon.aggregate(self.query) - self.refresh(representations) - - def set_filter(self, filter): - self.filter = filter - self.refresh() - - def get_default_query(self): - """ - Returns basic aggregate query for main table. - - Main table provides summary information about representation, - which could have multiple files. Details are accessible after - double click on representation row. - Columns: - 'created_dt' - max of created or updated (when failed) per repr - 'sync_dt' - same for remote side - 'local_site' - progress of repr on local side, 1 = finished - 'remote_site' - progress on remote side, calculates from files - 'state' - - 0 - queued - 1 - failed - 2 - paused (not implemented yet) - 3 - in progress - 4 - finished on both sides - - are calculated and must be calculated in DB because of - pagination - """ - return [ - {"$match": self._get_match_part()}, - {'$unwind': '$files'}, - # merge potentially unwinded records back to single per repre - {'$addFields': { - 'order_remote': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { - '$filter': {'input': '$files.sites', 'as': 'p', - 'cond': {'$eq': ['$$p.name', self.local_site]} - }} - }}, - {'$addFields': { - # prepare progress per file, presence of 'created_dt' denotes - # successfully finished load/download - 'progress_remote': {'$first': { - '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} - , 'progress_local': {'$first': { - '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} - # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { - '$cond': [{'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - }]}} - , 'updated_dt_local': {'$first': { - '$cond': [{'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - }]}} - , 'files_size': {'$ifNull': ["$files.size", 0]} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} - }}, - {'$group': { - '_id': '$_id' - # pass through context - same for representation - , 'context': {'$addToSet': '$context'} - # pass through files as a list - , 'files': {'$addToSet': '$files'} - # count how many files - , 'files_count': {'$sum': 1} - , 'files_size': {'$sum': '$files_size'} - # sum avg progress, finished = 1 - , 'avg_progress_remote': {'$avg': "$progress_remote"} - , 'avg_progress_local': {'$avg': "$progress_local"} - # select last touch of file - , 'updated_dt_remote': {'$max': "$updated_dt_remote"} - , 'failed_remote': {'$sum': '$failed_remote'} - , 'failed_local': {'$sum': '$failed_local'} - , 'updated_dt_local': {'$max': "$updated_dt_local"} - }}, - {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, - {"$skip": self._rec_loaded}, - {"$project": self.projection} - ] - - def _get_match_part(self): - """ - Extend match part with filter if present. - - Filter is set by user input. Each model has different fields to be - checked. - If performance issues are found, '$text' and text indexes should - be investigated. - """ - if not self.filter: - return { - "type": "representation", - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - else: - regex_str = '.*{}.*'.format(self.filter) - return { - "type": "representation", - '$or': [{'context.subset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.asset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } - } - class SyncServerDetailWindow(QtWidgets.QDialog): def __init__(self, _id, project, parent=None): @@ -668,6 +713,14 @@ class SyncServerDetailWindow(QtWidgets.QDialog): class SyncRepresentationDetailWidget(QtWidgets.QWidget): + """ + Widget to display list of synchronizable files for single repre. + + Args: + _id (str): representation _id + project (str): name of project with repre + parent (QDialog): SyncServerDetailWindow + """ active_changed = QtCore.Signal() # active index changed default_widths = ( @@ -683,8 +736,14 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def __init__(self, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) - log.debug( - "!!! SyncRepresentationDetailWidget _id:: {}".format(_id)) + + self.representation_id = _id + self.item = None # set to item that mouse was clicked over + self.project = project + + manager = ModulesManager() + self.sync_server = manager.modules_by_name["sync_server"] + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -693,7 +752,6 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - log.debug("!!! SyncRepresentationDetailWidget headers:: {}".format(headers)) model = SyncRepresentationDetailModel(headers, _id, project) self.table_view.setModel(model) @@ -702,31 +760,32 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): QtWidgets.QAbstractItemView.SingleSelection) self.table_view.setSelectionBehavior( QtWidgets.QTableView.SelectRows) - self.table_view.horizontalHeader().setSortIndicator(-1, Qt.AscendingOrder) + self.table_view.horizontalHeader().setSortIndicator(-1, + Qt.AscendingOrder) self.table_view.setSortingEnabled(True) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().hide() time_delegate = PrettyTimeDelegate(self) - column = self.table_view.model()._header.index("created_dt") + column = self.table_view.model().get_header_index("created_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("sync_dt") + column = self.table_view.model().get_header_index("sync_dt") self.table_view.setItemDelegateForColumn(column, time_delegate) - column = self.table_view.model()._header.index("local_site") + column = self.table_view.model().get_header_index("local_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("remote_site") + column = self.table_view.model().get_header_index("remote_site") delegate = ImageDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) - column = self.table_view.model()._header.index("size") + column = self.table_view.model().get_header_index("size") delegate = SizeDelegate(self) self.table_view.setItemDelegateForColumn(column, delegate) for column_name, width in self.default_widths: - idx = model._header.index(column_name) + idx = model.get_header_index(column_name) self.table_view.setColumnWidth(idx, width) layout = QtWidgets.QVBoxLayout(self) @@ -740,7 +799,16 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self._on_context_menu) def _show_detail(self): - pass + """ + Shows windows with error message for failed sync of a file. + """ + dt = max(self.item.created_dt, self.item.sync_dt) + detail_window = SyncRepresentationErrorWindow(self.item._id, + self.project, + dt, + self.item.tries, + self.item.error) + detail_window.exec() def _on_context_menu(self, point): """ @@ -750,16 +818,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): if not point_index.isValid(): return - item = self.table_view.model()._data[point_index.row()] - log.info('item:: {}'.format(item)) + self.item = self.table_view.model()._data[point_index.row()] menu = QtWidgets.QMenu() actions_mapping = {} - if item.state == STATUS[1]: - action = QtWidgets.QAction("Open detail") + + if self.item.state == STATUS[1]: + action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail menu.addAction(action) + remote_site, remote_progress = self.item.remote_site.split() + if remote_progress == '1': + action = QtWidgets.QAction("Reset local site") + actions_mapping[action] = self._reset_local_site + menu.addAction(action) + + local_site, local_progress = self.item.local_site.split() + if local_progress == '1': + action = QtWidgets.QAction("Reset remote site") + actions_mapping[action] = self._reset_remote_site + menu.addAction(action) + if not actions_mapping: action = QtWidgets.QAction("< No action >") actions_mapping[action] = None @@ -770,12 +850,28 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): to_run = actions_mapping[result] if to_run: to_run() - to_run() + + def _reset_local_site(self): + log.info("reset local site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'studio') # TEMP + + def _reset_remote_site(self): + log.info("reset remote site: {}".format(self.item._id)) + self.sync_server.reset_provider_for_file(self.project, + self.representation_id, + self.item._id, + 'gdrive') # TEMP class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): + """ + List of all syncronizable files per single representation. + """ PAGE_SIZE = 30 - # TODO add filename to sort + # TODO add filter filename DEFAULT_SORT = { "files.path": 1 } @@ -806,6 +902,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) + tries = attr.ib(default=None) + error = attr.ib(default=None) def __init__(self, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() @@ -816,77 +914,48 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = None self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id - log.debug("!!! init _id: {}".format(self._id)) self._initialized = False self.dbcon = AvalonMongoDB() self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project or 'petr_test' # TEMP + self.dbcon.Session["AVALON_PROJECT"] = self._project manager = ModulesManager() sync_server = manager.modules_by_name["sync_server"] # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project('petr_test') + sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT # in case we would like to hide/show some columns - self.projection = { - "files": 1, - 'progress_remote': 1, - 'progress_local': 1, - 'updated_dt_remote': 1, - 'updated_dt_local': 1, - 'status': { - '$switch': { - 'branches': [ - { - 'case': { - '$or': [{'$eq': ['$progress_remote', 0]}, - {'$eq': ['$progress_local', 0]}]}, - 'then': 0 - }, - { - 'case': { - '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 - }, - { - 'case': {'$or': [{'$and': [ - {'$gt': ['$progress_remote', 0]}, - {'$lt': ['$progress_remote', 1]} - ]}, - {'$and': [ - {'$gt': ['$progress_local', 0]}, - {'$lt': ['$progress_local', 1]} - ]} - ]}, - 'then': 2 - }, - { - 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 - }, - { - 'case': {'$and': [ - {'$eq': ['$progress_remote', 1]}, - {'$eq': ['$progress_local', 1]} - ]}, - 'then': 4 - }, - ], - 'default': -1 - } - } - } + self.projection = self.get_default_projection() self.query = self.get_default_query() log.debug("!!! init query: {}".format(self.query)) representations = self.dbcon.aggregate(self.query) self.refresh(representations) + self.timer = QtCore.QTimer() + self.timer.timeout.connect(self.tick) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def tick(self): + self.refresh(representations=None, load_records=self._rec_loaded) + self.timer.start(SyncRepresentationModel.REFRESH_SEC) + + def get_header_index(self, value): + """ + Returns index of 'value' in headers + + Args: + value (str): header name value + Returns: + (int) + """ + return self._header.index(value) + def data(self, index, role): item = self._data[index.row()] if role == Qt.DisplayRole: @@ -905,14 +974,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if orientation == Qt.Horizontal: return str(self._header[section]) - def refresh(self, representations=None): + def refresh(self, representations=None, load_records=0): self.beginResetModel() self._data = [] self._rec_loaded = 0 if not representations: - self.query = self.get_default_query() - log.debug("!!! init query: {}".format(self.query)) + self.query = self.get_default_query(load_records) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, @@ -930,18 +998,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ for repre in representations: # log.info("!!! repre:: {}".format(repre)) - - # log.debug("!!! files:: {}".format(repre.get("files", []))) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] for file in files: - created = {} - # log.info("!!! file:: {}".format(file)) - sites = file.get("sites") - # log.debug("!!! sites:: {}".format(sites)) - local_updated = remote_updated = None if repre.get('updated_dt_local'): local_updated = \ @@ -956,6 +1017,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): progress_remote = repre.get('progress_remote', '') progress_local = repre.get('progress_local', '') + errors = [] + if repre.get('failed_remote_error'): + errors.append(repre.get('failed_remote_error')) + if repre.get('failed_local_error'): + errors.append(repre.get('failed_local_error')) + item = self.SyncRepresentationDetail( repre.get("_id"), os.path.basename(file["path"]), @@ -965,13 +1032,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): '{} {}'.format(remote_site, progress_remote), file.get('size', 0), 1, - STATUS[repre.get("status", -1)] + STATUS[repre.get("status", -1)], + repre.get("tries"), + '\n'.join(errors) ) self._data.append(item) self._rec_loaded += 1 - # log.info("!!! _add_page_records _rec_loaded:: {}".format(self._rec_loaded)) - def canFetchMore(self, index): """ Check if there are more records than currently loaded @@ -1022,7 +1089,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = filter self.refresh() - def get_default_query(self): + def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or projecting is needed. @@ -1033,6 +1100,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): [(dict)] - list with single dict - appropriate for aggregate function for MongoDB """ + if limit == 0: + limit = SyncRepresentationModel.PAGE_SIZE + return [ {"$match": self._get_match_part()}, {"$unwind": "$files"}, @@ -1052,13 +1122,15 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_remote.created_dt"}, + [1], + [0]]}]}} , 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} + {'$size': "$order_local.created_dt"}, + [1], + [0]]}]}} # file might be successfully created or failed, not both , 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, @@ -1067,7 +1139,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_remote.last_failed_dt"}, "$order_remote.last_failed_dt", []] - }]}} + } + ]}} , 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", @@ -1075,14 +1148,29 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$size': "$order_local.last_failed_dt"}, "$order_local.last_failed_dt", []] - }]}} + } + ]}} , 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} , 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + , 'failed_remote_error': {'$first': { + '$cond': [{'$size': "$order_remote.error"}, + "$order_remote.error", [""]]}} + , 'failed_local_error': {'$first': { + '$cond': [{'$size': "$order_local.error"}, + "$order_local.error", [""]]}} + , 'tries': {'$first': { + '$cond': [{'$size': "$order_local.tries"}, + "$order_local.tries", + {'$cond': [ + {'$size': "$order_remote.tries"}, + "$order_remote.tries", + []] + }]}} }}, {"$sort": self.sort}, - {"$limit": self.PAGE_SIZE}, + {"$limit": limit}, {"$skip": self._rec_loaded}, {"$project": self.projection} ] @@ -1106,9 +1194,70 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): "type": "representation", "_id": self._id, '$or': [{'files.path': {'$regex': regex_str, - '$options': 'i'}}] + '$options': 'i'}}] } + def get_default_projection(self): + """ + Projection part for aggregate query. + + All fields with '1' will be returned, no others. + + Returns: + (dict) + """ + return { + "files": 1, + 'progress_remote': 1, + 'progress_local': 1, + 'updated_dt_remote': 1, + 'updated_dt_local': 1, + 'failed_remote_error': 1, + 'failed_local_error': 1, + 'tries': 1, + 'status': { + '$switch': { + 'branches': [ + { + 'case': { + '$or': [{'$eq': ['$progress_remote', 0]}, + {'$eq': ['$progress_local', 0]}]}, + 'then': 0 + }, + { + 'case': { + '$or': ['$failed_remote', '$failed_local']}, + 'then': 1 + }, + { + 'case': {'$or': [{'$and': [ + {'$gt': ['$progress_remote', 0]}, + {'$lt': ['$progress_remote', 1]} + ]}, + {'$and': [ + {'$gt': ['$progress_local', 0]}, + {'$lt': ['$progress_local', 1]} + ]} + ]}, + 'then': 2 + }, + { + 'case': {'$eq': ['dummy_placeholder', 'paused']}, + 'then': 3 + }, + { + 'case': {'$and': [ + {'$eq': ['$progress_remote', 1]}, + {'$eq': ['$progress_local', 1]} + ]}, + 'then': 4 + }, + ], + 'default': -1 + } + } + } + class ImageDelegate(QtWidgets.QStyledItemDelegate): """ @@ -1116,6 +1265,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): """ def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) + self.icons = {} def paint(self, painter, option, index): d = index.data(QtCore.Qt.DisplayRole) @@ -1124,9 +1274,15 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): else: return - # log.info("data:: {} - {}".format(provider, value)) - pix_url = "../providers/resources/{}.png".format(provider) - pixmap = QtGui.QPixmap(pix_url) + if not self.icons.get(provider): + resource_path = os.path.dirname(__file__) + resource_path = os.path.join(resource_path, "..", + "providers", "resources") + pix_url = "{}/{}.png".format(resource_path, provider) + pixmap = QtGui.QPixmap(pix_url) + self.icons[provider] = pixmap + else: + pixmap = self.icons[provider] point = QtCore.QPoint(option.rect.x() + (option.rect.width() - pixmap.width()) / 2, @@ -1137,9 +1293,59 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): painter.setOpacity(0.5) overlay_rect = option.rect overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) - #painter.setCompositionMode(QtGui.QPainter.CompositionMode_DestinationOver) - #painter.setBrush(painter.brush(Qt.white)) - painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0,0,0,200))) + painter.fillRect(overlay_rect, + QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) + + +class SyncRepresentationErrorWindow(QtWidgets.QDialog): + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWindow, self).__init__(parent) + self.setWindowFlags(QtCore.Qt.Window) + self.setFocusPolicy(QtCore.Qt.StrongFocus) + + self.setStyleSheet(style.load_stylesheet()) + self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) + self.resize(250, 200) + + body = QtWidgets.QWidget() + footer = QtWidgets.QWidget() + footer.setFixedHeight(20) + + container = SyncRepresentationErrorWidget(_id, project, dt, tries, msg, + parent=self) + body_layout = QtWidgets.QHBoxLayout(body) + body_layout.addWidget(container) + body_layout.setContentsMargins(0, 0, 0, 0) + + message = QtWidgets.QLabel() + message.hide() + + footer_layout = QtWidgets.QVBoxLayout(footer) + footer_layout.addWidget(message) + footer_layout.setContentsMargins(0, 0, 0, 0) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(body) + layout.addWidget(footer) + + self.setLayout(body_layout) + self.setWindowTitle("Sync Representation Error Detail") + + +class SyncRepresentationErrorWidget(QtWidgets.QWidget): + """ + Dialog to show when sync error happened, prints error message + """ + def __init__(self, _id, project, dt, tries, msg, parent=None): + super(SyncRepresentationErrorWidget, self).__init__(parent) + + layout = QtWidgets.QFormLayout(self) + layout.addRow(QtWidgets.QLabel("Last update date"), + QtWidgets.QLabel(pretty_timestamp(dt))) + layout.addRow(QtWidgets.QLabel("Retries"), + QtWidgets.QLabel(str(tries))) + layout.addRow(QtWidgets.QLabel("Error message"), + QtWidgets.QLabel(msg)) class SizeDelegate(QtWidgets.QStyledItemDelegate): @@ -1163,39 +1369,3 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) - -# Back up the reference to the exceptionhook -sys._excepthook = sys.excepthook - -def my_exception_hook(exctype, value, traceback): - # Print the error and traceback - print(exctype, value, traceback) - # Call the normal Exception hook after - sys._excepthook(exctype, value, traceback) - sys.exit(1) - -# Set the exception hook to our wrapping function -sys.excepthook = my_exception_hook - -if __name__ == '__main__': - import sys - from time import sleep - app = QtWidgets.QApplication(sys.argv) - #app.setWindowIcon(QtGui.QIcon(style.app_icon_path())) - os.environ["PYPE_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" - os.environ["AVALON_DB"] = "avalon" - os.environ["AVALON_TIMEOUT"] = '3000' - - widget = SyncServerWindow() - widget.show() - - # while True: - # # run some codes that use QAxWidget.dynamicCall() function - # # print some results - # sleep(30) - - try: - sys.exit(app.exec_()) - except: - print("Exiting") From a6bbee9abf7b6d8c587be9b525b279d82410ad1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 7 Jan 2021 19:21:13 +0100 Subject: [PATCH 07/50] dropping support for pype repositories as zip files --- igniter/bootstrap_repos.py | 295 +++++++++++++++++++++----- igniter/install_thread.py | 54 ++++- pype.py | 80 +++++-- setup.py | 70 +++--- tests/igniter/test_bootstrap_repos.py | 213 ++++++++++++++----- 5 files changed, 559 insertions(+), 153 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 999a6daa19..448282b30b 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -1,21 +1,21 @@ # -*- coding: utf-8 -*- """Bootstrap Pype repositories.""" -import sys +import functools +import logging as log import os import re -import logging as log import shutil +import sys import tempfile -from typing import Union, Callable, List -from zipfile import ZipFile from pathlib import Path -import functools - -from speedcopy import copyfile +from typing import Union, Callable, List, Tuple +from zipfile import ZipFile, BadZipFile from appdirs import user_data_dir -from pype.version import __version__ +from speedcopy import copyfile + from pype.lib import PypeSettingsRegistry +from pype.version import __version__ from .tools import load_environments @@ -39,6 +39,9 @@ class PypeVersion: client = None path = None + _version_regex = re.compile( + r"(?P\d+)\.(?P\d+)\.(?P\d+)(-?((?Pstaging)|(?P.+))(-(?P.+))?)?") # noqa: E501 + @property def version(self): """return formatted version string.""" @@ -58,8 +61,6 @@ class PypeVersion: variant: str = "production", client: str = None, path: Path = None): self.path = path - self._version_regex = re.compile( - r"(?P\d+)\.(?P\d+)\.(?P\d+)(-?((?Pstaging)|(?P.+))(-(?P.+))?)?") # noqa: E501 if major is None or minor is None or subversion is None: if version is None: @@ -91,8 +92,9 @@ class PypeVersion: return version - def _decompose_version(self, version_string: str) -> tuple: - m = re.match(self._version_regex, version_string) + @classmethod + def _decompose_version(cls, version_string: str) -> tuple: + m = re.search(cls._version_regex, version_string) if not m: raise ValueError( "Cannot parse version string: {}".format(version_string)) @@ -138,6 +140,27 @@ class PypeVersion: return False + @staticmethod + def version_in_str(string: str) -> Tuple: + """Find Pype version in given string. + + Args: + string (str): string to search. + + Returns: + tuple: True/False and PypeVersion if found. + + """ + try: + result = PypeVersion._decompose_version(string) + except ValueError: + return False, None + return True, PypeVersion(major=result[0], + minor=result[1], + subversion=result[2], + variant=result[3], + client=result[4]) + class BootstrapRepos: """Class for bootstrapping local Pype installation. @@ -163,6 +186,7 @@ class BootstrapRepos: self._log = log.getLogger(str(__class__)) self.data_dir = Path(user_data_dir(self._app, self._vendor)) self.registry = PypeSettingsRegistry() + self.zip_filter = [".pyc", "__pycache__"] # dummy progress reporter def empty_progress(x: int): @@ -225,7 +249,7 @@ class BootstrapRepos: """Copy zip created from Pype repositories to user data dir. This detect Pype version either in local "live" Pype repository - or in user provided path. Then it will zip in in temporary directory + or in user provided path. Then it will zip it in temporary directory and finally it will move it to destination which is user data directory. Existing files will be replaced. @@ -252,7 +276,7 @@ class BootstrapRepos: # create zip inside temporary directory. with tempfile.TemporaryDirectory() as temp_dir: temp_zip = \ - Path(temp_dir) / f"pype-repositories-v{version}.zip" + Path(temp_dir) / f"pype-v{version}.zip" self._log.info(f"creating zip: {temp_zip}") self._create_pype_zip(temp_zip, repo_dir) @@ -275,7 +299,7 @@ class BootstrapRepos: except shutil.Error as e: self._log.error(e) return None - return self.data_dir / temp_zip.name + return destination def _create_pype_zip( self, @@ -287,9 +311,6 @@ class BootstrapRepos: to later implement file filter to skip git related stuff to make it into archive. - Todo: - Implement file filter - Args: zip_path (str): path to zip file. include_dir (Path): repo directories to include. @@ -310,18 +331,46 @@ class BootstrapRepos: with ZipFile(zip_path, "w") as zip_file: for root, _, files in os.walk(include_dir.as_posix()): for file in files: + progress += repo_inc + self._progress_callback(int(progress)) + + # skip all starting with '.' + if file.startswith("."): + continue + + # skip if direct parent starts with '.' + if Path(root).parts[-1].startswith("."): + continue + + # filter + if file in self.zip_filter: + continue + zip_file.write( os.path.relpath(os.path.join(root, file), os.path.join(include_dir, '..')), os.path.relpath(os.path.join(root, file), os.path.join(include_dir)) ) - progress += repo_inc - self._progress_callback(int(progress)) # add pype itself if include_pype: for root, _, files in os.walk("pype"): for file in files: + progress += pype_inc + self._progress_callback(int(progress)) + + # skip all starting with '.' + if file.startswith("."): + continue + + # skip if direct parent starts with '.' + if Path(root).parts[-1].startswith("."): + continue + + # filter + if file in self.zip_filter: + continue + zip_file.write( os.path.relpath(os.path.join(root, file), os.path.join('pype', '..')), @@ -330,8 +379,7 @@ class BootstrapRepos: os.path.relpath(os.path.join(root, file), os.path.join('pype', '..'))) ) - progress += pype_inc - self._progress_callback(int(progress)) + zip_file.testzip() self._progress_callback(100) @@ -342,8 +390,10 @@ class BootstrapRepos: This will enable Python to import modules is second-level directories in zip file. + Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. + Args: - archive (str): path to archive. + archive (Path): path to archive. """ with ZipFile(archive, "r") as zip_file: @@ -362,8 +412,37 @@ class BootstrapRepos: os.environ["PYTHONPATH"] = os.pathsep.join(paths) + @staticmethod + def add_paths_from_directory(directory: Path) -> None: + """Add first level directories as paths to :mod:`sys.path`. + + This works the same as :meth:`add_paths_from_archive` but in + specified directory. + + Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. + + Args: + directory (Path): path to directory. + + """ + roots = [] + for item in directory.iterdir(): + if item.is_dir(): + root = item.as_posix() + if root not in roots: + roots.append(root) + sys.path.insert(0, root) + + pythonpath = os.getenv("PYTHONPATH", "") + paths = pythonpath.split(os.pathsep) + paths += roots + + os.environ["PYTHONPATH"] = os.pathsep.join(paths) + def find_pype( - self, pype_path: Path = None) -> Union[List[PypeVersion], None]: + self, + pype_path: Path = None, + include_zips: bool = False) -> Union[List[PypeVersion], None]: """Get ordered dict of detected Pype version. Resolution order for Pype is following: @@ -374,6 +453,8 @@ class BootstrapRepos: Args: pype_path (Path, optional): Try to find Pype on the given path. + include_zips (bool, optional): If set True it will try to find + Pype in zip files in given directory. Returns: dict of Path: Dictionary of detected Pype version. @@ -383,42 +464,87 @@ class BootstrapRepos: """ dir_to_search = self.data_dir - if os.getenv("PYPE_PATH"): - if Path(os.getenv("PYPE_PATH")).exists(): - dir_to_search = Path(os.getenv("PYPE_PATH")) - else: - try: - registry_dir = Path(self.registry.get_item("pypePath")) - if registry_dir.exists(): - dir_to_search = registry_dir - except ValueError: - # nothing found in registry, we'll use data dir - pass - - # if we have pyp_path specified, search only there. + # if we have pype_path specified, search only there. if pype_path: dir_to_search = pype_path + else: + if os.getenv("PYPE_PATH"): + if Path(os.getenv("PYPE_PATH")).exists(): + dir_to_search = Path(os.getenv("PYPE_PATH")) + else: + try: + registry_dir = Path(str(self.registry.get_item("pypePath"))) + if registry_dir.exists(): + dir_to_search = registry_dir + + except ValueError: + # nothing found in registry, we'll use data dir + pass # pype installation dir doesn't exists if not dir_to_search.exists(): return None _pype_versions = [] - file_pattern = re.compile(r"^pype-repositories-v(?P\d+\.\d+\.\d*.+?).zip$") # noqa: E501 + # iterate over directory in first level and find all that might + # contain Pype. for file in dir_to_search.iterdir(): - m = re.match( - file_pattern, - file.name) - if m: - try: - _pype_versions.append( - PypeVersion( - version=m.group("version"), path=file)) - except ValueError: - # cannot parse version string - print(m) - pass + + result = PypeVersion.version_in_str(file.stem) + + if result[0]: + detected_version = result[1] + + if file.is_dir(): + # if item is directory that might (based on it's name) + # contain Pype version, check if it really does contain + # Pype and that their versions matches. + version_check = PypeVersion( + version=BootstrapRepos.get_version(file)) + if version_check != detected_version: + self._log.error( + (f"dir version ({detected_version}) and " + f"its content version ({version_check}) " + "doesn't match. Skipping.")) + continue + + if file.is_file(): + + if not include_zips: + continue + + # skip non-zip files + if file.suffix.lower() != ".zip": + continue + + # open zip file, look inside and parse version from Pype + # inside it. If there is none, or it is different from + # version specified in file name, skip it. + try: + with ZipFile(file, "r") as zip_file: + with zip_file.open( + "pype/pype/version.py") as version_file: + zip_version = {} + exec(version_file.read(), zip_version) + version_check = PypeVersion( + version=zip_version["__version__"]) + + if version_check != detected_version: + self._log.error( + (f"zip version ({detected_version}) and " + f"its content version ({version_check}) " + "doesn't match. Skipping.")) + continue + except BadZipFile: + self._log.error(f"{file} is not zip file") + continue + except KeyError: + self._log.error("Zip not containing Pype") + continue + + detected_version.path = file + _pype_versions.append(detected_version) return sorted(_pype_versions) @@ -426,16 +552,16 @@ class BootstrapRepos: def _get_pype_from_mongo(mongo_url: str) -> Union[Path, None]: """Get path from Mongo database. - This sets environment variable ``AVALON_MONGO`` for + This sets environment variable ``PYPE_MONGO`` for :mod:`pype.settings` to be able to read data from database. It will then retrieve environment variables and among them - must be ``PYPE_ROOT``. + must be ``PYPE_PATH``. Args: mongo_url (str): mongodb connection url Returns: - Path: if path from ``PYPE_ROOT`` is found. + Path: if path from ``PYPE_PATH`` is found. None: if not. """ @@ -479,11 +605,18 @@ class BootstrapRepos: self._log.error(f"{pype_path} doesn't exists.") return None - # find pype zip files in location. In that location, there can be - # either "live" Pype repository, or multiple zip files. + # test if entered path isn't user data dir + if self.data_dir == pype_path: + self._log.error(f"cannot point to user data dir") + return None + + # find pype zip files in location. There can be + # either "live" Pype repository, or multiple zip files or even + # multiple pype version directories. This process looks into zip + # files and directories and tries to parse `version.py` file. versions = self.find_pype(pype_path) if versions: - self._log.info(f"found Pype zips in [ {pype_path} ].") + self._log.info(f"found Pype in [ {pype_path} ]") self._log.info(f"latest version found is [ {versions[-1]} ]") destination = self.data_dir / versions[-1].path.name @@ -503,13 +636,43 @@ class BootstrapRepos: if not destination.parent.exists(): destination.parent.mkdir(parents=True) + # latest version found is directory + if versions[-1].path.is_dir(): + # zip it, copy it and extract it + # create zip inside temporary directory. + self._log.info("Creating zip from directory ...") + with tempfile.TemporaryDirectory() as temp_dir: + temp_zip = \ + Path(temp_dir) / f"pype-v{versions[-1]}.zip" + self._log.info(f"creating zip: {temp_zip}") + + self._create_pype_zip(temp_zip, versions[-1].path) + if not os.path.exists(temp_zip): + self._log.error("make archive failed.") + return None + + destination = self.data_dir / temp_zip.name + + elif versions[-1].path.is_file(): + # in this place, it must be zip file as `find_pype()` is + # checking just that. + assert versions[-1].path.suffix.lower() == ".zip", ( + "Invalid file format" + ) try: + self._log.info("Copying zip to destination ...") copyfile(versions[-1].path.as_posix(), destination.as_posix()) except OSError: self._log.error( "cannot copy detected version to user data directory", exc_info=True) return None + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(versions[-1].path, "r") as zip_ref: + zip_ref.extractall(destination) + return destination # if we got here, it means that location is "live" Pype repository. @@ -518,4 +681,22 @@ class BootstrapRepos: if not repo_file.exists(): self._log.error(f"installing zip {repo_file} failed.") return None - return repo_file + + destination = self.data_dir / repo_file.stem + if destination.exists(): + try: + destination.unlink() + except OSError: + self._log.error( + f"cannot remove already existing {destination}", + exc_info=True) + return None + + destination.mkdir(parents=True) + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(versions[-1].path, "r") as zip_ref: + zip_ref.extractall(destination) + + return destination diff --git a/igniter/install_thread.py b/igniter/install_thread.py index 278877bdf7..e4253958e5 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- """Working thread for installer.""" import os +from zipfile import ZipFile from Qt.QtCore import QThread, Signal from .bootstrap_repos import BootstrapRepos +from .bootstrap_repos import PypeVersion from .tools import validate_mongo_connection @@ -68,14 +70,62 @@ class InstallThread(QThread): os.environ["PYPE_MONGO"] = self._mongo + self.message.emit( + f"Detecting installed Pype versions in {bs.data_dir}", False) + detected = bs.find_pype() + + if detected: + if PypeVersion(version=local_version) < detected[-1]: + self.message.emit(( + f"Latest installed version {detected[-1]} is newer " + f"then currently running {local_version}" + ), False) + self.message.emit("Skipping Pype install ...", False) + return + + if PypeVersion(version=local_version) == detected[-1]: + self.message.emit(( + f"Latest installed version is the same as " + f"currently running {local_version}" + ), False) + self.message.emit("Skipping Pype install ...", False) + return + + self.message.emit(( + "All installed versions are older then " + f"currently running one {local_version}" + ), False) + else: + self.message.emit("None detected.", False) + self.message.emit( f"We will use local Pype version {local_version}", False) + repo_file = bs.install_live_repos() if not repo_file: self.message.emit( - f"!!! install failed - {repo_file}", True) + f"!!! Install failed - {repo_file}", True) return - self.message.emit(f"installed as {repo_file}", False) + + destination = bs.data_dir / repo_file.stem + if destination.exists(): + try: + destination.unlink() + except OSError as e: + self.message.emit( + f"!!! Cannot remove already existing {destination}", + True) + self.message.emit(e.strerror, True) + return + + destination.mkdir(parents=True) + + # extract zip there + self.message.emit("Extracting zip to destination ...", False) + with ZipFile(repo_file, "r") as zip_ref: + zip_ref.extractall(destination) + + self.message.emit(f"Installed as {repo_file}", False) else: # if we have mongo connection string, validate it, set it to # user settings and get PYPE_PATH from there. diff --git a/pype.py b/pype.py index 769e8c8f6f..d613e8e0fc 100644 --- a/pype.py +++ b/pype.py @@ -6,22 +6,24 @@ Bootstrapping process of Pype is as follows: `PYPE_PATH` is checked for existence - either one from environment or from user settings. Precedence takes the one set by environment. -On this path we try to find zip files with `pype-repositories-v3.x.x.zip` -format. +On this path we try to find pype in directories version string in their names. +For example: `pype-v3.0.1-foo` is valid name, or even `foo_3.0.2` - as long +as version can be determined from its name _AND_ file `pype/pype/version.py` +can be found inside, it is considered Pype installation. -If no Pype repositories are found in `PYPE_PATH (user data dir) +If no Pype repositories are found in `PYPE_PATH` (user data dir) then **Igniter** (Pype setup tool) will launch its GUI. It can be used to specify `PYPE_PATH` or if it is _not_ specified, current -*"live"* repositories will be used to create such zip file and copy it to -appdata dir in user home. Version will be determined by version specified -in Pype module. +*"live"* repositories will be used to create zip file and copy it to +appdata dir in user home and extract it there. Version will be determined by +version specified in Pype module. -If Pype repositories zip file is found in default install location -(user data dir) or in `PYPE_PATH`, it will get list of those zips there and +If Pype repository directories are found in default install location +(user data dir) or in `PYPE_PATH`, it will get list of those dirs there and use latest one or the one specified with optional `--use-version` command line argument. If the one specified doesn't exist then latest available -version will be used. All repositories in that zip will be added +version will be used. All repositories in that dir will be added to `sys.path` and `PYTHONPATH`. If Pype is live (not frozen) then current version of Pype module will be @@ -29,9 +31,56 @@ used. All directories under `repos` will be added to `sys.path` and `PYTHONPATH`. Pype depends on connection to `MongoDB`_. You can specify MongoDB connection -string via `AVALON_MONGO` set in environment or it can be set in user +string via `PYPE_MONGO` set in environment or it can be set in user settings or via **Igniter** GUI. +So, bootstrapping Pype looks like this:: + +.. code-block:: bash + ++-------------------------------------------------------+ +| Determine MongoDB connection: | +| Use `PYPE_MONGO`, system keyring `pypeMongo` | ++--------------------------|----------------------------+ + .--- Found? --. + YES NO + | | + | +------v--------------+ + | | Fire up Igniter GUI |<---------\ + | | and ask User | | + | +---------------------+ | + | | + | | ++-----------------v------------------------------------+ | +| Get location of Pype: | | +| 1) Test for `PYPE_PATH` environment variable | | +| 2) Test `pypePath` in registry setting | | +| 3) Test user data directory | | +| ................................................... | | +| If running from frozen code: | | +| - Use latest one found in user data dir | | +| If running from live code: | | +| - Use live code and install it to user data dir | | +| * can be overridden with `--use-version` argument | | ++-------------------------|----------------------------+ | + .-- Is Pype found? --. | + YES NO | + | | | + | +--------------v------------------+ | + | | Look in `PYPE_PATH`, find | | + | | latest version and install it | | + | | to user data dir. | | + | +--------------|------------------+ | + | .-- Is Pype found? --. | + | YES NO ---------/ + | | + |<--------/ + | ++-------------v------------+ +| Run Pype | ++--------------------------+ + + Todo: Move or remove bootstrapping environments out of the code. @@ -44,9 +93,8 @@ import re import sys import traceback -from igniter.tools import load_environments, add_acre_to_sys_path - from igniter import BootstrapRepos +from igniter.tools import load_environments, add_acre_to_sys_path try: import acre @@ -94,7 +142,7 @@ def set_modules_environments(): _publish_paths.add(os.path.normpath(path)) module_envs["PYBLISHPLUGINPATH"] = os.pathsep.join(_publish_paths) - # Metge environments with current environments and update values + # Merge environments with current environments and update values if module_envs: parsed_envs = acre.parse(module_envs) env = acre.merge(parsed_envs, dict(os.environ)) @@ -143,7 +191,7 @@ def boot(): use_version, pype_versions) if version_path: # use specified - bootstrap.add_paths_from_archive(version_path) + bootstrap.add_paths_from_directory(version_path) else: if use_version is not None: @@ -151,7 +199,7 @@ def boot(): "latest available")) # use latest version_path = pype_versions[-1].path - bootstrap.add_paths_from_archive(version_path) + bootstrap.add_paths_from_directory(version_path) use_version = str(pype_versions[-1]) os.environ["PYPE_ROOT"] = version_path.as_posix() @@ -164,7 +212,7 @@ def boot(): use_version, pype_versions) if version_path: # use specified - bootstrap.add_paths_from_archive(version_path) + bootstrap.add_paths_from_directory(version_path) os.environ["PYPE_ROOT"] = pype_root repos = os.listdir(os.path.join(pype_root, "repos")) diff --git a/setup.py b/setup.py index 5e32a1afe1..fcc2ddaf2f 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,24 @@ # -*- coding: utf-8 -*- """Setup info for building Pype 3.0.""" -import sys import os +import sys + from cx_Freeze import setup, Executable from sphinx.setup_command import BuildDoc version = {} with open(os.path.join("pype", "version.py")) as fp: exec(fp.read(), version) -__version__ = version['__version__'] +__version__ = version["__version__"] +base = None +if sys.platform == "win32": + # base = "Win32GUI" + ... + +# ----------------------------------------------------------------------- +# build_exe +# Build options for cx_Freeze. Manually add/exclude packages and binaries install_requires = [ "appdirs", @@ -21,40 +30,47 @@ install_requires = [ "pathlib2", "PIL", "pymongo", + "pynput", + "jinxed", + "blessed", "Qt", - "speedcopy", - "win32ctypes" + "speedcopy" ] -base = None -if sys.platform == "win32": - base = "Win32GUI" +includes = [ + "repos/acre/acre", + "repos/avalon-core/avalon", + "repos/pyblish-base/pyblish", + "repos/maya-look-assigner/mayalookassigner" +] + +excludes = [] +bin_includes = [] +include_files = [ + "igniter", + "pype", + "repos", + "schema", + "setup", + "vendor", + "LICENSE", + "README.md", + "pype/version.py" +] + +if sys.platform == "win32": + install_requires.append("win32ctypes") -# Build options for cx_Freeze. Manually add/exclude packages and binaries buildOptions = dict( packages=install_requires, - includes=[ - 'repos/acre/acre', - 'repos/avalon-core/avalon', - 'repos/pyblish-base/pyblish', - 'repos/maya-look-assigner/mayalookassigner' - ], - excludes=[], - bin_includes=[], - include_files=[ - "igniter", - "pype", - "repos", - "schema", - "setup", - "vendor", - "LICENSE", - "README.md", - "pype/version.py"] + includes=includes, + excludes=excludes, + bin_includes=bin_includes, + include_files=include_files ) -executables = [Executable("pype.py", base=None, targetName="pype")] +executables = [Executable("pype.py", base=base, targetName="pype")] setup( name="pype", diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index 868b356771..a04632c3cb 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -2,9 +2,13 @@ """Test suite for repos bootstrapping (install).""" import os import sys +from collections import namedtuple from pathlib import Path -import pytest +from zipfile import ZipFile + import appdirs +import pytest + from igniter.bootstrap_repos import BootstrapRepos from igniter.bootstrap_repos import PypeVersion from pype.lib import PypeSettingsRegistry @@ -116,6 +120,20 @@ def test_get_version_path_from_list(): assert path == Path("/bar/baz") +def test_search_string_for_pype_version(printer): + strings = [ + ("3.0.1", True), + ("foo-3.0", False), + ("foo-3.0.1", True), + ("3", False), + ("foo-3.0.1-staging-client", True), + ("foo-3.0.1-bar-baz", True) + ] + for ver_string in strings: + printer(f"testing {ver_string[0]} should be {ver_string[1]}") + assert PypeVersion.version_in_str(ver_string[0])[0] == ver_string[1] + + def test_install_live_repos(fix_bootstrap, printer): rf = fix_bootstrap.install_live_repos() sep = os.path.sep @@ -125,8 +143,7 @@ def test_install_live_repos(fix_bootstrap, printer): f"{rf}{sep}avalon-unreal-integration", f"{rf}{sep}maya-look-assigner", f"{rf}{sep}pyblish-base", - f"{rf}{sep}pype", - f"{rf}{sep}pype-config" + f"{rf}{sep}pype" ] printer("testing zip creation") assert os.path.exists(rf), "zip archive was not created" @@ -147,92 +164,186 @@ def test_install_live_repos(fix_bootstrap, printer): def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): + test_pype = namedtuple("Pype", "prefix version suffix type valid") + test_versions_1 = [ - "pype-repositories-v5.5.1.zip", - "pype-repositories-v5.5.2-client.zip", - "pype-repositories-v5.5.3-client-strange.zip", - "pype-repositories-v5.5.4-staging.zip", - "pype-repositories-v5.5.5-staging-client.zip", - "pype-repositories-v5.6.3.zip", - "pype-repositories-v5.6.3-staging.zip" + test_pype(prefix="foo-v", version="5.5.1", + suffix=".zip", type="zip", valid=False), + test_pype(prefix="bar-v", version="5.5.2-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="baz-v", version="5.5.3-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="bum-v", version="5.5.4-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="zum-v", version="5.5.5-staging-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="fam-v", version="5.6.3", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="5.6.3-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="fim-v", version="5.6.3", + suffix=".zip", type="zip", valid=False), + test_pype(prefix="foo-v", version="5.6.4", + suffix=".txt", type="txt", valid=False), + test_pype(prefix="foo-v", version="5.7.1", + suffix="", type="dir", valid=False), ] test_versions_2 = [ - "pype-repositories-v7.2.6.zip", - "pype-repositories-v7.2.7-client.zip", - "pype-repositories-v7.2.8-client-strange.zip", - "pype-repositories-v7.2.9-staging.zip", - "pype-repositories-v7.2.10-staging-client.zip", - "pype-repositories-v7.0.1.zip", + test_pype(prefix="foo-v", version="10.0.0", + suffix=".txt", type="txt", valid=False), + test_pype(prefix="lom-v", version="7.2.6", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="bom-v", version="7.2.7-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="woo-v", version="7.2.8-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="loo-v", version="7.2.10-staging-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="kok-v", version="7.0.1", + suffix=".zip", type="zip", valid=True) ] test_versions_3 = [ - "pype-repositories-v3.0.0.zip", - "pype-repositories-v3.0.1.zip", - "pype-repositories-v4.1.0.zip", - "pype-repositories-v4.1.2.zip", - "pype-repositories-v3.0.1-client.zip", - "pype-repositories-v3.0.1-client-strange.zip", - "pype-repositories-v3.0.1-staging.zip", - "pype-repositories-v3.0.1-staging-client.zip", - "pype-repositories-v3.2.0.zip", + test_pype(prefix="foo-v", version="3.0.0", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="goo-v", version="3.0.1", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="hoo-v", version="4.1.0", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="4.1.2", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-client-strange", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-staging", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.0.1-staging-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="foo-v", version="3.2.0", + suffix=".zip", type="zip", valid=True) ] + + def _create_invalid_zip(path: Path): + with ZipFile(path, "w") as zf: + zf.writestr("test.foo", "test") + + def _create_valid_zip(path: Path, version: str): + with ZipFile(path, "w") as zf: + zf.writestr( + "pype/pype/version.py", f"__version__ = '{version}'\n\n") + + def _create_invalid_dir(path: Path): + path.mkdir(parents=True, exist_ok=True) + with open(path / "invalid", "w") as fp: + fp.write("invalid") + + def _create_valid_dir(path: Path, version: str): + pype_path = path / "pype" + version_path = path / "pype" / "version.py" + pype_path.mkdir(parents=True, exist_ok=True) + with open(version_path, "w") as fp: + fp.write(f"__version__ = '{version}'\n\n") + + def _build_test_item(path, item): + test_path = path / "{}{}{}".format(item.prefix, + item.version, + item.suffix) + if item.type == "zip": + if item.valid: + _create_valid_zip(test_path, item.version) + else: + _create_invalid_zip(test_path) + elif item.type == "dir": + if item.valid: + _create_valid_dir(test_path, item.version) + else: + _create_invalid_dir(test_path) + else: + with open(test_path, "w") as fp: + fp.write("foo") # in PYPE_PATH e_path = tmp_path_factory.mktemp("environ") + + # create files and directories for test for test_file in test_versions_1: - with open(e_path / test_file, "w") as fp: - fp.write(test_file) + _build_test_item(e_path, test_file) # in pypePath registry - r_path = tmp_path_factory.mktemp("pypePath") + p_path = tmp_path_factory.mktemp("pypePath") for test_file in test_versions_2: - with open(r_path / test_file, "w") as fp: - fp.write(test_file) + _build_test_item(p_path, test_file) # in data dir - for test_file in test_versions_3: - with open(os.path.join(fix_bootstrap.data_dir, test_file), "w") as fp: - fp.write(test_file) + d_path = tmp_path_factory.mktemp("dataPath") + for test_file in test_versions_2: + _build_test_item(d_path, test_file) - result = fix_bootstrap.find_pype() + # in provided path + g_path = tmp_path_factory.mktemp("providedPath") + for test_file in test_versions_3: + _build_test_item(g_path, test_file) + + result = fix_bootstrap.find_pype(g_path, True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - fix_bootstrap.data_dir / test_versions_3[3] - ), "not a latest version of Pype 3" + expected_path = Path( + g_path / "{}{}{}".format( + test_versions_3[3].prefix, + test_versions_3[3].version, + test_versions_3[3].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 3" monkeypatch.setenv("PYPE_PATH", e_path.as_posix()) - result = fix_bootstrap.find_pype() + result = fix_bootstrap.find_pype(include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - e_path / test_versions_1[5] - ), "not a latest version of Pype 1" + expected_path = Path( + e_path / "{}{}{}".format( + test_versions_1[5].prefix, + test_versions_1[5].version, + test_versions_1[5].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 1" monkeypatch.delenv("PYPE_PATH", raising=False) # mock appdirs user_data_dir def mock_user_data_dir(*args, **kwargs): - return r_path.as_posix() + return d_path.as_posix() monkeypatch.setattr(appdirs, "user_data_dir", mock_user_data_dir) fix_bootstrap.registry = PypeSettingsRegistry() - fix_bootstrap.registry.set_item("pypePath", r_path.as_posix()) + fix_bootstrap.registry.set_item("pypePath", d_path.as_posix()) - result = fix_bootstrap.find_pype() + result = fix_bootstrap.find_pype(include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. - assert result[-1].path == Path( - r_path / test_versions_2[4] - ), "not a latest version of Pype 2" + expected_path = Path( + d_path / "{}{}{}".format( + test_versions_2[4].prefix, + test_versions_2[4].version, + test_versions_2[4].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 2" - result = fix_bootstrap.find_pype(e_path) + result = fix_bootstrap.find_pype(e_path, True) assert result is not None, "no Pype version found" - assert result[-1].path == Path( - e_path / test_versions_1[5] - ), "not a latest version of Pype 1" + expected_path = Path( + e_path / "{}{}{}".format( + test_versions_1[5].prefix, + test_versions_1[5].version, + test_versions_1[5].suffix + ) + ) + assert result[-1].path == expected_path, "not a latest version of Pype 1" From 9938f54da095c1166f878c6864b71ab8695a5b24 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 8 Jan 2021 13:04:16 +0100 Subject: [PATCH 08/50] use settings directly for getting sync sites in integrator --- pype/plugins/global/publish/integrate_new.py | 26 +++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 133b4fc6ef..82690ed59d 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -896,7 +896,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): file_info = self.prepare_file_info(path, integrated_file_sizes[dest], - file_hash) + file_hash, + instance=instance) output_resources.append(file_info) return output_resources @@ -916,7 +917,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dest += '.{}'.format(self.TMP_FILE_EXT) return dest - def prepare_file_info(self, path, size=None, file_hash=None, sites=None): + def prepare_file_info(self, path, size=None, file_hash=None, sites=None, instance=None): """ Prepare information for one file (asset or resource) Arguments: @@ -933,15 +934,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): remote_site = None sync_server_presets = None - # manager = ModulesManager() - # sync_server = manager.modules_by_name["sync_server"] - # try: - # if sync_server.enabled: - # local_site, remote_site = sync_server.get_sites_for_project() - # except ValueError: - # log.debug(("There are not set presets for SyncServer." - # " No credentials provided, no synching possible"). - # format(str(sync_server_presets))) + if (instance.context.data["system_settings"] + ["modules"] + ["sync_server"] + ["enabled"]): + sync_server_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_server_presets["enabled"]: + local_site = sync_server_presets["config"].get("active_site", + "studio").strip() + remote_site = sync_server_presets["config"].get("remote_site") rec = { "_id": io.ObjectId(), From 03e5907a1c7de4117a0bea1b7b6d0e587a272af3 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 8 Jan 2021 13:55:48 +0100 Subject: [PATCH 09/50] add pype root to requirements path --- tools/build.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/build.ps1 b/tools/build.ps1 index 5aa826a065..146fe9a904 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -42,6 +42,8 @@ $current_dir = Get-Location $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName +Set-Location -Path $pype_root + $version_file = Get-Content -Path "$($pype_root)\pype\version.py" $result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') $pype_version = $result[0].Groups['version'].Value @@ -100,7 +102,7 @@ catch { Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Installing packages to new venv ..." & python -m pip install -U pip -& pip install -r .\requirements.txt +& pip install -r ("$($pype_root)\requirements.txt") Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline From 65f4088797e37aa3b202cf7b5dbc41097a48df23 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 8 Jan 2021 14:02:15 +0100 Subject: [PATCH 10/50] move run tools --- dev_mongo.ps1 | 2 -- dev_settings.ps1 | 2 -- dev_tray.ps1 | 2 -- tools/run_mongo.ps1 | 6 ++++++ tools/run_settings.ps1 | 6 ++++++ tools/run_tray.ps1 | 6 ++++++ 6 files changed, 18 insertions(+), 6 deletions(-) delete mode 100644 dev_mongo.ps1 delete mode 100644 dev_settings.ps1 delete mode 100644 dev_tray.ps1 create mode 100644 tools/run_mongo.ps1 create mode 100644 tools/run_settings.ps1 create mode 100644 tools/run_tray.ps1 diff --git a/dev_mongo.ps1 b/dev_mongo.ps1 deleted file mode 100644 index 9ad021e39d..0000000000 --- a/dev_mongo.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py mongodb diff --git a/dev_settings.ps1 b/dev_settings.ps1 deleted file mode 100644 index 3eab14dc37..0000000000 --- a/dev_settings.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py settings --dev diff --git a/dev_tray.ps1 b/dev_tray.ps1 deleted file mode 100644 index 44f3f69754..0000000000 --- a/dev_tray.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -.\venv\Scripts\Activate.ps1 -python pype.py tray --debug diff --git a/tools/run_mongo.ps1 b/tools/run_mongo.ps1 new file mode 100644 index 0000000000..6cc5674a8e --- /dev/null +++ b/tools/run_mongo.ps1 @@ -0,0 +1,6 @@ +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +& "$($pype_root)\venv\Scripts\Activate.ps1" + +python "$($pype_root)\pype.py" mongodb diff --git a/tools/run_settings.ps1 b/tools/run_settings.ps1 new file mode 100644 index 0000000000..fde2087da7 --- /dev/null +++ b/tools/run_settings.ps1 @@ -0,0 +1,6 @@ +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +& "$($pype_root)\venv\Scripts\Activate.ps1" + +python "$($pype_root)\pype.py" settings --dev diff --git a/tools/run_tray.ps1 b/tools/run_tray.ps1 new file mode 100644 index 0000000000..41ada4219e --- /dev/null +++ b/tools/run_tray.ps1 @@ -0,0 +1,6 @@ +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +& "$($pype_root)\venv\Scripts\Activate.ps1" + +python "$($pype_root)\pype.py" tray --debug From 18d63433e0f190321038cf1b6c196d7a8f06f34f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 8 Jan 2021 19:29:25 +0100 Subject: [PATCH 11/50] fixed tests, renamed entry point --- igniter/bootstrap_repos.py | 30 ++++++++++++++++++++------- igniter/install_thread.py | 10 ++++++++- setup.py | 12 +++++++---- pype.py => start.py | 1 + tests/igniter/test_bootstrap_repos.py | 27 ++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 13 deletions(-) rename pype.py => start.py (99%) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 448282b30b..29f6c7249c 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -346,12 +346,12 @@ class BootstrapRepos: if file in self.zip_filter: continue - zip_file.write( - os.path.relpath(os.path.join(root, file), - os.path.join(include_dir, '..')), - os.path.relpath(os.path.join(root, file), + file_name = os.path.relpath( + os.path.join(root, file), + os.path.join(include_dir, '..')) + arc_name = os.path.relpath(os.path.join(root, file), os.path.join(include_dir)) - ) + zip_file.write(file_name, arc_name) # add pype itself if include_pype: for root, _, files in os.walk("pype"): @@ -491,7 +491,13 @@ class BootstrapRepos: # contain Pype. for file in dir_to_search.iterdir(): - result = PypeVersion.version_in_str(file.stem) + # if file, strip extension, in case of dir not. + if file.is_dir(): + name = file.name + else: + name = file.stem + + result = PypeVersion.version_in_str(name) if result[0]: detected_version = result[1] @@ -500,8 +506,16 @@ class BootstrapRepos: # if item is directory that might (based on it's name) # contain Pype version, check if it really does contain # Pype and that their versions matches. - version_check = PypeVersion( - version=BootstrapRepos.get_version(file)) + try: + # add one 'pype' level as inside dir there should + # be many other repositories. + version_str = BootstrapRepos.get_version( + file / "pype") + version_check = PypeVersion(version=version_str) + except ValueError: + self._log.error( + f"cannot determine version from {file}") + continue if version_check != detected_version: self._log.error( (f"dir version ({detected_version}) and " diff --git a/igniter/install_thread.py b/igniter/install_thread.py index e4253958e5..e31de731ed 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Working thread for installer.""" import os +import sys from zipfile import ZipFile from Qt.QtCore import QThread, Signal @@ -96,7 +97,14 @@ class InstallThread(QThread): f"currently running one {local_version}" ), False) else: - self.message.emit("None detected.", False) + # we cannot build install package from frozen code. + if getattr(sys, 'frozen', False): + self.message.emit("None detected.", True) + self.message.emit(("Please set path to Pype sources to " + "build installation."), False) + return + else: + self.message.emit("None detected.", False) self.message.emit( f"We will use local Pype version {local_version}", False) diff --git a/setup.py b/setup.py index fcc2ddaf2f..38b8cf1dbc 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,7 @@ __version__ = version["__version__"] base = None if sys.platform == "win32": - # base = "Win32GUI" - ... + base = "Win32GUI" # ----------------------------------------------------------------------- # build_exe @@ -34,7 +33,9 @@ install_requires = [ "jinxed", "blessed", "Qt", - "speedcopy" + "speedcopy", + "googleapiclient", + "httplib2" ] includes = [ @@ -70,7 +71,10 @@ buildOptions = dict( ) -executables = [Executable("pype.py", base=base, targetName="pype")] +executables = [ + Executable("start.py", base=None, targetName="pype_console"), + Executable("start.py", base=base, targetName="pype") +] setup( name="pype", diff --git a/pype.py b/start.py similarity index 99% rename from pype.py rename to start.py index d613e8e0fc..ad5411a679 100644 --- a/pype.py +++ b/start.py @@ -167,6 +167,7 @@ def boot(): m = re.search(r"--use-version=(?P\d+\.\d+\.\d*.+?)", arg) if m and m.group('version'): use_version = m.group('version') + sys.argv.remove(arg) break if not os.getenv("PYPE_MONGO"): diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index a04632c3cb..730c6f2917 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -224,6 +224,17 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): test_pype(prefix="foo-v", version="3.2.0", suffix=".zip", type="zip", valid=True) ] + + test_versions_4 = [ + test_pype(prefix="foo-v", version="10.0.0", + suffix="", type="dir", valid=True), + test_pype(prefix="lom-v", version="11.2.6", + suffix=".zip", type="dir", valid=False), + test_pype(prefix="bom-v", version="7.2.7-client", + suffix=".zip", type="zip", valid=True), + test_pype(prefix="woo-v", version="7.2.8-client-strange", + suffix=".zip", type="txt", valid=False) + ] def _create_invalid_zip(path: Path): with ZipFile(path, "w") as zf: @@ -286,6 +297,12 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): for test_file in test_versions_3: _build_test_item(g_path, test_file) + # dir vs zip preference + dir_path = tmp_path_factory.mktemp("dirZipPath") + for test_file in test_versions_4: + _build_test_item(dir_path, test_file) + + printer("testing finding Pype in given path ...") result = fix_bootstrap.find_pype(g_path, True) # we should have results as file were created assert result is not None, "no Pype version found" @@ -347,3 +364,13 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): ) ) assert result[-1].path == expected_path, "not a latest version of Pype 1" + + result = fix_bootstrap.find_pype(dir_path, True) + assert result is not None, "no Pype versions found" + expected_path = Path( + e_path / "{}{}{}".format( + test_versions_4[0].prefix, + test_versions_4[0].version, + test_versions_4[0].suffix + ) + ) From e03c337660ed0e4f35a38393595e91980914119a Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 11 Jan 2021 21:58:45 +0100 Subject: [PATCH 12/50] filtering zip file --- igniter/bootstrap_repos.py | 123 ++++++++++++++++++++++-------------- igniter/install_thread.py | 3 +- pype/lib/terminal_splash.py | 10 ++- setup.py | 1 + start.py | 5 +- 5 files changed, 90 insertions(+), 52 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 29f6c7249c..0c764002a8 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -169,15 +169,20 @@ class BootstrapRepos: data_dir (Path): local Pype installation directory. live_repo_dir (Path): path to repos directory if running live, otherwise `None`. + registry (PypeSettingsRegistry): Pype registry object. + zip_filter (list): List of files to exclude from zip + pype_filter (list): list of top level directories not to include in + zip in Pype repository. """ - def __init__(self, progress_callback: Callable = None): + def __init__(self, progress_callback: Callable = None, message = None): """Constructor. Args: progress_callback (callable): Optional callback method to report progress. + message (QtCore.Signal, optional): Signal to report messages back. """ # vendor and app used to construct user data dir @@ -187,6 +192,10 @@ class BootstrapRepos: self.data_dir = Path(user_data_dir(self._app, self._vendor)) self.registry = PypeSettingsRegistry() self.zip_filter = [".pyc", "__pycache__"] + self.pype_filter = [ + "build", "docs", "tests", "repos", "tools", "venv" + ] + self._message = message # dummy progress reporter def empty_progress(x: int): @@ -308,8 +317,9 @@ class BootstrapRepos: """Pack repositories and Pype into zip. We are using :mod:`zipfile` instead :meth:`shutil.make_archive` - to later implement file filter to skip git related stuff to make - it into archive. + because we need to decide what file and directories to include in zip + and what not. They are determined by :attr:`zip_filter` on file level + and :attr:`pype_filter` on top level directory in Pype repository. Args: zip_path (str): path to zip file. @@ -317,69 +327,82 @@ class BootstrapRepos: include_pype (bool): add Pype module itself. """ - repo_files = sum(len(files) for _, _, files in os.walk(include_dir)) + include_dir = include_dir.resolve() + + def _filter_dir(path: Path, path_filter: List) -> List[Path]: + """Recursively crawl over path and filter.""" + result = [] + for item in path.iterdir(): + if item.name in path_filter: + continue + if item.name.startswith('.'): + continue + if item.is_dir(): + result.extend(_filter_dir(item, path_filter)) + else: + result.append(item) + return result + + pype_list = [] + # get filtered list of files in repositories (repos directory) + repo_list = _filter_dir(include_dir, self.zip_filter) + # count them + repo_files = len(repo_list) + + # there must be some files, otherwise `include_dir` path is wrong assert repo_files != 0, f"No repositories to include in {include_dir}" pype_inc = 0 if include_pype: - pype_files = sum(len(files) for _, _, files in os.walk( - include_dir.parent)) + # get filtered list of file in Pype repository + pype_list = _filter_dir(include_dir.parent, self.zip_filter) + pype_files = len(pype_list) repo_inc = 48.0 / float(repo_files) pype_inc = 48.0 / float(pype_files) else: repo_inc = 98.0 / float(repo_files) progress = 0 + with ZipFile(zip_path, "w") as zip_file: - for root, _, files in os.walk(include_dir.as_posix()): - for file in files: - progress += repo_inc - self._progress_callback(int(progress)) + file: Path + for file in repo_list: + progress += repo_inc + self._progress_callback(int(progress)) - # skip all starting with '.' - if file.startswith("."): - continue + # archive name is relative to repos dir + arc_name = file.relative_to(include_dir) + zip_file.write(file, arc_name) - # skip if direct parent starts with '.' - if Path(root).parts[-1].startswith("."): - continue - - # filter - if file in self.zip_filter: - continue - - file_name = os.path.relpath( - os.path.join(root, file), - os.path.join(include_dir, '..')) - arc_name = os.path.relpath(os.path.join(root, file), - os.path.join(include_dir)) - zip_file.write(file_name, arc_name) # add pype itself if include_pype: - for root, _, files in os.walk("pype"): - for file in files: - progress += pype_inc - self._progress_callback(int(progress)) + pype_root = include_dir.parent.resolve() + # generate list of filtered paths + dir_filter = [pype_root / f for f in self.pype_filter] - # skip all starting with '.' - if file.startswith("."): - continue + file: Path + for file in pype_list: + progress += pype_inc + self._progress_callback(int(progress)) - # skip if direct parent starts with '.' - if Path(root).parts[-1].startswith("."): - continue + # if file resides in filtered path, skip it + is_inside = None + df: Path + for df in dir_filter: + try: + is_inside = file.resolve().relative_to(df) + except ValueError: + pass - # filter - if file in self.zip_filter: - continue + if is_inside: + continue - zip_file.write( - os.path.relpath(os.path.join(root, file), - os.path.join('pype', '..')), - os.path.join( - 'pype', - os.path.relpath(os.path.join(root, file), - os.path.join('pype', '..'))) - ) + processed_path = file + self._log.debug(f"processing {processed_path}") + self._print(f"- processing {processed_path}", False) + zip_file.write(file, + "pype" / file.relative_to(pype_root)) + + # test if zip is ok zip_file.testzip() self._progress_callback(100) @@ -714,3 +737,7 @@ class BootstrapRepos: zip_ref.extractall(destination) return destination + + def _print(self, message, error=False): + if self._message: + self._message.emit(message, error) diff --git a/igniter/install_thread.py b/igniter/install_thread.py index e31de731ed..bca8848f7e 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -46,7 +46,8 @@ class InstallThread(QThread): self.message.emit("Installing Pype ...", False) # find local version of Pype - bs = BootstrapRepos(progress_callback=self.set_progress) + bs = BootstrapRepos( + progress_callback=self.set_progress, message=self.message) local_version = bs.get_local_version() # if user did entered nothing, we install Pype from local version. diff --git a/pype/lib/terminal_splash.py b/pype/lib/terminal_splash.py index 7a94f2243e..0d148bd6e2 100644 --- a/pype/lib/terminal_splash.py +++ b/pype/lib/terminal_splash.py @@ -4,12 +4,20 @@ import blessed from pathlib import Path from time import sleep +NO_TERMINAL = False -term = blessed.Terminal() +try: + term = blessed.Terminal() +except AttributeError: + # this happens when blessed cannot find proper terminal. + # If so, skip printing ascii art animation. + NO_TERMINAL = True def play_animation(): """Play ASCII art Pype animation.""" + if NO_TERMINAL: + return print(term.home + term.clear) frame_size = 7 splash_file = Path(__file__).parent / "splash.txt" diff --git a/setup.py b/setup.py index 38b8cf1dbc..287f8706c8 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ install_requires = [ "jsonschema", "OpenTimelineIO", "pathlib2", + "pkg_resources", "PIL", "pymongo", "pynput", diff --git a/start.py b/start.py index ad5411a679..083fc35d02 100644 --- a/start.py +++ b/start.py @@ -203,10 +203,11 @@ def boot(): bootstrap.add_paths_from_directory(version_path) use_version = str(pype_versions[-1]) - os.environ["PYPE_ROOT"] = version_path.as_posix() + os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) else: # run through repos and add them to sys.path and PYTHONPATH - pype_root = os.path.dirname(os.path.realpath(__file__)) + pype_root = os.path.normpath( + os.path.dirname(os.path.realpath(__file__))) local_version = bootstrap.get_local_version() if use_version and use_version != local_version: version_path = BootstrapRepos.get_version_path_from_list( From 1044ba33574a015f508d47c5f1a474ac9244dedf Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 11 Jan 2021 22:00:16 +0100 Subject: [PATCH 13/50] add PYPE_EXECUTABLE --- start.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/start.py b/start.py index 083fc35d02..e590599863 100644 --- a/start.py +++ b/start.py @@ -229,6 +229,8 @@ def boot(): paths += repos os.environ["PYTHONPATH"] = os.pathsep.join(paths) + os.environ["PYPE_EXECUTABLE"] = sys.executable + # DEPRECATED: remove when `pype-config` dissolves into Pype for good. # .-=-----------------------=-=. ^ .=-=--------------------------=-. os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"] From bc9a7c19895e57f0ab5900a7cf8387fdf53ee798 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 11 Jan 2021 22:26:03 +0100 Subject: [PATCH 14/50] fix hound bitching --- igniter/bootstrap_repos.py | 14 ++++++++------ tests/igniter/test_bootstrap_repos.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 0c764002a8..04d119bf1f 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -176,7 +176,7 @@ class BootstrapRepos: """ - def __init__(self, progress_callback: Callable = None, message = None): + def __init__(self, progress_callback: Callable = None, message=None): """Constructor. Args: @@ -497,7 +497,8 @@ class BootstrapRepos: dir_to_search = Path(os.getenv("PYPE_PATH")) else: try: - registry_dir = Path(str(self.registry.get_item("pypePath"))) + registry_dir = Path( + str(self.registry.get_item("pypePath"))) if registry_dir.exists(): dir_to_search = registry_dir @@ -542,7 +543,7 @@ class BootstrapRepos: if version_check != detected_version: self._log.error( (f"dir version ({detected_version}) and " - f"its content version ({version_check}) " + f"its content version ({version_check}) " "doesn't match. Skipping.")) continue @@ -569,8 +570,9 @@ class BootstrapRepos: if version_check != detected_version: self._log.error( - (f"zip version ({detected_version}) and " - f"its content version ({version_check}) " + (f"zip version ({detected_version}) " + f"and its content version " + f"({version_check}) " "doesn't match. Skipping.")) continue except BadZipFile: @@ -644,7 +646,7 @@ class BootstrapRepos: # test if entered path isn't user data dir if self.data_dir == pype_path: - self._log.error(f"cannot point to user data dir") + self._log.error("cannot point to user data dir") return None # find pype zip files in location. There can be diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index 730c6f2917..db6890f13b 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -235,7 +235,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): test_pype(prefix="woo-v", version="7.2.8-client-strange", suffix=".zip", type="txt", valid=False) ] - + def _create_invalid_zip(path: Path): with ZipFile(path, "w") as zf: zf.writestr("test.foo", "test") From cd15d111f3b835b336dc6536e7bc0d6fd7cab634 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 12 Jan 2021 12:00:40 +0100 Subject: [PATCH 15/50] move acre and pyblish-base to requirements --- .gitmodules | 7 ----- build.ps1 | 74 +++++++++++++++++++++++++++++++++++++----------- igniter/tools.py | 4 +-- requirements.txt | 2 ++ setup.py | 4 +-- start.py | 1 - 6 files changed, 64 insertions(+), 28 deletions(-) diff --git a/.gitmodules b/.gitmodules index f08a36506c..20aa3a3e8d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,19 +2,12 @@ path = repos/avalon-core url = git@github.com:pypeclub/avalon-core.git branch = develop -[submodule "repos/pyblish-base"] - path = repos/pyblish-base - url = git@github.com:pyblish/pyblish-base.git [submodule "repos/avalon-unreal-integration"] path = repos/avalon-unreal-integration url = git@github.com:pypeclub/avalon-unreal-integration.git [submodule "repos/maya-look-assigner"] path = repos/maya-look-assigner url = git@github.com:pypeclub/maya-look-assigner.git -[submodule "repos/acre"] - path = repos/acre - url = git@github.com:antirotor/acre.git - branch = fix/unformatted-tokens [submodule "pype/modules/ftrack/python2_vendor/ftrack-python-api"] path = pype/modules/ftrack/python2_vendor/ftrack-python-api url = https://bitbucket.org/ftrack/ftrack-python-api.git diff --git a/build.ps1 b/build.ps1 index 39ce90e36a..ec118a43f3 100644 --- a/build.ps1 +++ b/build.ps1 @@ -13,7 +13,6 @@ PS> .\build.ps1 #> - function Exit-WithCode($exitcode) { # Only exit this host process if it's a child of another PowerShell parent process... $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId @@ -23,6 +22,18 @@ function Exit-WithCode($exitcode) { exit $exitcode } +if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - https://github.com/PowerShell/PowerShell/releases" + Exit-WithCode 1 +} + +$arguments=$ARGS +if($arguments -eq "--skip-venv") { + $skip_venv=$true +} + $art = @' @@ -76,22 +87,39 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { Exit-WithCode 1 } Write-Host "OK [ $p ]" -ForegroundColor green -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Creating virtual env ..." -& python -m venv venv -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Entering venv ..." -try { - . (".\venv\Scripts\Activate.ps1") + + +if ($skip_venv -ne $true) { + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Creating virtual env ..." + & python -m venv venv + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Entering venv ..." + try { + . (".\venv\Scripts\Activate.ps1") + } + catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 + } + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Installing packages to new venv ..." + & pip install -r .\requirements.txt +} else { + Write-Host "*** " -NoNewline -ForegroundColor yellow + Write-Host "Skipping creaton of venv ..." + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Entering venv ..." + try { + . (".\venv\Scripts\Activate.ps1") + } + catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 + } } -catch { - Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 -} -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Installing packages to new venv ..." -& pip install -r .\requirements.txt Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline @@ -99,7 +127,21 @@ Get-ChildItem . -Filter "*.pyc" -Force -Recurse | Remove-Item -Force Get-ChildItem . -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse Write-Host "OK" -ForegroundColor green +# store original PYTHONPATH +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Storing original PYTHONPATH ... " -NoNewline +$original_pythonpath = $env:PYTHONPATH +Write-Host "OK" -ForegroundColor green +$new_pythonpath = Get-ChildItem -Directory -Path .\ | Microsoft.PowerShell.Utility\Join-String -Property FullName -DoubleQuote -Separator ';' +$env:PYTHONPATH = $env:PYTHONPATH + ";" + $new_pythonpath +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Adding repos to PYTHONPATH ..." + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype ..." & python setup.py build +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Restoring original PYTHONPATH ... " -NoNewline +$env:PYTHONPATH = $original_pythonpath +Write-Host "OK" -ForegroundColor green deactivate diff --git a/igniter/tools.py b/igniter/tools.py index a06c444e62..387a1934b1 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -82,13 +82,13 @@ def validate_path_string(path: str) -> (bool, str): def add_acre_to_sys_path(): - """Add full path of acre module to sys.path on ignitation.""" + """Add full path of acre module to sys.path on ignition.""" try: # Skip if is possible to import import acre except ImportError: - # Full path to acred repository related to current file + # Full path to acre repository related to current file acre_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "repos", diff --git a/requirements.txt b/requirements.txt index f7d46a8dcc..3263431948 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +git+https://github.com/antirotor/acre.git@fix/unformatted-tokens aiohttp aiohttp_json_rpc appdirs @@ -18,6 +19,7 @@ log4mongo git+https://github.com/pypeclub/OpenTimelineIO.git@develop pathlib2 Pillow +pyblish-base pycodestyle pydocstyle pylint diff --git a/setup.py b/setup.py index 287f8706c8..964c088e4f 100644 --- a/setup.py +++ b/setup.py @@ -73,8 +73,8 @@ buildOptions = dict( executables = [ - Executable("start.py", base=None, targetName="pype_console"), - Executable("start.py", base=base, targetName="pype") + Executable("start.py", base=None, target_name="pype_console"), + Executable("start.py", base=base, target_name="pype") ] setup( diff --git a/start.py b/start.py index dc34c51ba7..b18e908417 100644 --- a/start.py +++ b/start.py @@ -110,7 +110,6 @@ def set_environments() -> None: better handling of environments """ - # FIXME: remove everything except global env = load_environments(["global"]) env = acre.merge(env, dict(os.environ)) os.environ.clear() From 924ae33a5159f5ab3ac648288980407b5e64f6e1 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 12 Jan 2021 12:01:19 +0100 Subject: [PATCH 16/50] remove submodules --- repos/acre | 1 - repos/pyblish-base | 1 - 2 files changed, 2 deletions(-) delete mode 160000 repos/acre delete mode 160000 repos/pyblish-base diff --git a/repos/acre b/repos/acre deleted file mode 160000 index 3d06232bd4..0000000000 --- a/repos/acre +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3d06232bd424df4350efe64ab459944b4096ca74 diff --git a/repos/pyblish-base b/repos/pyblish-base deleted file mode 160000 index 3290fd5b51..0000000000 --- a/repos/pyblish-base +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3290fd5b51b496d431cc2ca608639e21e727ccbd From 84c8e9e8b11c35d5556fd39bf35522fba9ac3f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 12 Jan 2021 15:33:31 +0100 Subject: [PATCH 17/50] fixes for build, acre in pypeclub fork --- build.ps1 | 4 ++++ igniter/install_dialog.py | 7 ++++--- igniter/install_thread.py | 3 ++- igniter/tools.py | 22 ---------------------- requirements.txt | 2 +- setup.py | 14 ++++---------- start.py | 10 +++------- 7 files changed, 18 insertions(+), 44 deletions(-) diff --git a/build.ps1 b/build.ps1 index ec118a43f3..782b3c8f20 100644 --- a/build.ps1 +++ b/build.ps1 @@ -103,6 +103,10 @@ if ($skip_venv -ne $true) { Write-Host $_.Exception.Message Exit-WithCode 1 } + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Updating pip ..." + & python -m pip install --upgrade pip + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Installing packages to new venv ..." & pip install -r .\requirements.txt diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index 93c94977c6..404f1b2320 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -149,10 +149,10 @@ class InstallDialog(QtWidgets.QDialog): self.setLayout(mongo_layout) def _mongo_changed(self, mongo: str): - self.parent()._mongo_url = mongo + self.parent().mongo_url = mongo def get_mongo_url(self): - return self.parent()._mongo_url + return self.parent().mongo_url def set_valid(self): self._mongo_input.setStyleSheet( @@ -303,7 +303,7 @@ class InstallDialog(QtWidgets.QDialog): options |= QtWidgets.QFileDialog.DontUseNativeDialog options |= QtWidgets.QFileDialog.ShowDirsOnly - filename, _ = QtWidgets.QFileDialog.getOpenFileName( + filename, _ = QtWidgets.QFileDialog.getExistingDirectory( parent=self, caption='Select path', directory=os.getcwd(), @@ -378,6 +378,7 @@ class InstallDialog(QtWidgets.QDialog): if len(self._path) < 1: self._mongo.setVisible(False) + return path def _update_console(self, msg: str, error: bool = False) -> None: """Display message in console. diff --git a/igniter/install_thread.py b/igniter/install_thread.py index bca8848f7e..bb4ab1c9dd 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -146,10 +146,11 @@ class InstallThread(QThread): bs.registry.set_secure_item("pypeMongo", self._mongo) os.environ["PYPE_MONGO"] = self._mongo + self.message.emit(f"processing {self._path}", True) repo_file = bs.process_entered_location(self._path) if not repo_file: - self.message.emit(f"!!! Cannot install", True) + self.message.emit("!!! Cannot install", True) return def set_path(self, path: str) -> None: diff --git a/igniter/tools.py b/igniter/tools.py index 387a1934b1..e7cb99865d 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Tools used in **Igniter** GUI.""" import os -import sys import uuid from urllib.parse import urlparse @@ -81,26 +80,6 @@ def validate_path_string(path: str) -> (bool, str): return False, "Not implemented yet" -def add_acre_to_sys_path(): - """Add full path of acre module to sys.path on ignition.""" - try: - # Skip if is possible to import - import acre - - except ImportError: - # Full path to acre repository related to current file - acre_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "repos", - "acre" - ) - # Add path to sys.path - sys.path.append(acre_dir) - - # Validate that acre can be imported - import acre - - def load_environments(sections: list = None) -> dict: """Load environments from Pype. @@ -114,7 +93,6 @@ def load_environments(sections: list = None) -> dict: dict of str: loaded and processed environments. """ - add_acre_to_sys_path() import acre from pype import settings diff --git a/requirements.txt b/requirements.txt index 3263431948..5164159842 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/antirotor/acre.git@fix/unformatted-tokens +git+https://github.com/pypeclub/acre.git aiohttp aiohttp_json_rpc appdirs diff --git a/setup.py b/setup.py index 964c088e4f..99e1fe75b9 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ install_requires = [ "keyring", "clique", "jsonschema", - "OpenTimelineIO", + "opentimelineio", "pathlib2", "pkg_resources", "PIL", @@ -39,13 +39,7 @@ install_requires = [ "httplib2" ] -includes = [ - "repos/acre/acre", - "repos/avalon-core/avalon", - "repos/pyblish-base/pyblish", - "repos/maya-look-assigner/mayalookassigner" -] - +includes = [] excludes = [] bin_includes = [] include_files = [ @@ -63,7 +57,7 @@ include_files = [ if sys.platform == "win32": install_requires.append("win32ctypes") -buildOptions = dict( +build_options = dict( packages=install_requires, includes=includes, excludes=excludes, @@ -83,7 +77,7 @@ setup( description="Ultimate pipeline", cmdclass={"build_sphinx": BuildDoc}, options={ - "build_exe": buildOptions, + "build_exe": build_options, "build_sphinx": { "project": "Pype", "version": __version__, diff --git a/start.py b/start.py index b18e908417..82dfc1dce9 100644 --- a/start.py +++ b/start.py @@ -93,14 +93,10 @@ import re import sys import traceback -from igniter import BootstrapRepos -from igniter.tools import load_environments, add_acre_to_sys_path +import acre -try: - import acre -except ImportError: - add_acre_to_sys_path() - import acre +from igniter import BootstrapRepos +from igniter.tools import load_environments def set_environments() -> None: From 5e06361968c8a186ba29b25ebb88838c017de0d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 12 Jan 2021 22:53:53 +0100 Subject: [PATCH 18/50] streamline pype installation, updated tools --- README.md | 97 ++++++++++++++++--- igniter/bootstrap_repos.py | 26 +++++ igniter/install_dialog.py | 7 +- igniter/install_thread.py | 6 +- pype/cli.py | 15 ++- pype/pype_commands.py | 30 +++++- requirements.txt | 1 - tests/igniter/test_bootstrap_repos.py | 2 - tools/build.ps1 | 90 ++++++++++++++--- tools/create_env.ps1 | 103 ++++++++------------ tools/create_zip.ps1 | 106 ++++++++++++++++++++ tools/make_docs.ps1 | 4 +- tools/run_mongo.ps1 | 3 +- tools/run_settings.ps1 | 3 +- tools/run_tests.bat | 2 - build.ps1 => tools/run_tests.ps1 | 133 +++++++++++--------------- tools/run_tray.ps1 | 3 +- 17 files changed, 447 insertions(+), 184 deletions(-) create mode 100644 tools/create_zip.ps1 delete mode 100644 tools/run_tests.bat rename build.ps1 => tools/run_tests.ps1 (51%) diff --git a/README.md b/README.md index 4ad52c7e36..d206a2b73b 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,106 @@ -# Pype -## Introduction +Pype +==== -Multi-platform open-source pipeline built around the [Avalon](https://getavalon.github.io/) platform, expanding it with extra features and integrations. Pype connects asset database, project management and time tracking into a single modular system. It has tight integration with [ftrack](https://www.ftrack.com/en/), but it can also run independently. +Introduction +------------ + +Multi-platform open-source pipeline built around the [Avalon](https://getavalon.github.io/) platform, +expanding it with extra features and integrations. Pype connects asset database, project management +and time tracking into a single modular system. It has tight integration +with [ftrack](https://www.ftrack.com/en/), but it can also run independently. To get all the key information about the project, go to [PYPE.club](http://pype.club) -## Hardware requirements +Requirements +------------ +Pype will run on most typical hardware configurations commonly found in studios around the world. +It is installed on artist computer and can take up 3Gb of space depending on number of versions +and other dependencies. -Pype should be installed centrally on a fast network storage with at least read access right for all workstations and users in the Studio. Full Deplyoyment with all dependencies and both Development and Production branches installed takes about 1GB of data, however to ensure smooth updates and general working comfort, we recommend allocating at least at least 4GB of storage dedicated to PYPE deployment. +For well functioning [ftrack](https://www.ftrack.com/en/) event server, we recommend a +linux virtual server with [Ubuntu](https://ubuntu.com/) or [CentosOS](https://www.centos.org/). +CPU and RAM allocation need differ based on the studio size, but a 2GB of RAM, with a +dual core CPU and around 4GB of storage should suffice. -For well functioning [ftrack](https://www.ftrack.com/en/) event server, we recommend a linux virtual server with [Ubuntu](https://ubuntu.com/) or [CentosOS](https://www.centos.org/). CPU and RAM allocation need differ based on the studio size, but a 2GB of RAM, with a dual core CPU and around 4GB of storage should suffice. +Pype needs running [mongodb](https://www.mongodb.com/) server with good connectivity as it is +heavily used by Pype. Depending on project size and number of artists working connection speed and +latency influence performance experienced by artists. If remote working is required, this mongodb +server must be accessible from Internet or cloud solution can be used. Reasonable backup plan +or high availability options are recommended. -## Building Pype +Building Pype +------------- ### Windows You will need [Python 3.7 and newer](https://www.python.org/downloads/) and [git](https://git-scm.com/downloads). +More tools might be needed for installing dependencies (for example for **OpenTimelineIO**) - mostly +development tools like [CMake](https://cmake.org/) and [Visual Studio](https://visualstudio.microsoft.com/cs/downloads/) Clone repository: ```sh git clone --recurse-submodules git@github.com:pypeclub/pype.git ``` -Run PowerShell script `build.ps1`. It will create *venv*, install all -required dependencies and build Pype. After it is finished, you will find -Pype in `build` folder. +#### To build Pype: -You might need more tools for installing dependencies (for example for **OpenTimelineIO**) - mostly -development tools like [CMake](https://cmake.org/) and [Visual Studio](https://visualstudio.microsoft.com/cs/downloads/) +1) Run `.\tools\create_env.ps1` to create virtual environment in `.\venv` +2) Run `.\tools\build.ps1` to build pype executables in `.\build\` + +To create distributable Pype versions, run `./tools/create_zip.ps1` - that will +create zip file with name `pype-vx.x.x.zip` parsed from current pype repository and +copy it to user data dir, or you can specify `--path /path/to/zip` to force it there. + +You can then point **Igniter** - Pype setup tool - to directory containing this zip and +it will install it on current computer. Pype is build using [CX_Freeze](https://cx-freeze.readthedocs.io/en/latest) to freeze itself and all dependencies. + +Running Pype +------------ + +Pype can by executed either from live sources (this repository) or from +*"frozen code"* - executables that can be build using steps described above. + +If Pype is executed from live sources, it will use Pype version included in them. If +it is executed from frozen code it will try to find latest Pype version installed locally +on current computer and if it is not found, it will ask for its location. On that location +pype can be either in directories or zip files. Pype will try to find latest version and +install it to user data directory (on Windows to `%LOCALAPPDATA%\pypeclub\pype`). + +### From sources +Pype can be run directly from sources by activating virtual environment: +```powershell +.\venv\Scripts\Activate.ps1 +``` +and running: +```powershell +python start.py tray +``` +This will use current Pype version with sources. You can override this with `--use-version=x.x.x` and +then Pype will try to find locally installed specified version (present in user data directory). + +### From frozen code + +You need to build Pype first. This will produce two executables - `pype.exe` and `pype_console.exe`. +First one will act as GUI application and will not create console (useful in production environments). +The second one will create console and will write output there - useful for headless application and +debugging purposes. If you need pype version installed, just run `./tools/create_zip.ps1` without +arguments and it will create zip file that pype can use. + + +Building documentation +---------------------- + +Top build API documentation, run `.\tools\make_docs.ps1`. It will create html documentation +from current sources in `.\docs\build`. + +**Note that it needs existing virtual environment.** + +Running tests +------------- + +To run tests, execute `.\tools\run_tests.ps1`. + +**Note that it needs existing virtual environment.** \ No newline at end of file diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 04d119bf1f..ba4a21d5d7 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -743,3 +743,29 @@ class BootstrapRepos: def _print(self, message, error=False): if self._message: self._message.emit(message, error) + + def extract_pype(self, version: PypeVersion): + if not version.path: + raise ValueError( + f"version {version} is not associated with any file") + + destination = self.data_dir / version.path.stem + if destination.exists(): + try: + destination.unlink() + except OSError as e: + msg = f"!!! Cannot remove already existing {destination}" + self._log.error(msg) + self._log.error(e.strerror) + self._print(msg, True) + self._print(e.strerror, True) + return + + destination.mkdir(parents=True) + + # extract zip there + self._print("Extracting zip to destination ...") + with ZipFile(version.path, "r") as zip_ref: + zip_ref.extractall(destination) + + self._print(f"Installed as {version.path.stem}") diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index 404f1b2320..74d9435815 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -19,7 +19,7 @@ class InstallDialog(QtWidgets.QDialog): def __init__(self, parent=None): super(InstallDialog, self).__init__(parent) - self._mongo_url = "" + self._mongo_url = os.getenv("PYPE_MONGO", "") self.setWindowTitle("Pype - Configure Pype repository path") self._icon_path = os.path.join( @@ -154,6 +154,9 @@ class InstallDialog(QtWidgets.QDialog): def get_mongo_url(self): return self.parent().mongo_url + def set_mongo_url(self, mongo: str): + self._mongo_input.setText(mongo) + def set_valid(self): self._mongo_input.setStyleSheet( """ @@ -175,6 +178,8 @@ class InstallDialog(QtWidgets.QDialog): ) self._mongo = MongoWidget(self) + if self._mongo_url: + self._mongo.set_mongo_url(self._mongo_url) # Bottom button bar # -------------------------------------------------------------------- diff --git a/igniter/install_thread.py b/igniter/install_thread.py index bb4ab1c9dd..37232bb88e 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -74,7 +74,7 @@ class InstallThread(QThread): self.message.emit( f"Detecting installed Pype versions in {bs.data_dir}", False) - detected = bs.find_pype() + detected = bs.find_pype(include_zips=True) if detected: if PypeVersion(version=local_version) < detected[-1]: @@ -83,6 +83,8 @@ class InstallThread(QThread): f"then currently running {local_version}" ), False) self.message.emit("Skipping Pype install ...", False) + if detected[-1].path.suffix.lower() == ".zip": + bs.extract_pype(detected[-1]) return if PypeVersion(version=local_version) == detected[-1]: @@ -91,6 +93,8 @@ class InstallThread(QThread): f"currently running {local_version}" ), False) self.message.emit("Skipping Pype install ...", False) + if detected[-1].path.suffix.lower() == ".zip": + bs.extract_pype(detected[-1]) return self.message.emit(( diff --git a/pype/cli.py b/pype/cli.py index 9d66ff988a..93858f15d0 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- """Package for handling pype command line arguments.""" import os + +import click + # import sys from .pype_commands import PypeCommands -import click @click.group(invoke_without_command=True) @@ -258,3 +260,14 @@ def launch(app, project, asset, task, def validate_config(): """Validate all json configuration files for errors.""" PypeCommands().validate_jsons() + + +@main.command() +@click.option("-p", "--path", help="Path to zip file", default=None) +def generate_zip(path): + """Generate Pype zip from current sources. + + If PATH is not provided, it will create zip file in user data dir. + + """ + PypeCommands().generate_zip(path) diff --git a/pype/pype_commands.py b/pype/pype_commands.py index 36e5ac639f..7c2f83a562 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- """Implementation of Pype commands.""" import os -import subprocess import sys +from pathlib import Path + from pype.lib import PypeLogger @@ -61,3 +62,30 @@ class PypeCommands: def validate_jsons(self): pass + + @staticmethod + def generate_zip(out_path: str): + """Generate zip file from current sources. + + Args: + out_path (str): Path to generated zip file. + + """ + from igniter import bootstrap_repos + + # create zip file + + + bs = bootstrap_repos.BootstrapRepos() + if out_path: + out_path = Path(out_path) + bs.data_dir = out_path.parent + + print(f">>> Creating zip in {bs.data_dir} ...") + repo_file = bs.install_live_repos() + if not repo_file: + print("!!! Error while creating zip file.") + exit(1) + + print(f">>> Created {repo_file}") + diff --git a/requirements.txt b/requirements.txt index 5164159842..8320b586c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ git+https://github.com/pypeclub/acre.git -aiohttp aiohttp_json_rpc appdirs arrow diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index db6890f13b..c0ce1be012 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -138,11 +138,9 @@ def test_install_live_repos(fix_bootstrap, printer): rf = fix_bootstrap.install_live_repos() sep = os.path.sep expected_paths = [ - f"{rf}{sep}acre", f"{rf}{sep}avalon-core", f"{rf}{sep}avalon-unreal-integration", f"{rf}{sep}maya-look-assigner", - f"{rf}{sep}pyblish-base", f"{rf}{sep}pype" ] printer("testing zip creation") diff --git a/tools/build.ps1 b/tools/build.ps1 index 146fe9a904..a233281585 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -13,6 +13,38 @@ PS> .\build.ps1 #> +function Start-Progress { + param([ScriptBlock]$code) + $scroll = "/-\|/-\|" + $idx = 0 + $job = Invoke-Command -ComputerName $env:ComputerName -ScriptBlock { $code } -AsJob + + $origpos = $host.UI.RawUI.CursorPosition + + # $origpos.Y -= 1 + + while (($job.State -eq "Running") -and ($job.State -ne "NotStarted")) + { + $host.UI.RawUI.CursorPosition = $origpos + Write-Host $scroll[$idx] -NoNewline + $idx++ + if ($idx -ge $scroll.Length) + { + $idx = 0 + } + Start-Sleep -Milliseconds 100 + } + # It's over - clear the activity indicator. + $host.UI.RawUI.CursorPosition = $origpos + Write-Host ' ' + <# + .SYNOPSIS + Display spinner for running job + .PARAMETER code + Job to display spinner for + #> +} + function Exit-WithCode($exitcode) { # Only exit this host process if it's a child of another PowerShell parent process... @@ -23,7 +55,17 @@ function Exit-WithCode($exitcode) { exit $exitcode } -$art = @' +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} + +$art = @" ____________ @@ -34,10 +76,13 @@ $art = @' \ \____\ \ \_____\ \__\\__\\__\ \/____/ \/_____/ . PYPE Club . -'@ +"@ Write-Host $art -ForegroundColor DarkGreen +# Enable if PS 7.x is needed. +# Show-PSWarning + $current_dir = Get-Location $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName @@ -68,10 +113,10 @@ if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { Write-Host "!!! Python not detected" -ForegroundColor red Exit-WithCode 1 } -$version_command = @' +$version_command = @" import sys print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) -'@ +"@ $p = & python -c $version_command $env:PYTHON_VERSION = $p @@ -86,9 +131,7 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { Exit-WithCode 1 } Write-Host "OK [ $p ]" -ForegroundColor green -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Creating virtual env ..." -& python -m venv "$($pype_root)\venv" + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Entering venv ..." try { @@ -96,13 +139,18 @@ try { } catch { Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Trying to create env ..." + & "$($script_dir)\create_env.ps1" + try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") + } + catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 + } } -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Installing packages to new venv ..." -& python -m pip install -U pip -& pip install -r ("$($pype_root)\requirements.txt") Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline @@ -112,7 +160,19 @@ Write-Host "OK" -ForegroundColor green Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype ..." -Set-Location -Path $pype_root -& python setup.py build +$out = & python setup.py build 2>&1 + +Set-Content -Path "$($pype_root)\build\build.log" -Value $out + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "deactivating venv ..." deactivate + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "restoring current directory" Set-Location -Path $current_dir + +Write-Host "*** " -NoNewline -ForegroundColor Cyan +Write-Host "All done. You will find Pype and build log in " -NoNewLine +Write-Host "'.\build'" -NoNewline -ForegroundColor Green +Write-Host " directory." diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 index c5afbe12b2..7354ee72ed 100644 --- a/tools/create_env.ps1 +++ b/tools/create_env.ps1 @@ -21,22 +21,21 @@ function Exit-WithCode($exitcode) { exit $exitcode } + + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} $current_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $current_dir).parent.FullName -if ($PSVersionTable.PSVersion.Major -lt 7) { - Write-Host "!!! " -NoNewline -ForegroundColor Red - Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" - Write-Host "Please update to at least 7.0 - https://github.com/PowerShell/PowerShell/releases" - Exit-WithCode 1 -} - -$arguments=$ARGS -if($arguments -eq "--skip-venv") { - $skip_venv=$true -} - -$art = @' +$art = @" ____________ @@ -47,10 +46,13 @@ $art = @' \ \____\ \ \_____\ \__\\__\\__\ \/____/ \/_____/ . PYPE Club . -'@ +"@ Write-Host $art -ForegroundColor DarkGreen +# Enable if PS 7.x is needed. +# Show-PSWarning + $version_file = Get-Content -Path "$($pype_root)\pype\version.py" $result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') $pype_version = $result[0].Groups['version'].Value @@ -85,42 +87,30 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { } Write-Host "OK [ $p ]" -ForegroundColor green +Write-Host "--- " -NoNewline -ForegroundColor yellow +Write-Host "Cleaning venv directory ..." +Remove-Item -Recurse -Force "$($pype_root)\venv\*" -if ($skip_venv -ne $true) { - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Creating virtual env ..." - & python -m venv venv - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Entering venv ..." - try { - . (".\venv\Scripts\Activate.ps1") - } - catch { - Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 - } - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Updating pip ..." - & python -m pip install --upgrade pip - - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Installing packages to new venv ..." - & pip install -r .\requirements.txt -} else { - Write-Host "*** " -NoNewline -ForegroundColor yellow - Write-Host "Skipping creaton of venv ..." - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Entering venv ..." - try { - . (".\venv\Scripts\Activate.ps1") - } - catch { - Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 - } +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Creating virtual env ..." +& python -m venv venv +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . (".\venv\Scripts\Activate.ps1") } +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 +} +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Updating pip ..." +& python -m pip install --upgrade pip + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Installing packages to new venv ..." +& pip install -r "$($pype_root)\requirements.txt" Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline @@ -128,21 +118,6 @@ Get-ChildItem "$($pype_root)" -Filter "*.pyc" -Force -Recurse | Remove-Item -For Get-ChildItem "$($pype_root)" -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse Write-Host "OK" -ForegroundColor green -# store original PYTHONPATH Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Storing original PYTHONPATH ... " -NoNewline -$original_pythonpath = $env:PYTHONPATH -Write-Host "OK" -ForegroundColor green -$new_pythonpath = Get-ChildItem -Directory -Path .\ | Microsoft.PowerShell.Utility\Join-String -Property FullName -DoubleQuote -Separator ';' -$env:PYTHONPATH = $env:PYTHONPATH + ";" + $new_pythonpath -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Adding repos to PYTHONPATH ..." - -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Building Pype ..." -& python setup.py build -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Restoring original PYTHONPATH ... " -NoNewline -$env:PYTHONPATH = $original_pythonpath -Write-Host "OK" -ForegroundColor green +Write-Host "Deactivating venv ..." deactivate diff --git a/tools/create_zip.ps1 b/tools/create_zip.ps1 new file mode 100644 index 0000000000..b2d0319534 --- /dev/null +++ b/tools/create_zip.ps1 @@ -0,0 +1,106 @@ +<# +.SYNOPSIS + Helper script create virtual env. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\build.ps1 + +#> + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + + +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } +} +$current_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $current_dir).parent.FullName + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +# Enable if PS 7.x is needed. +# Show-PSWarning + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" +$result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') +$pype_version = $result[0].Groups['version'].Value +if (-not $pype_version) { + Write-Host "!!! " -ForegroundColor yellow -NoNewline + Write-Host "Cannot determine Pype version." + Exit-WithCode 1 +} + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Detecting host Python ... " -NoNewline +if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { + Write-Host "!!! Python not detected" -ForegroundColor red + Exit-WithCode 1 +} +$version_command = @' +import sys +print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) +'@ + +$p = & python -c $version_command +$env:PYTHON_VERSION = $p +$m = $p -match '(\d+)\.(\d+)' +if(-not $m) { + Write-Host "!!! Cannot determine version" -ForegroundColor red + Exit-WithCode 1 +} +# We are supporting python 3.6 and up +if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { + Write-Host "FAILED Version [ $p ] is old and unsupported" -ForegroundColor red + Exit-WithCode 1 +} +Write-Host "OK [ $p ]" -ForegroundColor green + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . (".\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host $_.Exception.Message + Exit-WithCode 1 +} +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Generating zip from current sources ..." +& python "$($pype_root)\start.py" generate-zip $ARGS + +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Deactivating venv ..." +deactivate diff --git a/tools/make_docs.ps1 b/tools/make_docs.ps1 index 30032d41a6..951cab4c8e 100644 --- a/tools/make_docs.ps1 +++ b/tools/make_docs.ps1 @@ -17,7 +17,7 @@ $current_dir = Get-Location $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName -$art = @' +$art = @" ____________ @@ -28,7 +28,7 @@ $art = @' \ \____\ \ \_____\ \__\\__\\__\ \/____/ \/_____/ . PYPE Club . -'@ +"@ Write-Host $art -ForegroundColor DarkGreen diff --git a/tools/run_mongo.ps1 b/tools/run_mongo.ps1 index 6cc5674a8e..45d4679940 100644 --- a/tools/run_mongo.ps1 +++ b/tools/run_mongo.ps1 @@ -3,4 +3,5 @@ $pype_root = (Get-Item $script_dir).parent.FullName & "$($pype_root)\venv\Scripts\Activate.ps1" -python "$($pype_root)\pype.py" mongodb +python "$($pype_root)\start.py" mongodb +deactivate diff --git a/tools/run_settings.ps1 b/tools/run_settings.ps1 index fde2087da7..64d35a4b81 100644 --- a/tools/run_settings.ps1 +++ b/tools/run_settings.ps1 @@ -3,4 +3,5 @@ $pype_root = (Get-Item $script_dir).parent.FullName & "$($pype_root)\venv\Scripts\Activate.ps1" -python "$($pype_root)\pype.py" settings --dev +python "$($pype_root)\start.py" settings --dev +deactivate \ No newline at end of file diff --git a/tools/run_tests.bat b/tools/run_tests.bat deleted file mode 100644 index 48ccf9e7cc..0000000000 --- a/tools/run_tests.bat +++ /dev/null @@ -1,2 +0,0 @@ -set PYTHONPATH=".;%PYTHONPATH%" -pytest -x --capture=sys --print -W ignore::DeprecationWarning ./tests diff --git a/build.ps1 b/tools/run_tests.ps1 similarity index 51% rename from build.ps1 rename to tools/run_tests.ps1 index 782b3c8f20..498a3c3d7d 100644 --- a/build.ps1 +++ b/tools/run_tests.ps1 @@ -1,18 +1,3 @@ -<# -.SYNOPSIS - Helper script to build Pype. - -.DESCRIPTION - This script will detect Python installation, create venv and install - all necessary packages from `requirements.txt` needed by Pype to be - included during application freeze on Windows. - -.EXAMPLE - -PS> .\build.ps1 - -#> - function Exit-WithCode($exitcode) { # Only exit this host process if it's a child of another PowerShell parent process... $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId @@ -22,19 +7,16 @@ function Exit-WithCode($exitcode) { exit $exitcode } -if ($PSVersionTable.PSVersion.Major -lt 7) { - Write-Host "!!! " -NoNewline -ForegroundColor Red - Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" - Write-Host "Please update to at least 7.0 - https://github.com/PowerShell/PowerShell/releases" - Exit-WithCode 1 +function Show-PSWarning() { + if ($PSVersionTable.PSVersion.Major -lt 7) { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "You are using old version of PowerShell. $($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + Write-Host "Please update to at least 7.0 - " -NoNewline -ForegroundColor Gray + Write-Host "https://github.com/PowerShell/PowerShell/releases" -ForegroundColor White + Exit-WithCode 1 + } } - -$arguments=$ARGS -if($arguments -eq "--skip-venv") { - $skip_venv=$true -} - -$art = @' +$art = @" ____________ @@ -45,11 +27,20 @@ $art = @' \ \____\ \ \_____\ \__\\__\\__\ \/____/ \/_____/ . PYPE Club . -'@ +"@ Write-Host $art -ForegroundColor DarkGreen -$version_file = Get-Content -Path ".\pype\version.py" +# Enable if PS 7.x is needed. +# Show-PSWarning + +$current_dir = Get-Location +$script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent +$pype_root = (Get-Item $script_dir).parent.FullName + +Set-Location -Path $pype_root + +$version_file = Get-Content -Path "$($pype_root)\pype\version.py" $result = [regex]::Matches($version_file, '__version__ = "(?\d+\.\d+.\d+)"') $pype_version = $result[0].Groups['version'].Value if (-not $pype_version) { @@ -61,7 +52,7 @@ if (-not $pype_version) { Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype [ " -NoNewline -ForegroundColor white Write-host $pype_version -NoNewline -ForegroundColor green -Write-Host " ]..." -ForegroundColor white +Write-Host " ] ..." -ForegroundColor white Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Detecting host Python ... " -NoNewline @@ -69,10 +60,10 @@ if (-not (Get-Command "python" -ErrorAction SilentlyContinue)) { Write-Host "!!! Python not detected" -ForegroundColor red Exit-WithCode 1 } -$version_command = @' +$version_command = @" import sys print('{0}.{1}'.format(sys.version_info[0], sys.version_info[1])) -'@ +"@ $p = & python -c $version_command $env:PYTHON_VERSION = $p @@ -88,64 +79,48 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { } Write-Host "OK [ $p ]" -ForegroundColor green - -if ($skip_venv -ne $true) { - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Creating virtual env ..." - & python -m venv venv - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Entering venv ..." - try { - . (".\venv\Scripts\Activate.ps1") - } - catch { +Write-Host ">>> " -NoNewline -ForegroundColor green +Write-Host "Entering venv ..." +try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") +} +catch { + Write-Host "!!! Failed to activate" -ForegroundColor red + Write-Host ">>> " -NoNewline -ForegroundColor green + Write-Host "Trying to create env ..." + & "$($script_dir)\create_env.ps1" + try { + . ("$($pype_root)\venv\Scripts\Activate.ps1") + } + catch { Write-Host "!!! Failed to activate" -ForegroundColor red Write-Host $_.Exception.Message Exit-WithCode 1 - } - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Updating pip ..." - & python -m pip install --upgrade pip - - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Installing packages to new venv ..." - & pip install -r .\requirements.txt -} else { - Write-Host "*** " -NoNewline -ForegroundColor yellow - Write-Host "Skipping creaton of venv ..." - Write-Host ">>> " -NoNewline -ForegroundColor green - Write-Host "Entering venv ..." - try { - . (".\venv\Scripts\Activate.ps1") - } - catch { - Write-Host "!!! Failed to activate" -ForegroundColor red - Write-Host $_.Exception.Message - Exit-WithCode 1 - } + } } Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline -Get-ChildItem . -Filter "*.pyc" -Force -Recurse | Remove-Item -Force -Get-ChildItem . -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse +Get-ChildItem $pype_root -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse Write-Host "OK" -ForegroundColor green -# store original PYTHONPATH Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Storing original PYTHONPATH ... " -NoNewline +Write-Host "Testing Pype ..." $original_pythonpath = $env:PYTHONPATH -Write-Host "OK" -ForegroundColor green -$new_pythonpath = Get-ChildItem -Directory -Path .\ | Microsoft.PowerShell.Utility\Join-String -Property FullName -DoubleQuote -Separator ';' -$env:PYTHONPATH = $env:PYTHONPATH + ";" + $new_pythonpath +$env:PYTHONPATH="$($pype_root);$($env:PYTHONPATH)" +pytest -x --capture=sys --print -W ignore::DeprecationWarning "$($pype_root)/tests" +$env:PYTHONPATH = $original_pythonpath Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Adding repos to PYTHONPATH ..." +Write-Host "deactivating venv ..." +deactivate Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Building Pype ..." -& python setup.py build -Write-Host ">>> " -NoNewline -ForegroundColor green -Write-Host "Restoring original PYTHONPATH ... " -NoNewline -$env:PYTHONPATH = $original_pythonpath -Write-Host "OK" -ForegroundColor green -deactivate +Write-Host "restoring current directory" +Set-Location -Path $current_dir + + + + + + diff --git a/tools/run_tray.ps1 b/tools/run_tray.ps1 index 41ada4219e..f794f85929 100644 --- a/tools/run_tray.ps1 +++ b/tools/run_tray.ps1 @@ -3,4 +3,5 @@ $pype_root = (Get-Item $script_dir).parent.FullName & "$($pype_root)\venv\Scripts\Activate.ps1" -python "$($pype_root)\pype.py" tray --debug +python "$($pype_root)\start.py" tray --debug +deactivate From 057f6437feede45f97df34ad15e92491fa194dba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 12 Jan 2021 22:55:52 +0100 Subject: [PATCH 19/50] fix hound --- pype/pype_commands.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pype/pype_commands.py b/pype/pype_commands.py index 7c2f83a562..1ec4d2c553 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -74,8 +74,6 @@ class PypeCommands: from igniter import bootstrap_repos # create zip file - - bs = bootstrap_repos.BootstrapRepos() if out_path: out_path = Path(out_path) @@ -88,4 +86,3 @@ class PypeCommands: exit(1) print(f">>> Created {repo_file}") - From b88f2a846169252e5e1d6a0874137d33e59903c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 13 Jan 2021 00:00:51 +0100 Subject: [PATCH 20/50] add run command --- pype/cli.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pype/cli.py b/pype/cli.py index 93858f15d0..e07786e76a 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Package for handling pype command line arguments.""" import os +import sys import click @@ -271,3 +272,25 @@ def generate_zip(path): """ PypeCommands().generate_zip(path) + + +@main.command( + context_settings=dict( + ignore_unknown_options=True, + allow_extra_args=True)) +@click.argument("script", required=True, type=click.Path(exists=True)) +def run(script): + """Run python script.""" + import runpy + + if not script: + print("Error: missing path to script file.") + else: + + args = sys.argv + args.remove("run") + args.remove(script) + sys.argv = args + args_string = " ".join(args[1:]) + print(f"... running: {script} {args_string}") + runpy.run_path(script, run_name="__main__", ) From 47b4385947175abe7cc89eda4ed0d043b6893467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 13 Jan 2021 10:44:38 +0100 Subject: [PATCH 21/50] check for build and venv dirs --- pype/cli.py | 3 ++- tools/build.ps1 | 15 ++++++++++++++- tools/create_env.ps1 | 17 ++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pype/cli.py b/pype/cli.py index e07786e76a..1f7834a91e 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -23,6 +23,7 @@ def main(ctx): @main.command() @click.option("-d", "--dev", is_flag=True, help="Settings in Dev mode") def settings(dev=False): + """Show Pype Settings UI.""" PypeCommands().launch_settings_gui(dev) @@ -280,7 +281,7 @@ def generate_zip(path): allow_extra_args=True)) @click.argument("script", required=True, type=click.Path(exists=True)) def run(script): - """Run python script.""" + """Run python script in Pype context.""" import runpy if not script: diff --git a/tools/build.ps1 b/tools/build.ps1 index a233281585..389846d8ea 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -98,9 +98,22 @@ if (-not $pype_version) { Exit-WithCode 1 } +# Create build directory if not exist +if (-not (Test-Path -PathType Container -Path "$($pype_root)\build")) { + New-Item -ItemType Directory -Force -Path "$($pype_root)\build" +} + Write-Host "--- " -NoNewline -ForegroundColor yellow Write-Host "Cleaning build directory ..." -Remove-Item -Recurse -Force "$($pype_root)\build\*" +try { + Remove-Item -Recurse -Force "$($pype_root)\build\*" +} +catch { + Write-Host "!!! " -NoNewline -ForegroundColor Red + Write-Host "Cannot clean build directory, possibly because process is using it." + Write-Host $_.Exception.Message + Exit-WithCode 1 +} Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype [ " -NoNewline -ForegroundColor white diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 index 7354ee72ed..1fee947a38 100644 --- a/tools/create_env.ps1 +++ b/tools/create_env.ps1 @@ -87,9 +87,24 @@ if(($matches[1] -lt 3) -or ($matches[2] -lt 7)) { } Write-Host "OK [ $p ]" -ForegroundColor green +# Create venv directory if not exist +if (-not (Test-Path -PathType Container -Path "$($pype_root)\venv")) { + New-Item -ItemType Directory -Force -Path "$($pype_root)\venv" +} + Write-Host "--- " -NoNewline -ForegroundColor yellow Write-Host "Cleaning venv directory ..." -Remove-Item -Recurse -Force "$($pype_root)\venv\*" + +try { + Remove-Item -Recurse -Force "$($pype_root)\venv\*" +} +catch { + Write-Host "!!! " -NoNewline -ForegroundColor red + Write-Host "Cannot clean venv directory, possibly another process is using it." + Write-Host $_.Exception.Message + Exit-WithCode 1 +} + Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Creating virtual env ..." From 066bb450498884d26d11d938e8d27b7e80debbf9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 14 Jan 2021 17:00:12 +0100 Subject: [PATCH 22/50] fix ftrack server launch --- pype/modules/ftrack/ftrack_server/socket_thread.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index 6a5fe2c9d6..c638c9fa03 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -57,9 +57,15 @@ class SocketThread(threading.Thread): env = os.environ.copy() env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) + executable_args = [ + sys.executable + ] + if getattr(sys, "frozen", False): + executable_args.append("run") + self.subproc = subprocess.Popen( [ - sys.executable, + *executable_args, self.filepath, *self.additional_args, str(self.port) From 60b2cb46cb8d761eb9d696917596fb2956f1983d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Jan 2021 18:12:12 +0100 Subject: [PATCH 23/50] SyncServer GUI - better handling when no project is configured --- pype/modules/sync_server/sync_server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 24dd6d4bf5..415f1e7d15 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -141,17 +141,19 @@ class SyncServer(PypeModule, ITrayModule): self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) + + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow() except ValueError: log.info("No system setting for sync. Not syncing.") + self.enabled = False except KeyError: log.info(( "There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.presets)), exc_info=True) - - from .tray.app import SyncServerWindow - self.widget = SyncServerWindow() + self.enabled = False def tray_start(self): """ @@ -192,6 +194,9 @@ class SyncServer(PypeModule, ITrayModule): ) def tray_menu(self, parent_menu): + if not self.enabled: + return + from Qt import QtWidgets """Add menu or action to Tray(or parent)'s menu""" action = QtWidgets.QAction("SyncServer", parent_menu) From f82281adb9978cac89be32a2add8eb8411cfaf76 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 14 Jan 2021 18:22:15 +0100 Subject: [PATCH 24/50] fix create_env, add icon, igniter as subprocess, refactored start --- igniter/__init__.py | 8 +- igniter/bootstrap_repos.py | 143 ++++++++++++++++++-- igniter/install_thread.py | 7 +- igniter/pype.ico | Bin 0 -> 109515 bytes setup.py | 17 ++- start.py | 182 +++++++++++++++++++++----- tests/igniter/test_bootstrap_repos.py | 5 + tools/create_env.ps1 | 3 +- 8 files changed, 311 insertions(+), 54 deletions(-) create mode 100644 igniter/pype.ico diff --git a/igniter/__init__.py b/igniter/__init__.py index 8de58cd6d4..eda37c5af3 100644 --- a/igniter/__init__.py +++ b/igniter/__init__.py @@ -9,10 +9,12 @@ from .bootstrap_repos import BootstrapRepos def run(): - app = QtWidgets.QApplication(sys.argv) + """Show Igniter dialog.""" + # app = QtWidgets.QApplication(sys.argv) d = InstallDialog() - d.show() - sys.exit(app.exec_()) + d.exec_() + #d.show() + #sys.exit(app.exec_()) __all__ = [ diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index ba4a21d5d7..2a71887fbf 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -140,6 +140,22 @@ class PypeVersion: return False + def is_staging(self) -> bool: + """Test if current version is staging one.""" + return self.variant == "staging" + + def get_main_version(self) -> str: + """Return main version component. + + This returns x.x.x part of version from possibly more complex one + like x.x.x-foo-bar. + + Returns: + str: main version component + + """ + return "{}.{}.{}".format(self.major, self.minor, self.subversion) + @staticmethod def version_in_str(string: str) -> Tuple: """Find Pype version in given string. @@ -227,7 +243,7 @@ class BootstrapRepos: return v.path @staticmethod - def get_local_version() -> str: + def get_local_live_version() -> str: """Get version of local Pype.""" return __version__ @@ -273,7 +289,7 @@ class BootstrapRepos: # version and use it as a source. Otherwise repo_dir is user # entered location. if not repo_dir: - version = self.get_local_version() + version = self.get_local_live_version() repo_dir = self.live_repo_dir else: version = self.get_version(repo_dir) @@ -516,11 +532,7 @@ class BootstrapRepos: for file in dir_to_search.iterdir(): # if file, strip extension, in case of dir not. - if file.is_dir(): - name = file.name - else: - name = file.stem - + name = file.name if file.is_dir() else file.stem result = PypeVersion.version_in_str(name) if result[0]: @@ -540,7 +552,10 @@ class BootstrapRepos: self._log.error( f"cannot determine version from {file}") continue - if version_check != detected_version: + + version_main = version_check.get_main_version() + detected_main = detected_version.get_main_version() + if version_main != detected_main: self._log.error( (f"dir version ({detected_version}) and " f"its content version ({version_check}) " @@ -568,7 +583,10 @@ class BootstrapRepos: version_check = PypeVersion( version=zip_version["__version__"]) - if version_check != detected_version: + version_main = version_check.get_main_version() + detected_main = detected_version.get_main_version() + + if version_main != detected_main: self._log.error( (f"zip version ({detected_version}) " f"and its content version " @@ -769,3 +787,110 @@ class BootstrapRepos: zip_ref.extractall(destination) self._print(f"Installed as {version.path.stem}") + + def install_version(self, pype_version: PypeVersion, force: bool = False): + """Install Pype version to user data directory. + + Args: + pype_version (PypeVersion): Pype version to install. + force (bool, optional): Force overwrite existing version. + + Returns: + Path: Path to installed Pype. + + Raises: + PypeVersionExists: If not forced and this version already exist + in user data directory. + PypeVersionInvalid: If version to install is invalid. + PypeVersionIOError: If copying or zipping fail. + + """ + + # test if version is located (in user data dir) + is_inside = False + try: + is_inside = pype_version.path.resolve().relative_to( + self.data_dir) + except ValueError: + # if relative path cannot be calculated, Pype version is not + # inside user data dir + pass + + if is_inside: + raise PypeVersionExists("Pype already inside user data dir") + + # determine destination directory name + # for zip file strip suffix + destination = self.data_dir / pype_version.path.stem + + # test if destination file already exist, if so lets delete it. + # we consider path on location as authoritative place. + if destination.exists() and force: + try: + destination.unlink() + except OSError: + self._log.error( + f"cannot remove already existing {destination}", + exc_info=True) + return None + else: + raise PypeVersionExists(f"{destination} already exist.") + + # create destination parent directories even if they don't exist. + if not destination.exists(): + destination.mkdir(parents=True) + + # version is directory + if pype_version.path.is_dir(): + # create zip inside temporary directory. + self._log.info("Creating zip from directory ...") + with tempfile.TemporaryDirectory() as temp_dir: + temp_zip = \ + Path(temp_dir) / f"pype-v{pype_version}.zip" + self._log.info(f"creating zip: {temp_zip}") + + self._create_pype_zip(temp_zip, pype_version.path) + if not os.path.exists(temp_zip): + self._log.error("make archive failed.") + raise PypeVersionIOError("Zip creation failed.") + + # set zip as version source + pype_version.path = temp_zip + + elif pype_version.path.is_file(): + # check if file is zip (by extension) + if pype_version.path.suffix.lower() != ".zip": + raise PypeVersionInvalid("Invalid file format") + + try: + # copy file to destination + self._log.info("Copying zip to destination ...") + copyfile(pype_version.path.as_posix(), destination.as_posix()) + except OSError as e: + self._log.error( + "cannot copy version to user data directory", + exc_info=True) + raise PypeVersionIOError( + "can't copy version to destination") from e + + # extract zip there + self._log.info("extracting zip to destination ...") + with ZipFile(pype_version.path, "r") as zip: + zip.extractall(destination) + + return destination + + +class PypeVersionExists(Exception): + """Exception for handling existing Pype version.""" + pass + + +class PypeVersionInvalid(Exception): + """Exception for handling invalid Pype version.""" + pass + + +class PypeVersionIOError(Exception): + """Exception for handling IO errors in Pype version.""" + pass diff --git a/igniter/install_thread.py b/igniter/install_thread.py index 37232bb88e..ad24913ed7 100644 --- a/igniter/install_thread.py +++ b/igniter/install_thread.py @@ -48,7 +48,7 @@ class InstallThread(QThread): # find local version of Pype bs = BootstrapRepos( progress_callback=self.set_progress, message=self.message) - local_version = bs.get_local_version() + local_version = bs.get_local_live_version() # if user did entered nothing, we install Pype from local version. # zip content of `repos`, copy it to user data dir and append @@ -93,8 +93,6 @@ class InstallThread(QThread): f"currently running {local_version}" ), False) self.message.emit("Skipping Pype install ...", False) - if detected[-1].path.suffix.lower() == ".zip": - bs.extract_pype(detected[-1]) return self.message.emit(( @@ -150,6 +148,9 @@ class InstallThread(QThread): bs.registry.set_secure_item("pypeMongo", self._mongo) os.environ["PYPE_MONGO"] = self._mongo + if os.getenv("PYPE_PATH") == self._path: + ... + self.message.emit(f"processing {self._path}", True) repo_file = bs.process_entered_location(self._path) diff --git a/igniter/pype.ico b/igniter/pype.ico new file mode 100644 index 0000000000000000000000000000000000000000..746fc36ba29274127740c24feb3fb834424bfdbc GIT binary patch literal 109515 zcmeF)2V774{|E4IlOkzY8HFTVL}gRP6-h?2vx*|~+M&L72yyLIWF%WMFJ)$2%HAq7 zvk;~A{k>lKp6<6Vu2L$$o5%m@ob&yjb3Wt!S?7HH?@uIB5UGla7Z+h`AaYR_i3W*8 zA_IfZ$Nu>I5i7YKY9~-w2i7GS}iNs>*abX2f>nXSa{_uVdMk3Mh znF=Cf9OG8_jj`5ez)dr*hML`RSOBKVGpQoM5EvP~fkpEl_pUr^3 z;1C>vHNg7`Y4F;i-~-iUKIivLI|w8)ZeqVQ2-jO;|AqAXP1rI`;X7U*DE&^j$2A<6 zk~zl|`*$D#7D6HT-u2~hj*65Lwr0>7v_ZI*+pnb`gU%5UO9e$JeeG z$|1XbjcX@ETHbjR-U~8&J@54f+DWhbx9gdIe+Y$Skj)3nl$rWt{EXm%-1UcX%=Yzf z>EFfo&7mK>0@?I;;h634->y%RUiU%nT7Eu4=33q(EAa`+%=N6x;jj+aZ)B~)u?Mg&ib49dGS~An-a{SOFXXWeXSgmcar`BI1SjFRloGau zfLl_KA$-Q~HASKj3sKrQV}Scu0>_`UKNxuA#~F!4A}a7t9UvaIz}PPThL!kS3~6!u z&ej#*FNT4_dEcvRoHrL_+vwN!9|`ySmu+`^7cIRP&rcg$GLgQg^qQ}wD|{A#bN;RT z9FK)_Ww+LuNT1`e>@{E8zx|!`SAXLk8*?drw{N8Hl}qV!oQaal1Lxgxxs*QJ2JrmPDC%f;KL+Q(pKP8X{KEKGO^|gKP97_Ld^WT+Bq`x|c(!Y+h()%>_mEm0`(qEWE z=|>{1I>2>C7YK*1)wld-!*VEn`TJ((_&xGiH4oG0vrAs~|FmcRJSzWCmOYQkKSW47 zlj$$Od6{WX-#h1T+^3;z+L=s$JkI0#hGT4I@%iF1><15Ezmy#x()TkFe_qSyrL5?W z^E&{qRe|*MsPRZ6vpRZ7;6RZ7szh@atmQ6UhK2UvikZ_-#5KGh)9(!%;RE;&jF6B z`Y;#*;T(j3H`Ie7kXFVo@9^a((sN5<`v>q|k}UXybB_YwVSjmZ=_mY72HPj&ybs<& z3h+MHfHAEPd}pBm-@K>r1Ux+m;47j2bh1pa}R~A)c*&Ze*iL@e-oU`cRJiT6nHKCX#LBNf!{lhM_Q4vAjq9ODy#{pLBG zg^5rLeuezY$A|rg?`Qa|!7(6HYc|=&hHG-O{mAEEIKG5@usv0f<|os!$S3*dIB*U) zW~Hqgn6J$I;dzi-?O&+xuWi{Myue(Vr_5wOEl&9){~Vjb!5K0=9^u^lQvZwz%di}3 zLRuNV7X+M_o&2-iK85|@51hY_zysh|`L%ZPz0dee-vKn?d)d!!`{DQ(3Ve6LXA}0n zUhpSe0e=6z{K|iS64$tZF8u8L3+p$|KPtenLkskwCw%QWMLxgs#}{zj7$^llJO3Hh z&rDB-G4_waCy>wMza6s=b3V(B=SMy}as8a(`5nhB2R~z5W*Pr2zw+N*#`zvl26ChR z_adFN{4+fn)v$jWvf7U?;+ougerNq31XCyioChlc*OgP@A;{P7_l{5Dnjye@{p|YZ zc+L763LNv;XSu%#A|b17pJij4~dZ}-^$ zZr$gj{Ik!n?lX+TnJECv33*lj$w-^+W&^Z^Vvw0^r=P>~A+PfP5^1jlD`1~hg!H^+ zW{2lPN%#ZKK~A>muicAv9t&q-9<+e8wI;Ti$#RBscnvWBqk;G2{c}PR?!)=(BXAAG zcJc(S1Lr8tb4HMV0Za9xo0emp})e((VIV*U#O-+?y= zJLn?aa{otk#&x#P3hD#LMZW7~T4{bTyI));y&q%9G8Bd4IfV7hF@^b7g>1ExFEU>6 zL7vpW=YKGSe?XM>M2dnj#*fKsSRa;`64~wJ59dd{Gs~`RUllCEz4d5_{_p}<7nXeePkw$ zlAb>TTV8JkY41z16_U&Jme)wJ?2Vy2I0Ns^wG`Jttb?2?KcBZ+1J|<~-~a@{zAWJR zdw}n%#zGayQvQVVvE_QS2$;ZdSOEdRb?~3y0>6P8C_C z9kUEyd(OH5V-DO-Tmt2 z*F_mx!7TV*|L2&I>Vfjypm=#7}>%Znu4A?&T97{SfgwMF< z<9U47&hlot9}D%C)h*Hz+F4G;pX=XOa0r~BAvA`j&`in#TjFsOXaIGgI+O!7$W~l4 zTMt4y*ydFEg=dp1xaVqc2liR^U799C=+Bez**F*uy`d%40k(J62ggEXP=Fk*7vX-` z=2ZDpaWBrzZy^$HO5MWtYxw;QxCX4Vv#=k0fOWI zzVHXM1NIM&AF7Ze^&s2_+nkDjcFHbX%XMu6yaCn&*E*c@$3t__h5zUB3-uto<@)a+ zOom3F4cYDwxJOz&Vw<1hFI#S5pYsZzb&mtrye%Nx?Ev@qRmv~a!wck@bL#})GXm!Y zp$wUw_R`Ojxi-JYUnoE4me&vnT>qDVAMOXZR;asQrTjuYaL!~u;QC5SC`Wo=Gr1#N zhwZOc{%GWRAB+S(8>GcClf_OyO}Gx*U#>}n4!E}Fvrv}Thq&ffZU3prJJ$#& zp&wL)EI&8knw%~_^TjbL78oC{UD#J|!F}MnVa87A$JyNq;{ewLSsn*)O-`4e>(}=X z4$Hv}xMrCF-mn`Yfb%))D!b)po&E_OpfseFC%uDzXGb_6+X~PhHUZb1KbkLLopJ#8 zV7oMj`d|s|f$wg7-~w=c`=jXy*FD5NHb5UJ4|!dF_Q@x(8CYh%(32Mb@%w)&ztI2Jqg?6j zUl@k*Sz3IsZ2)YyagYK(8h_Rw=al@?{#kyu|Bb+Lj?ZU&U&G&L_#U`4@V%ZB@LgAS z%Fa4_4F16T=|g^K|Ll`Hf%9ECV7{sX+k+j<0FHf(-H*myxGoa+SOwjnEaY|hg|%NO z@-iK`rf>vT7!S*Vap!YAuVBRjmCE#3t z4I&^0SoZ9;eWrCDT)+s}$MQPy``&#G;9AmQ-ccEg0>#+UR${&V2v)@+%b@;E8pZ%V5$2PD8 zt}U|M?tkI<%dYnfWy{X}tp1U?KEJnr&X>_}1o*7e066CVP`elENw^-{{9b;pskx>c z1$?gJS~|<~oltIJi+c!tM<4rKr)`19AdItlwdLA~F@FR1U=Iw3`jG9of2I5exG%?3 zjw^hpl2_#x(iPhK3FKuqbb;zn9I_qvZC3X z4171(9hw5y^*T@p(${!G{X3i~!N752h;I0vCW!D-wx1P%cE+gji{b{KSk zI#32!<{XW;e0>S`$Cmwy&*xoXJa_{8-K-oo6Ze||?!Y(?hF)L=H9-$JwzKRx9`AoC zzZ%lxI=VKnzghnzZE@c=&=T0bIhS!KBK(iOBovZl*hVURJ{GoOkw@D1Ke~;S zUN3AjricHIf`sY)vlV%KK2G22OOJ(ZT7GeXuvLiUa<%;f9da*)k6*Ak-D}$M<1;FUCA`%Y5@Q)+6)I_hh^# z*Yhu5UgSZ(4f1yxmWBD{yFUJ=$oKbruUi+)p`H}?$xONxj&pL$dz%8^r!x=C3*YfF ze{2I8@-JV$oIaLM566t72JjuLJ}`eZfbXE0XA`Ie)xZ$SLkVENO8*TE`?=D7I{^gFD7W*puJ!9VzxE7uUoUb@0vb{coSMUay_m}VpZUEcI zI+y_V&x%)G1ZKw@)z&x%1);ZfUW049H89dHO^7l^0 z{SzS$p1>*K{Ms2RK!*CtV)4H?9rw#n-!jKchkc*pHOI$R&>!5v2ZG@aFy8Ty1kAf^ z{-5LcI&dCc3S*!DTC=T{#=`Mfr+pRIBwm7SYWJW zWAwB4Q;;6(kafy_#XjZ@Y@hm&p^kF6K3F%LlZmm{gi2ruPB05L0%Om<&G>R`WPVvs z0+xYom}45-a5x-*H6Vumz_Enufs(+vfa$Rv$i^qT#rxuVx%$PqyRRa0a-h z^@IUn0UQTPK~YeFyof)?f7b7KSOJINCger`XZ%^GtlLL$0_H;}r~p5^|1$=R1^XrY zXKNS@n}Ge8b9+26k8G#x>$35eNRRoMP?iMjb3WsEaT&IO2XupmP#JWfFyux2Yaw06 ze;h1_gTVNIZT@HZekRr#`^`Iuf)Mb6j*uJiXABq%#-tJSf;qrB@d+^AKU#iX8x1F6 z84QJXU;<^K2;^1#*}ujD^K}4jKwiY3eTw~s<7OlT!#v2X__Ob`otXpc(zd>E(W&C>~{b}F_$Kf8lh3v$i zZTwsD=W{?o{69ZavH#~){I!v`F#fw?pKHII9{;(nISu|W8~Q>$C>a0$i9hF_yy*X2 z|J48+7!C_zFI)w-$?WuhUduJa9XJYWVKQ_BwmBUr82|Hj{x60YR0a!h0C(UTfWPPQ z`7aqbj{gY0Gva&0aM%U&VHofo0NWhfUP1ixF8*8#szVvzGZEkY4T43${`v?Q!>q(! z7%#Z}01tud$u%$zxAjf!~1hfE(YW%uKmq^N@H?Kz^BMd5z`oCWxk3vNVqXONDo_yryzT$8_0Kj?5jY08 z!!}?WxC>9BUKId_*7Yz|`1&)CqOoq--3)sdu=P>qcce3ecyT8ldK<@s( zQvc`pR3374ew3?^40Vt1I2M!uz7OgNYy+!+W2YzBLT-=$LL1oCJunbY+ne?O#` zAznD52zN=b2_>jjSYn!s}z`!DkTv*Rs)f8;^7?(%c|+5h=$ z#qpo>YZowrqM!_1mvIfny3HzHCs4ya=XGLS87JPKvC1&k;8?cqgngd#J;J%)-O9%+ zr;p{zkRRipggkQo=lbtHgur6x2WG&rlzoeROb^O_1<#QudmgVV16<3=_Eq+04PcvK z`^xkF`X7ydJnB9c9zrOr1wL=J0j|BP!*5_BJ?3Y!I48>!zUMW34yy?@z!<7PCE%RS zHlYO^A2<$V=(9N;kKDeWTzT?i|4&99KLOvLK7}yY2Gf9JfDN<;D`+Rh&t!3}D39Os zd~4vko%d)3&4FvShF}V8TYNXextntW>quw=+?PETK2P7~c09A1o=jPoXO@L+oqeGL zjD{t!AFe|Tq=F!kPukjbgfVarw zW8fHY0Q_J%u#GO2?(N8v9x z2zwy_e1YwR?ZO@Y04LxW!M0EnxTe$u*)|~CzkYUi&hq_{SK0iry=no=!gih?#(&N= zYy%{W10S%@cJLCOL$nn4vr4#@+h;Pk)_4Mua2Kw^Ibhq^4Qvk!U?TJbu0J>y=z|Uv z0TuXS48UjUb)Myb9NOmIULQb-;Q~Zkzf125RbgrvJJ@3KO7rw!v#13TY+=KRN%U@ z2mB7EUfy^iyuZ)Y}5A^yTR@CkAJ2=9UOLJaWP^f6okt{Yc^8#n;l0LKB& z1?*!P+JNkF4tMYG-cB}cY2LK3&2Q_!%*H9N&tv;wTet~_fop?F&6DWZq zWM~WD4OkAJAeVl@_@Di;iO(i{Zr~d65UhhK&=c4Osz6C73VFUBKpteoQImXI&j1IR}~ z{Bt(`vTcFuM!u)I1xH{FOoHxU4tyryT0k}i8TNB{y8q&Ka%mTg|2f|lxE|p90=_@w zns7Dn-EkYJ416!ZwIJ)B3gVyhF_7((xxJr=n8pF$9SY9{d~fUkb%F1O)PZY3wu6HB z|9{0_cz2M5*m6B^4Yt5&Xaz=~3ql(xi2wgr{AI@*t_}HY$hE*6=mm{{^MDQ%hJyJ2 z|HhwV!3#JGJ}@5mj<_Kh0M`N;)&O5D*mC#-q$k_n3f6x=x9u_hoC9va0pM>&W1u7O zH$%RAEr|cmje+cVoEQ6C3qFF2uoo7AGc*Ri7t9cU9Lx649Pa(14EL6%UGV<@=eEBD z)bneIfOD`FCPO=@2x=f7m-OSDZUAy=6^#EsH})KF_#E&N*aq&wG4KXH57vPaz;z&> z1%LYQLr5dNZDGG4{y#tddZ={tU<1ag3zC|8|XQzGG_kktA^%I|?xW58c!+O{X`+;rY0mK5^QEv3@%%s7# z$Z_gCY=B9?e<#ueDuD(tCh2j?=>{NsZ)sZj?fH-K=Wi(d9gTH$6Ar>Mm;f%|2u?5< zh5>(Hn+P+2?^t#K$BIbc_#w;%ymw~e^s~-k{5dX#K>*AImZKH$cL7cKug71Q^Erp} zcT|q){@?+rjdY}!uPzK6Ec`yLBi`u}p&=wqk zbH+MgJBkL5FF$L1|K5EWf3|DRX@_71j0ZblzFDsSdi)bn2OogXa(Cb`%mLO>72q0K z2~E|J%sJU$6!yL06~)%zHun|5^L7 zo(=-nN$r9CToKaO$8!6uJAO}WAM8)8*MYDeIPSg&A-;@UTmoj-(kz&4N!LTqwl z%lI?x`@l8nCYS+SrgM#?~^WHa8@s}STQHQeQ0ow}O495<7LpVP9dMbE9sAH00<1qe$Nix`Htj10n8=vnH?%x=;oTpc?Rbstq{7 z0yqLuzst|2J?2cEfxa3f52^IA#^Z|8pOGi@M;u znow8+tfyAM7*~O+()~KvvafTTZw;JFM*_!;EpQC3!c&NY6p)QcZtgSwuYl{2tuO-| zp(Pjr$Fl!={F6}!pWr<_gK*dm(}8{37ObGHbiXsUoEsd#8QA9+!wv|AI}i=;AQ5si z-tuX&?LCLH;0qI=8*t332paHTk3Z{(V{|-lth^6=SIYGp$4d`zgUKKUwu@OX4>;DZ zhkbA!crC|#_ItJ|wk`RXsgoaQVbf7S#$DaE+{YUQp`7!>iGocM|97q6;-JCa{!DDzN#eI4L z>{lG;*yQV)b-zDMf%aes9QTE|b3QJJ|NmF~;}BD>S1!U{;IrvqXa;(~cbOSt zE_?hRUH|38xSJpD!8J=P+=9cf3b^j;0v!9b;fp?x&(hmNLHzUdc`dtbi1Fw1$OYI4 z9QQfzH-$=|0U6?-)A5$SAJR*2Thg@i+w*^R>pVYQ&-nAXlxsdO=nEY8ji3amfoweF z!z7vhA%P{yD$aWX##` z`P|2Ke;BL)W2Bj$7WNC`pY!qOyXLpRasLYJh3U`{ z*!RD9$B)m_W0KSHmcO5DTG%g$f6m8Wc;5RP^~-VJ8(e_%ey%_76~sSRpC35x^Eb<@ zz<$rzj{v^wsRVobUO5pTARazP}B8)?W&vpfl75 zec-!3)_FnvvtRGoS(oSJwVdx`f$#V(!y#A!qd|y$W$d%>7lsV6&*}dCz5B_f|E>NX zg!8UL3~)@#$@=<#yVrZvPZXSo-LM3N*qdSB0JMSYzFdFalWz~mgKYczR{Znw-+%l{ z?So@K$NVe6*e?Li_Z@+;F9#YRJnwxm_RGiTXCEUkUyFY&e9k=TdK?>W}x(rW zK1;82`FP~^v20$Xd6k|2$KkVOuph1h=Q`Gx5O?nXN@U|4hrGW7j%jSyvN4Xr@jbW! z;cyO40O!G7!2Zs9nFXVvCp3rZpa+aSpY7S_(__v3{PssKe{8E-P!qUz83l`AH(Ua? zfA$Tbudohz{Hu_SH`{VFJO;-5K5#t042*XOuzzm{_VGV~@tz1np$~L~7GMe-?@K{3 zVC(zjBy%2lWr)>SoSAU)#%jJ*xW&1S( zw(EW{8CJkvI1AU|Hn6X7pY`{vxPrWLJpUV5mJ@Ik*v~n}`NKL`3^QRI2yt(X{d!;w zWr6*q5PT71e3l{Rvd4KIaJdY~=1p3bqSzWhL$Cu^m=3F82OI=G4;+^6^Ekg_d?^S5 zf&DuGHUi_#_RV>24!8s7Jx8zyD`0>B4OlOw;Fp=-<-+~NA<81(R@Lzt+jnhf1>Jyc zfc3HfmIK?uO6fk2^D~a&v=o*A<2)Zcf$e$+fpcL+5XO1-kHR2~ z@fqsxiwe%?AD|o=@-Ho032Y649$0Vr9rH_L;13uHLxHhonHcYG zz?ioI3*dT=b0GVH4scFnzhK-|KnWD#i?KhShx?1{p$ziz=lX@?8_Ql5SQo4lYp{dv z&=Yz{_j#P(F}_{F9$1DB&<;4Zv!6GG`cMaqp#m`Oj5*^i#QKYPI*JP!%ztif`0=pq1{+*nuV||<{7V+=T=VK2$ykSDx^_e)viOfd&`LVdrLgqLm z2ab`A^!tl(zuY`#{$%Vg>-yhRP5SjM&y<=uiHor_R3&;9A zUFLj?wByh76XW@vQdjwO@sfeeIe#? z(}+ERP8aqMb%V7uBo$a_HlD@tjES(mEjy%9En&+-x`Lo=rl;gQ$3~hUOymBRkhWoA5ZI?Cj8R z+rHW_5=I-nnw(r|m&epmW6G@-J+$>ujdVD&H`J|53loKMB^{@QJ@OX640C(z);iwd z-P>?U{bFL=rbNQ~aYx%NIJ`u8LWPLml1^W9@Mv?(bZWQb9ti5?ty+1vd_sCD5*B2h$xq!ZWHTG%}?o$7IBf23WsPq@~E-e(qdEUVC4x5A&r z{neZXiAStSS^xN6**kATXEalnRNi0ySew*tnvVSkHC7vcvbT}=j>J%0M5KSmadNq~ zmO&!la-HkCYqUJ#(e@}Z(7f`im9bWV3tp<-efs;yKHe!qr`^&R-B@9%sM(CM)fTo@ z?o`BJd!sj{9Q8uHhxxotHk@iTMme@Zh|;Qnke_QG6CXuE+`H-4;v6-CF#@=XK)B^Boi;-9)i=3rf9yRQ1ylFPFV8N)bM5 zPc$A-OELVuq}xG!>X&Y$u_Is~_`_JCfTf9}zQrF#Fq08C84}P&t%7-jG+j90t zg;$<>Ys7VmmcA%jED92}R%#=%t<E9_JwQM_;r=40(Tco_< zQSDU<7AGGqeq=I4DL|{q?D&%P=i4V1fBLtwSBa^5B_=E!*243GUdUUMu8K`I85qAW z)Uo6B+TKqJx9YF&)~xXv?+s72w>MhYua<_$I9|ngY2na;iVZ4QUkcfCShC*hxWCpZ zzr`)~cZsz(?I@YL(-o+fOm7z&X*l%6BW<;zi~X8Cjd(w+=lxO> zS}o~0?x<}8+rgEWl}J3*rsdOCT@)iPdCadh+5d!(`MTfVHS0G0&7#qY^qOmi&NW6%o(fDqeS&mfkk>a7f;eM zXt+*!h=I6{lX9IWL1&V0#mBEV^i?dk_Vn9zQ85qve0qO8>JaZ@yfi1iV3R_Q4PDLj1qCiIY*xU>7jYRiVUFcH7L-?Z8C zpw*8qb%>J8b)VQUMRjG{_3ktN^fJ3RSkb9Zq)n_wWhIpq6^G8<@1@*!UVOC5F~66! zkGnJ<+E72GaGlj}H<}#K>}PGXa!!b6WUcK{eVcXIqHpifx@d*)rAtkl8}DBk=GK15 zwWy+1XB|=#l|CS;nLMk-%)ZT=FDR=zGePa{lq&U0cYWS&<=oTl2M?a8h(U61?dp@- zKaW4-wx>+O%E!g~zW1`dzfjFWcX*X$iwCKXQL-ti9~975DLi!EVb3$$Z+Ru#{MFL$ zW3caK_j==^{?hqXg1)!(L9xSx8-(aq<-I9^zx{EYh1Ga8h&+SO5C$+VDp zhkm10-|SpC#N^}D(iIQ3h*;@=)j#NcoKf=8#mbF!|14ZmMMu1+p0R=`rA&!e^_FhZ zeIL~0boJfgO{Q)tE*_mce~P7jLXn|io6irahtvb}d4ZH1r5e}>V_fjG+=!H9om0c)t|BPu)6}zy`?eg zjX^IjSO0}aQpd-cDlb(I@KjDq(K@hDwd7ajL)J>lzEjxPb3l_CX({fT7psz3^JE)E zi}1n{`y*YH`q@{UqVo2h)%Buj-{XV1yWCQ4SG9bpw4uz%1=cSV@p+x{Llna`EsYiK z`Pqni*6gk~bG%t+RPLGK1`4rz_7~ow6l;B>%Ko+nvx+*eo--J=bHAgWL4fkeImaSL zJM3$xCW=l{e5GX4+3BkDBc~?`hqdeJcpNA*Vfgt&wk3wByG#u;)ob1@>gN5v7e!S> z{U*H$>C{irztl;LB zXTZGGeU0locm9Rn+?jV#qr{D*3QF)+B>p?A~1buChsLt1c-M16@`-UF`xHzx6 zQb#AX-I)H{U5BnN8FO;d{Pt!y+bGwuGT5(ar#Qi~e#a#}@BX%QnvwW$?5f|ci}wAs zr0H{sDE82lJ(eTuC3UpY|CkZ?RD!a zT${I8)wlB2W1@p9u@N69%(g8OK5<-iEWBRMKW|iO&zYn9bT>tZtSYW%kSK~SlrZO5 z-+{)5|1eo0deO@3y>n0TDun|sZ6Zw%KNz#8;lQ|uR*K;c)f)vooHU_z#PpO3k1C!p zH@GSq^Kkd9!nZ411~2zJsoYV2L#%U8QDxEgMt%)r52bvnny4JI`nScFh9cWj*Bb<> zH#IMQXGgbp?jN4^tudh0?|ur`b`R*}9U!TvP*gNf!(f%7p@OY{%G?E_TRsbvy$6df zcOB@J*eSJ;d9f;MeWKkWT}$;|H=^bzQ57exsY8EPY|seN9B{U1$M%*gf381XN71;U z!o+6k%|;h}^W>iXp8Lvnu8;eMYwGOOsB@sw1&wg!ZJh^PXEQ6?QcEEqYSe6_GG1Di zB~>REezqafqve+VBSor{B3+l?D-w5dq3iF}PU+p;{=%Z4Vt4IXW>ful+RfZz(9AJu z(ACKz?N+*}n$dT9-d}$^<@zb#*ln{XE!(in+Pg~6t*YK?9)<(0P@e%%|XxSCTFr@Bsd9${LS4S&e^n;f56BEm|pBo$FVm3|x)|>gkT@{T>^cZO9J$QiCOf}zT zUbn+G9TSB`m0ln4se4OroB3O=#+28Ka@Nu=KW&jH+ThxPhOW(9D{eA*wSA7Ak@!jF ztLn4*6#1)t>6O!x74G}jc)HI@(Ll{1%*)LnU zqG3g!@O!=Ndd^*2scux5&9vKBH@6pka4glL!HtV%K`lGonEd#fR`9~@@hx=SlSEz> z>RLGrE??HJ#iHaMBNDGTH@08WVn@*Ag`yh{2Rk($ceLSxi8>ksjjC5XHEWaWY1Ofd z5`0V)ep9-Bzk%_1<4fmf-ZV;HImF_~VV~nhA%pw9yJ%L=BUNL<+TG$0i$lUk2R63a zp-}orznkTE%yqbAd{TLL(ItBo9>r+RKj^r0mDeDZTCSC+-&6~`__q8<4ntK+xcP@a zE$!K2$NgD*TP&~s@V?cV;{C(wDH~455U|8Z>H0=Br%gLsYSc~`?|MpC|J-l8b#^yf z(k1jz`JtQ6*DH~_epV^hmRr4!R4=qu&3Cv{<7Dsh6Wc5qmsBm~&5aoqUk~#-)wFDQ zg3joMUVC30ZP4KHhTU~_430fBcTrl|PqY0W`&IkBnsMuF<(=A>T3;%&c5D-iNvRHg zWlN2lIec8?irDwP?He3hP`E_0S794bYc0nQqLvQMD_14W9k$r`^vl4>W7n2UR%rin zS46m(>XT@b#UZ*u9vVk1Ozxf8QQZQAShFPo=d9OG4vf-yC0X8HwR+0J?tvAYRrkh6 z#E4SWBWp){Mrz;Rxcc_ydtGK#j0!v3uOvo=Qytq5HcD>v#|tfouw#o=cNf_jy)Sa1 ziP!Fav5 zjd5J?di_MZO zk2R@1%)?%?D|}$;z|pU5M34Slqu=RSTu|YLI*G{rF z^j#!cRXX5fsGf)4!UhdyHcn~Ty_3>z&5#709_2;zT*qt)Kfd;6^ulLu8XkRLwBL11 zp?$4HgBt3=<5TL^Rg8=_EAO{V!!Ag*U$R<>y@4lU4!QR2pEOG`add+9#U`f13~rR# zeKxeZ#Kbsce!@I9N3~MYFPr-^zzW3h7?5UaD z+ptIy7Ie)$s~qc+VxJJBFnL6`)`~+E|L&!$i0upQs*K>t^&+Lm2XOrxqQoLXz4uF z^-bPLJZ(Qn=C+QtI1pMpT1PZK^+jvHm0W;gU3VgEyjkN^XVHPziT9etYphi@nUu7v zXxbvuCUv5vV}!>^GsUz5TX~6+Ev-am3NEVm){ju^u4Lk+BQh5?Qi!=cUbJkDIIwDp z6LV?i|9;^|i}s=+3cWWKwzNNj+0n$y)22<|@Y)|Qn?;N`w;|T_t@Dk!!4(^5hL|*R z38)=@;M3MOc4q@O`mFd=>XF1=?eir={R`Qr+FY;wI?;YiO8Mz&&kjvf%pOQetleU9 zC#lEQ5kUbK{c-bRcOFl#XYqQ9TcU+u2gOnG3!6$NUec&;nD!W;^(cAOi`AzSgV$Fq zeE-w7(#Z`~7AWEYz+d51E1zCH{*HIv@AEO?!i->R^ybp7r+1zoTr$z*{&GFbk}>*o zKXtQGtf$}=6lMOY$IB%1V;!t~v_-=_-lX=7vDo_ffYAz(mwCl;an~QX-?`UvSnA!~ zdwWaH-F6S%8Ly8FcQ5PmX!Kxb{T*+PsKm}L_o+d<_#OLiIiq?Vhk2?sdv)C^>e(BQ zd99-*^A?B(M8`C4^sMjFL#d9)h--_dPqqYhxa&5hb=5dSaWAt|zDH9BM2G2#`*&Gz z>uT+f@BL23XvWz~ZauC%D+OJk^%Ktul3Q2WEmxP+Ogs`Ckh1EvGwMdEzjg1m79+y- zN9}CiWI)Z}S)YdQxqqidy@Z3IZqap#a=V3_N=Rzed-r!@!l)_DAKad8 zu3&LlP4W=+yG(2oJ=TBovY0_(rY&pFN_==p|E^oinrORTYdlz#YMzx3na{CuOzaS~ zXLI}@W$)Jsz3;s(_3{a;z+Y>ZQdp?NTE$Br46NqAcRjoCw)^;~6&fFUk4d}Qubxwh zG501uvh`6rte)I-f9SZ-itBEdv)K9jz7L5dP`!HV>q(02?=?o_b?l!0&VvW+dh>3R z+OfEg%bTRmn~nRcT|T$`$?I*gb}rRIbcYPPvvr6s16U@@1tg7nR#m}izx%7JK3bxy6UN?Lcx1}H4;`!{E8B?14rmkW*K~Vm zwSX5D=R}S@@xJ9M8{O2!%RT14_Sq`w8NBOK)1%|2-yAV`RB9dEzoqW7(vk%)c2{V3 z(YSupgzfRWEHukkKJ;4qp%aI*RU4>x<) z`u*D<-?`CyWzDyyt)_;JOa1%qt^oIQy?h!Ltz6BeiO2OL$8brhGvm&;5k-WT*yj?c zZmwjlZ1VfrEq`r(yt|xs>UPhns~TLsz5L?c3+f*GjE!%oJDup8_}ezj_QOxDt$Smi zYoWV$KaMg}GEQ)wuv)ESVoI$ihc`J`l>9hy`K1{iXD|1twC=Z7ZhPyb9Kg(1+xqo3 z)0&n^HzWMCN>&k@He43F?U>l6@YG>ZVT-4nF%y>#^jW-g`RTXaf*)D&v838_tF=x& zoXf6KD5FrWr&8o-yH}D@OWOMC#_O*>tP~mdFmCoNEN+~J4D(v_G9jSBxmn7whR>S( zX+GuledBGdxGL_%e*2ZGyJOmfc-QXtcfEzr|F&B* z{!E4bcuH8QV>0XfUb9=;3a{!G{kTKfH*9ILVi~QXD^<6hyJ%4l4N zr<5M;|9ja(U5pyfR`s4Tt6FgFDT7j8*%kXOpItNbTOOOJ8#J?8d(nkf z)mojkKhbVj{m7jz;}hx(4R>i06MpYejMMLR&u%WDU$>(9gUNHeDsYqthCj!ktvU1n8MVpF~2!Kx##SEzk?kyyL= zHO0MeTJ0(OwrQhxzJp_D94phQDLUDPeJ`Ci9lfEN=;NaoqO!_Cq8Ac6u9{kJ-x1nQ z$|g%jjCyda&(*-?kxs9Vv(>b3QctHs?UFicCzpwi^^5GQa8MyuEo8*&x0|Q*F`w$M zBAOVrR9!MDZlaIG=Ow_M={rj~GA9TOuqTcCsuw~_Y^!{eh-h!pK>~t=jI6p3RVYITO zwy}=rqPUKt@BOggsFx)SH^fJ4?->vp_1mV2T|K6%^=sJOZ@2SJ@sXzGr|2fP8*`~y z6;TC?ps;OfzRKR8L`y`qF5fJdFg~$QGhd5I;?(Ke_i9DgHUE?ljVBr%S0hDj(fCQ~ zDf72?h`yVun_710Y-LB|;?d3>&ZHJe?p3RbPuaLH^qd~Zd$&LowKY{rNQPN%iCD0NOTTsIo_btkvWA(jkfqUB-Fb`84#h`}Ecc)q1U};Tv_l z)TfkoQ6Bo`FnQH;8n^q*r{`n*UZgZ`q@B1in6JNxXuB7JT7F#0m*wlF+ zlXu-(_sHMS`*rFKk(d2+#fc@?%{$>S|5!>&*~26C)x=SUjbo1#_IP@5-qNWHqo3)n z2@1X$RddaK|M@-RJi^2JRrTHDc1pFvz9hd$(_l&K1()x7EErZ+QuC9pwX$yK*Y$lR z&q}0Bt5`*}_NMpRyLeD{^AFfL@aU?$yH0c%+xO(?!%3m7R`Z^N6ZVQ`KS8y4!_of2w_GD!S#_uexVj+}{)PdVne zX+uf%6wCTAdQU3ycjALmH&3`c>E2R(dO^j@iOO)(+$!<5jy4|~{% zbVB~vZ4|Yzzhw!-o!kAL-z@9>Hl>EvDf1(-N(iW}f1~y_Enmjii~XD3{G)aawI!Yg zt2>#@d*1oN?{?e!Z!taZdBShAcz;~iyAy5idPJL^8hZTTpc3xBwu&Z2`}OkNU^3V~ z;$DEqy5>SJE<<>PXANHAZnr{=rA_GN#i*?+%k zY!py;l&gWbiu;_={c8G!47l*B!Ga@ky=$I5Gsk&H$iruId{T_@qlWlh$m}3Tm%?YO z_4V0qX=MBK`lUuaf>RDfc>EpjapGf={t(S^x_`G79kD&o@JZPR(Z!QuPY?d9_nkqF zZJ(}8N_lU3s?NhZ6`CB2@9{v=duzz*Il;XG!(-|i#C0h?(`w7R-G^p9y%nmsy5~Mo z&5xH4O*_1=o`I{@INfWVi$u;jFwkq|?^{|cFA^CiuM%ziy`E(!V>s zn%Z`2rH)HBN36Tj{%DO`e{E>xxg`L<1eCmUuVJvCkENgew3hShJ)00SYMG^OY8yqv z-97$ZaY(J7O+tsasu3kRDK!|l#kbyr$8RPseBH|6l1jg|M*}Vgd5gy>8!1e7?bcef z?Yc-jxqh!BWxT7*>++}NAC~?~u~qj6MmH!WZu)7ff>X+eXLyl!T~tY7cl%pI-Ng;n z`n5TFvCo07Hf_ZXH@R*!vaF}R=7oj*;2nWmOqQ;4O}2In-~Q*Whbk#=`!_O~@v`i@ z_@=9M0!wO$Vi#D<(72(c=9v<|RC!+~7v1W$7CdrmUf66xj zu$!@g>J~xEdz@S`S|_BVV))kk_w3de3kfRhu6)dbGlW za}lAFlPeY-Xn5w#x)-mK)Fbyh^u057_JX3_1#lwE2GY7Wb3RHw`OK?W}UU znnUI5qBytR7UDZs$49G;c%nno)A;m?D@Yc*e0I?lFFhp@(1hmQy^wR1wstQob+YVADNwo%FCoeq7&3n$lX zR4L(Js_jw95XqmS1q%-0IZF5Po6X{PZ_DUK8L9ct-*t6DwcdJt!)%QE^(yNbQ_)uC z#(JOXu}vNXg@(OLE@fu8{((n{)Z24cJ~a?;?_Ianl>0q99sXYc^aKn0HdU079`W;2 zIO!17h=7LswD^XC{}TQtYr!=XE-V!#(b@L}-7bHNzQB(;|FY2EDtrL^p!$Jho%Hj_ z%DlL>BOOAFwReutJ3V$s8QM*wyD z3I10(>UR`#{j2(Z=5hjNJHPJuKhOUGi`1YAQAwDKaNd;yj}-F@LWssiVx*F zL;}!VhysYzbN0n5zv@l2IW>S%+obG^ z)1B}FWyY@V0vAw9;K>L;uMxvfF{7;HRX#;zT9QUXG+`(TcT5w$kk@G>Ks8h~N$CVs z0*K#7O!2MydzsS*kTEQ6vKX4t`s(+{!#UMnjrlXiuRD9@ z1!wN}S@OCi2tP5T0b~S_m7GRLV3l6x10YKj($B5QKbap%L(A#|cqiu^@=rw)R^hLy zKj9?ItpQN@hZtBxg38wMoJ+g;BMD?F(wU~DHy|B>Cu0KqLY_bjEAv{#5THhxqO&>r z0Ca;=5}+xtb6x+j*owa*f8Ud8dfKu*%ye8UZR&&djbPW!L z%yFs_0Pg%0{ul-X@*I^opd_mOi`n%RbLRoXSZ<68sNkzTml1#-mw-~+N9ix z7;-uZ4PXx_B??iC(tAmF|17gKUiWNL(W0X;a}fZ)Bo%&5yX2P>Sz6mW#dGR$2^b>) zeVm0&^(M3i&_J&jn~3PoS$CjVA{;9LV;3M2i7qYcT@>cj0OHTj*sk2tFDVNDPz`s@ z_|;x?>J(J<)EYno-CQ2f_o0;Tr>sMeUUR+n(Lcb)i;Tj|;R68rH!3e^YyrJVelE3h zTHJ--;ac%=i3oI^;&s>+s!hg}-gs{fbydoIF+-$a6JQ7E1*L;#to7BYw!a#>M)7AQ z(AVU+xEpZK(9f>-7d3*RPC%tiC5z4stwmwZjzID8(3i0mp&vtb?UrQz90T#XkO25t zG3CA*tK0YWHw?ot48t%C!!QiPFbu;m48t%C!!QiPFbu;m48t%C>krGo}x P00000NkvXXu0mjf8ME^t literal 0 HcmV?d00001 diff --git a/setup.py b/setup.py index 99e1fe75b9..fbf8f3ec1d 100644 --- a/setup.py +++ b/setup.py @@ -2,12 +2,16 @@ """Setup info for building Pype 3.0.""" import os import sys +from pathlib import Path from cx_Freeze import setup, Executable from sphinx.setup_command import BuildDoc version = {} -with open(os.path.join("pype", "version.py")) as fp: + +pype_root = Path(os.path.dirname(__file__)) + +with open(pype_root / "pype" / "version.py") as fp: exec(fp.read(), version) __version__ = version["__version__"] @@ -65,10 +69,13 @@ build_options = dict( include_files=include_files ) +icon_path = pype_root / "igniter" / "pype.ico" executables = [ - Executable("start.py", base=None, target_name="pype_console"), - Executable("start.py", base=base, target_name="pype") + Executable("start.py", base=None, + target_name="pype_console", icon=icon_path.as_posix()), + Executable("start.py", base=base, + target_name="pype", icon=icon_path.as_posix()) ] setup( @@ -82,8 +89,8 @@ setup( "project": "Pype", "version": __version__, "release": __version__, - "source_dir": "./docs/source", - "build_dir": "./docs/build" + "source_dir": (pype_root / "docs" / "source").as_posix(), + "build_dir": (pype_root / "docs" / "build").as_posix() } }, executables=executables diff --git a/start.py b/start.py index 82dfc1dce9..55fb979b8d 100644 --- a/start.py +++ b/start.py @@ -84,6 +84,10 @@ So, bootstrapping Pype looks like this:: Todo: Move or remove bootstrapping environments out of the code. +Attributes: + silent_commands (list): list of commands for which we won't print Pype + logo and info header. + .. _MongoDB: https://www.mongodb.com/ @@ -92,6 +96,7 @@ import os import re import sys import traceback +import subprocess import acre @@ -99,6 +104,9 @@ from igniter import BootstrapRepos from igniter.tools import load_environments +silent_commands = ["run", "igniter"] + + def set_environments() -> None: """Set loaded environments. @@ -106,14 +114,49 @@ def set_environments() -> None: better handling of environments """ - env = load_environments(["global"]) + env = {} + try: + env = load_environments(["global"]) + except OSError as e: + print(f"!!! {e}") + exit() + env = acre.merge(env, dict(os.environ)) os.environ.clear() os.environ.update(env) +def run(arguments: list, env: dict = None) -> int: + """Use correct executable to run stuff. + + This passing arguments to correct Pype executable. If Pype is run from + live sources, executable will be `python` in virtual environment. + If running from frozen code, executable will be `pype`. Its equivalent in + live code is `python start.py`. + + Args: + arguments (list): Argument list to pass Pype. + env (dict, optional): Dictionary containing environment. + + Returns: + int: Process return code. + + """ + if getattr(sys, 'frozen', False): + interpreter = [sys.executable] + else: + interpreter = [sys.executable, __file__] + + interpreter.extend(arguments) + + p = subprocess.Popen(interpreter, env=env) + p.wait() + print(f">>> done [{p.returncode}]") + return p.returncode + + def set_modules_environments(): - """Set global environments for pype's modules. + """Set global environments for pype modules. This requires to have pype in `sys.path`. """ @@ -129,10 +172,9 @@ def set_modules_environments(): if publish_plugin_dirs: publish_paths_str = os.environ.get("PYBLISHPLUGINPATH") or "" publish_paths = publish_paths_str.split(os.pathsep) - _publish_paths = set() - for path in publish_paths: - if path: - _publish_paths.add(os.path.normpath(path)) + _publish_paths = { + os.path.normpath(path) for path in publish_paths if path + } for path in publish_plugin_dirs: _publish_paths.add(os.path.normpath(path)) module_envs["PYBLISHPLUGINPATH"] = os.pathsep.join(_publish_paths) @@ -149,61 +191,133 @@ def boot(): """Bootstrap Pype.""" from pype.lib.terminal_splash import play_animation - play_animation() - - # find pype versions bootstrap = BootstrapRepos() - pype_versions = bootstrap.find_pype() - # check for `--use-version=3.0.0` argument. + # ------------------------------------------------------------------------ + # Process arguments + # ------------------------------------------------------------------------ + + # don't play for silenced commands + if all(item not in sys.argv for item in silent_commands): + play_animation() + + # check for `--use-version=3.0.0` argument and `--use-staging` use_version = None - + use_staging = False for arg in sys.argv: m = re.search(r"--use-version=(?P\d+\.\d+\.\d*.+?)", arg) if m and m.group('version'): use_version = m.group('version') sys.argv.remove(arg) break + if "--use-staging" in sys.argv: + use_staging = True + sys.argv.remove("--use-staging") + # handle igniter + # this is helper to run igniter before anything else + if "igniter" in sys.argv: + import igniter + igniter.run() + return + + # ------------------------------------------------------------------------ + # Determine mongodb connection + # ------------------------------------------------------------------------ + + # try env variable if not os.getenv("PYPE_MONGO"): + # try system keyring + pype_mongo = "" try: pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") except ValueError: print("*** No DB connection string specified.") print("--- launching setup UI ...") - import igniter - igniter.run() - return - else: + run(["igniter"]) + try: + pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") + except ValueError: + print("!!! Still no DB connection string.") + exit() + finally: os.environ["PYPE_MONGO"] = pype_mongo + # ------------------------------------------------------------------------ + # Load environments from database + # ------------------------------------------------------------------------ + set_environments() + + # ------------------------------------------------------------------------ + # Find Pype versions + # ------------------------------------------------------------------------ + + pype_versions = bootstrap.find_pype(include_zips=True) + pype_version = pype_versions[-1] + if getattr(sys, 'frozen', False): if not pype_versions: - import igniter - igniter.run() + print('*** No Pype versions found.') + print("--- launching setup UI ...") + run(["igniter"]) + pype_versions = bootstrap.find_pype() + if not pype_versions: + print('!!! Still no Pype versions found.') + return + # find only staging versions + if use_staging: + staging_versions = [v for v in pype_versions if v.is_staging()] + if not staging_versions: + print("!!! No staging versions detected.") + return + staging_versions.sort() + # get latest + pype_version = staging_versions[-1] + + # get path of version specified in `--use-version` version_path = BootstrapRepos.get_version_path_from_list( use_version, pype_versions) - if version_path: - # use specified - bootstrap.add_paths_from_directory(version_path) - - else: + if not version_path: if use_version is not None: print(("!!! Specified version was not found, using " "latest available")) - # use latest - version_path = pype_versions[-1].path - bootstrap.add_paths_from_directory(version_path) - use_version = str(pype_versions[-1]) + # specified version was not found so use latest detected. + version_path = pype_version.path + # test if latest detected is installed (in user data dir) + is_inside = False + try: + is_inside = pype_version.path.resolve().relative_to( + bootstrap.data_dir) + except ValueError: + # if relative path cannot be calculated, Pype version is not + # inside user data dir + pass + + if not is_inside: + # install latest version to user data dir + version_path = bootstrap.install_version( + pype_version, force=True) + + # inject version to Python environment (sys.path, ...) + bootstrap.add_paths_from_directory(version_path) + + # add stuff from `/lib` to PYTHONPATH. + os.environ["PYTHONPATH"] += os.pathsep + os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "lib") + ) + + # set PYPE_ROOT to point to currently used Pype version. os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) else: # run through repos and add them to sys.path and PYTHONPATH + # set root pype_root = os.path.normpath( os.path.dirname(os.path.realpath(__file__))) - local_version = bootstrap.get_local_version() + # get current version of Pype + local_version = bootstrap.get_local_live_version() if use_version and use_version != local_version: version_path = BootstrapRepos.get_version_path_from_list( use_version, pype_versions) @@ -248,10 +362,14 @@ def boot(): info.insert(0, ">>> Using Pype from [ {} ]".format( os.path.dirname(cli.__file__))) - info_length = len(max(info, key=len)) - info.insert(0, f"*** Pype [{__version__}] " + "-" * info_length) + t_width = os.get_terminal_size().columns + _header = f"*** Pype [{__version__}] " + + info.insert(0, _header + "-" * (t_width - len(_header))) for i in info: - t.echo(i) + # don't show for running scripts + if all(item not in sys.argv for item in silent_commands): + t.echo(i) try: cli.main(obj={}, prog_name="pype") @@ -302,7 +420,7 @@ def get_info() -> list: if log_components["auth_db"]: infos.append((" - auth source", log_components["auth_db"])) - maximum = max([len(i[0]) for i in infos]) + maximum = max(len(i[0]) for i in infos) formatted = [] for info in infos: padding = (maximum - len(info[0])) + 1 diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index c0ce1be012..34ddc12550 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -108,6 +108,11 @@ def test_pype_version(): assert v11.client == "client" +def test_get_main_version(): + ver = PypeVersion(1, 2, 3, variant="staging", client="foo") + assert ver.get_main_version() == "1.2.3" + + def test_get_version_path_from_list(): versions = [ PypeVersion(1, 2, 3, path=Path('/foo/bar')), diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 index 1fee947a38..bb04368964 100644 --- a/tools/create_env.ps1 +++ b/tools/create_env.ps1 @@ -105,14 +105,13 @@ catch { Exit-WithCode 1 } - Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Creating virtual env ..." & python -m venv venv Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Entering venv ..." try { - . (".\venv\Scripts\Activate.ps1") + . ("$($pype_root)\venv\Scripts\Activate.ps1") } catch { Write-Host "!!! Failed to activate" -ForegroundColor red From a6e91195fa7866dee709971016d2afc63dfbb3ec Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Jan 2021 19:07:09 +0100 Subject: [PATCH 25/50] SyncServer GUI - safer creation of gui --- pype/modules/sync_server/tray/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index aa52b06da9..8b11445157 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -45,6 +45,7 @@ class SyncServerWindow(QtWidgets.QDialog): container = QtWidgets.QWidget() projects = SyncProjectListWidget(parent=self) + projects.refresh() # force selection of default repres = SyncRepresentationWidget(project=projects.current_project, parent=self) From 432dc13b9d2bf1598972cd3db17c5ebe39b5cd3f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 14 Jan 2021 20:47:24 +0100 Subject: [PATCH 26/50] bugfixes and refactors --- docs/source/pype.hosts.aftereffects.rst | 7 +++ ...pype.hosts.resolve.otio.davinci_export.rst | 7 +++ ...pype.hosts.resolve.otio.davinci_import.rst | 7 +++ docs/source/pype.hosts.resolve.otio.rst | 17 +++++++ docs/source/pype.hosts.resolve.otio.utils.rst | 7 +++ .../pype.hosts.resolve.todo-rendering.rst | 7 +++ docs/source/pype.hosts.tvpaint.api.rst | 7 +++ docs/source/pype.hosts.tvpaint.rst | 15 ++++++ .../pype.lib.abstract_collect_render.rst | 7 +++ .../pype.lib.abstract_expected_files.rst | 7 +++ docs/source/pype.lib.abstract_metaplugins.rst | 7 +++ .../pype.lib.abstract_submit_deadline.rst | 7 +++ docs/source/pype.lib.applications.rst | 7 +++ docs/source/pype.lib.avalon_context.rst | 7 +++ docs/source/pype.lib.deprecated.rst | 7 +++ docs/source/pype.lib.editorial.rst | 7 +++ docs/source/pype.lib.env_tools.rst | 7 +++ docs/source/pype.lib.ffmpeg_utils.rst | 7 +++ docs/source/pype.lib.path_tools.rst | 7 +++ docs/source/pype.lib.plugin_tools.rst | 7 +++ docs/source/pype.lib.python_module_tools.rst | 7 +++ docs/source/pype.lib.terminal_splash.rst | 7 +++ .../pype.modules.clockify.clockify_module.rst | 7 +++ .../pype.modules.deadline.deadline_module.rst | 7 +++ docs/source/pype.modules.deadline.rst | 15 ++++++ .../pype.modules.ftrack.ftrack_module.rst | 7 +++ .../pype.modules.ftrack.lib.settings.rst | 7 +++ .../pype.modules.ftrack.tray.ftrack_tray.rst | 7 +++ docs/source/pype.modules.launcher_action.rst | 7 +++ ...ype.modules.log_viewer.log_view_module.rst | 7 +++ docs/source/pype.modules.log_viewer.rst | 23 +++++++++ .../pype.modules.log_viewer.tray.app.rst | 7 +++ .../pype.modules.log_viewer.tray.models.rst | 7 +++ docs/source/pype.modules.log_viewer.tray.rst | 17 +++++++ .../pype.modules.log_viewer.tray.widgets.rst | 7 +++ docs/source/pype.modules.settings_action.rst | 7 +++ .../pype.modules.standalonepublish_action.rst | 7 +++ docs/source/pype.modules.sync_server.rst | 16 ++++++ .../pype.modules.sync_server.sync_server.rst | 7 +++ .../source/pype.modules.sync_server.utils.rst | 7 +++ ...es.websocket_server.hosts.aftereffects.rst | 7 +++ ....publish.validate_vray_referenced_aovs.rst | 7 +++ docs/source/pype.settings.constants.rst | 7 +++ docs/source/pype.settings.handlers.rst | 7 +++ ...ype.tests.test_lib_restructuralization.rst | 7 +++ docs/source/pype.tools.tray.pype_tray.rst | 7 +++ docs/source/pype.tools.tray.rst | 15 ++++++ docs/source/pype.tools.workfiles.app.rst | 7 +++ docs/source/pype.tools.workfiles.model.rst | 7 +++ docs/source/pype.tools.workfiles.rst | 17 +++++++ docs/source/pype.tools.workfiles.view.rst | 7 +++ igniter/__init__.py | 6 +-- igniter/bootstrap_repos.py | 49 ++++++++++++++----- igniter/install_dialog.py | 9 ++-- start.py | 19 +++++-- 55 files changed, 497 insertions(+), 22 deletions(-) create mode 100644 docs/source/pype.hosts.aftereffects.rst create mode 100644 docs/source/pype.hosts.resolve.otio.davinci_export.rst create mode 100644 docs/source/pype.hosts.resolve.otio.davinci_import.rst create mode 100644 docs/source/pype.hosts.resolve.otio.rst create mode 100644 docs/source/pype.hosts.resolve.otio.utils.rst create mode 100644 docs/source/pype.hosts.resolve.todo-rendering.rst create mode 100644 docs/source/pype.hosts.tvpaint.api.rst create mode 100644 docs/source/pype.hosts.tvpaint.rst create mode 100644 docs/source/pype.lib.abstract_collect_render.rst create mode 100644 docs/source/pype.lib.abstract_expected_files.rst create mode 100644 docs/source/pype.lib.abstract_metaplugins.rst create mode 100644 docs/source/pype.lib.abstract_submit_deadline.rst create mode 100644 docs/source/pype.lib.applications.rst create mode 100644 docs/source/pype.lib.avalon_context.rst create mode 100644 docs/source/pype.lib.deprecated.rst create mode 100644 docs/source/pype.lib.editorial.rst create mode 100644 docs/source/pype.lib.env_tools.rst create mode 100644 docs/source/pype.lib.ffmpeg_utils.rst create mode 100644 docs/source/pype.lib.path_tools.rst create mode 100644 docs/source/pype.lib.plugin_tools.rst create mode 100644 docs/source/pype.lib.python_module_tools.rst create mode 100644 docs/source/pype.lib.terminal_splash.rst create mode 100644 docs/source/pype.modules.clockify.clockify_module.rst create mode 100644 docs/source/pype.modules.deadline.deadline_module.rst create mode 100644 docs/source/pype.modules.deadline.rst create mode 100644 docs/source/pype.modules.ftrack.ftrack_module.rst create mode 100644 docs/source/pype.modules.ftrack.lib.settings.rst create mode 100644 docs/source/pype.modules.ftrack.tray.ftrack_tray.rst create mode 100644 docs/source/pype.modules.launcher_action.rst create mode 100644 docs/source/pype.modules.log_viewer.log_view_module.rst create mode 100644 docs/source/pype.modules.log_viewer.rst create mode 100644 docs/source/pype.modules.log_viewer.tray.app.rst create mode 100644 docs/source/pype.modules.log_viewer.tray.models.rst create mode 100644 docs/source/pype.modules.log_viewer.tray.rst create mode 100644 docs/source/pype.modules.log_viewer.tray.widgets.rst create mode 100644 docs/source/pype.modules.settings_action.rst create mode 100644 docs/source/pype.modules.standalonepublish_action.rst create mode 100644 docs/source/pype.modules.sync_server.rst create mode 100644 docs/source/pype.modules.sync_server.sync_server.rst create mode 100644 docs/source/pype.modules.sync_server.utils.rst create mode 100644 docs/source/pype.modules.websocket_server.hosts.aftereffects.rst create mode 100644 docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst create mode 100644 docs/source/pype.settings.constants.rst create mode 100644 docs/source/pype.settings.handlers.rst create mode 100644 docs/source/pype.tests.test_lib_restructuralization.rst create mode 100644 docs/source/pype.tools.tray.pype_tray.rst create mode 100644 docs/source/pype.tools.tray.rst create mode 100644 docs/source/pype.tools.workfiles.app.rst create mode 100644 docs/source/pype.tools.workfiles.model.rst create mode 100644 docs/source/pype.tools.workfiles.rst create mode 100644 docs/source/pype.tools.workfiles.view.rst diff --git a/docs/source/pype.hosts.aftereffects.rst b/docs/source/pype.hosts.aftereffects.rst new file mode 100644 index 0000000000..3c2b2dda41 --- /dev/null +++ b/docs/source/pype.hosts.aftereffects.rst @@ -0,0 +1,7 @@ +pype.hosts.aftereffects package +=============================== + +.. automodule:: pype.hosts.aftereffects + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.davinci_export.rst b/docs/source/pype.hosts.resolve.otio.davinci_export.rst new file mode 100644 index 0000000000..498f96a7ed --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.davinci_export.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.davinci\_export module +============================================== + +.. automodule:: pype.hosts.resolve.otio.davinci_export + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.davinci_import.rst b/docs/source/pype.hosts.resolve.otio.davinci_import.rst new file mode 100644 index 0000000000..30f43cc9fe --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.davinci_import.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.davinci\_import module +============================================== + +.. automodule:: pype.hosts.resolve.otio.davinci_import + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.otio.rst b/docs/source/pype.hosts.resolve.otio.rst new file mode 100644 index 0000000000..523d8937ca --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.rst @@ -0,0 +1,17 @@ +pype.hosts.resolve.otio package +=============================== + +.. automodule:: pype.hosts.resolve.otio + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.hosts.resolve.otio.davinci_export + pype.hosts.resolve.otio.davinci_import + pype.hosts.resolve.otio.utils diff --git a/docs/source/pype.hosts.resolve.otio.utils.rst b/docs/source/pype.hosts.resolve.otio.utils.rst new file mode 100644 index 0000000000..765f492732 --- /dev/null +++ b/docs/source/pype.hosts.resolve.otio.utils.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.otio.utils module +==================================== + +.. automodule:: pype.hosts.resolve.otio.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.resolve.todo-rendering.rst b/docs/source/pype.hosts.resolve.todo-rendering.rst new file mode 100644 index 0000000000..8ea80183ce --- /dev/null +++ b/docs/source/pype.hosts.resolve.todo-rendering.rst @@ -0,0 +1,7 @@ +pype.hosts.resolve.todo\-rendering module +========================================= + +.. automodule:: pype.hosts.resolve.todo-rendering + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.tvpaint.api.rst b/docs/source/pype.hosts.tvpaint.api.rst new file mode 100644 index 0000000000..43273e8ec5 --- /dev/null +++ b/docs/source/pype.hosts.tvpaint.api.rst @@ -0,0 +1,7 @@ +pype.hosts.tvpaint.api package +============================== + +.. automodule:: pype.hosts.tvpaint.api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.hosts.tvpaint.rst b/docs/source/pype.hosts.tvpaint.rst new file mode 100644 index 0000000000..561be3a9dc --- /dev/null +++ b/docs/source/pype.hosts.tvpaint.rst @@ -0,0 +1,15 @@ +pype.hosts.tvpaint package +========================== + +.. automodule:: pype.hosts.tvpaint + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 10 + + pype.hosts.tvpaint.api diff --git a/docs/source/pype.lib.abstract_collect_render.rst b/docs/source/pype.lib.abstract_collect_render.rst new file mode 100644 index 0000000000..d6adadc271 --- /dev/null +++ b/docs/source/pype.lib.abstract_collect_render.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_collect\_render module +========================================= + +.. automodule:: pype.lib.abstract_collect_render + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_expected_files.rst b/docs/source/pype.lib.abstract_expected_files.rst new file mode 100644 index 0000000000..904aeb3375 --- /dev/null +++ b/docs/source/pype.lib.abstract_expected_files.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_expected\_files module +========================================= + +.. automodule:: pype.lib.abstract_expected_files + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_metaplugins.rst b/docs/source/pype.lib.abstract_metaplugins.rst new file mode 100644 index 0000000000..9f2751b630 --- /dev/null +++ b/docs/source/pype.lib.abstract_metaplugins.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_metaplugins module +===================================== + +.. automodule:: pype.lib.abstract_metaplugins + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.abstract_submit_deadline.rst b/docs/source/pype.lib.abstract_submit_deadline.rst new file mode 100644 index 0000000000..a57222add3 --- /dev/null +++ b/docs/source/pype.lib.abstract_submit_deadline.rst @@ -0,0 +1,7 @@ +pype.lib.abstract\_submit\_deadline module +========================================== + +.. automodule:: pype.lib.abstract_submit_deadline + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.applications.rst b/docs/source/pype.lib.applications.rst new file mode 100644 index 0000000000..8d1ff9b2c6 --- /dev/null +++ b/docs/source/pype.lib.applications.rst @@ -0,0 +1,7 @@ +pype.lib.applications module +============================ + +.. automodule:: pype.lib.applications + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.avalon_context.rst b/docs/source/pype.lib.avalon_context.rst new file mode 100644 index 0000000000..067ea3380f --- /dev/null +++ b/docs/source/pype.lib.avalon_context.rst @@ -0,0 +1,7 @@ +pype.lib.avalon\_context module +=============================== + +.. automodule:: pype.lib.avalon_context + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.deprecated.rst b/docs/source/pype.lib.deprecated.rst new file mode 100644 index 0000000000..ec5ee58d67 --- /dev/null +++ b/docs/source/pype.lib.deprecated.rst @@ -0,0 +1,7 @@ +pype.lib.deprecated module +========================== + +.. automodule:: pype.lib.deprecated + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.editorial.rst b/docs/source/pype.lib.editorial.rst new file mode 100644 index 0000000000..d32e495e51 --- /dev/null +++ b/docs/source/pype.lib.editorial.rst @@ -0,0 +1,7 @@ +pype.lib.editorial module +========================= + +.. automodule:: pype.lib.editorial + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.env_tools.rst b/docs/source/pype.lib.env_tools.rst new file mode 100644 index 0000000000..cb470207c8 --- /dev/null +++ b/docs/source/pype.lib.env_tools.rst @@ -0,0 +1,7 @@ +pype.lib.env\_tools module +========================== + +.. automodule:: pype.lib.env_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.ffmpeg_utils.rst b/docs/source/pype.lib.ffmpeg_utils.rst new file mode 100644 index 0000000000..968a3f39c8 --- /dev/null +++ b/docs/source/pype.lib.ffmpeg_utils.rst @@ -0,0 +1,7 @@ +pype.lib.ffmpeg\_utils module +============================= + +.. automodule:: pype.lib.ffmpeg_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.path_tools.rst b/docs/source/pype.lib.path_tools.rst new file mode 100644 index 0000000000..c19c41eea3 --- /dev/null +++ b/docs/source/pype.lib.path_tools.rst @@ -0,0 +1,7 @@ +pype.lib.path\_tools module +=========================== + +.. automodule:: pype.lib.path_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.plugin_tools.rst b/docs/source/pype.lib.plugin_tools.rst new file mode 100644 index 0000000000..6eadc5d3be --- /dev/null +++ b/docs/source/pype.lib.plugin_tools.rst @@ -0,0 +1,7 @@ +pype.lib.plugin\_tools module +============================= + +.. automodule:: pype.lib.plugin_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.python_module_tools.rst b/docs/source/pype.lib.python_module_tools.rst new file mode 100644 index 0000000000..c916080bce --- /dev/null +++ b/docs/source/pype.lib.python_module_tools.rst @@ -0,0 +1,7 @@ +pype.lib.python\_module\_tools module +===================================== + +.. automodule:: pype.lib.python_module_tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.lib.terminal_splash.rst b/docs/source/pype.lib.terminal_splash.rst new file mode 100644 index 0000000000..06038f0f09 --- /dev/null +++ b/docs/source/pype.lib.terminal_splash.rst @@ -0,0 +1,7 @@ +pype.lib.terminal\_splash module +================================ + +.. automodule:: pype.lib.terminal_splash + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.clockify.clockify_module.rst b/docs/source/pype.modules.clockify.clockify_module.rst new file mode 100644 index 0000000000..85f8e75ad1 --- /dev/null +++ b/docs/source/pype.modules.clockify.clockify_module.rst @@ -0,0 +1,7 @@ +pype.modules.clockify.clockify\_module module +============================================= + +.. automodule:: pype.modules.clockify.clockify_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.deadline.deadline_module.rst b/docs/source/pype.modules.deadline.deadline_module.rst new file mode 100644 index 0000000000..43e7198a8b --- /dev/null +++ b/docs/source/pype.modules.deadline.deadline_module.rst @@ -0,0 +1,7 @@ +pype.modules.deadline.deadline\_module module +============================================= + +.. automodule:: pype.modules.deadline.deadline_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.deadline.rst b/docs/source/pype.modules.deadline.rst new file mode 100644 index 0000000000..7633b2b950 --- /dev/null +++ b/docs/source/pype.modules.deadline.rst @@ -0,0 +1,15 @@ +pype.modules.deadline package +============================= + +.. automodule:: pype.modules.deadline + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.deadline.deadline_module diff --git a/docs/source/pype.modules.ftrack.ftrack_module.rst b/docs/source/pype.modules.ftrack.ftrack_module.rst new file mode 100644 index 0000000000..4188ffbed8 --- /dev/null +++ b/docs/source/pype.modules.ftrack.ftrack_module.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.ftrack\_module module +========================================= + +.. automodule:: pype.modules.ftrack.ftrack_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.ftrack.lib.settings.rst b/docs/source/pype.modules.ftrack.lib.settings.rst new file mode 100644 index 0000000000..255d52178a --- /dev/null +++ b/docs/source/pype.modules.ftrack.lib.settings.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.lib.settings module +======================================= + +.. automodule:: pype.modules.ftrack.lib.settings + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst b/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst new file mode 100644 index 0000000000..147647e9b4 --- /dev/null +++ b/docs/source/pype.modules.ftrack.tray.ftrack_tray.rst @@ -0,0 +1,7 @@ +pype.modules.ftrack.tray.ftrack\_tray module +============================================ + +.. automodule:: pype.modules.ftrack.tray.ftrack_tray + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.launcher_action.rst b/docs/source/pype.modules.launcher_action.rst new file mode 100644 index 0000000000..a63408e747 --- /dev/null +++ b/docs/source/pype.modules.launcher_action.rst @@ -0,0 +1,7 @@ +pype.modules.launcher\_action module +==================================== + +.. automodule:: pype.modules.launcher_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.log_view_module.rst b/docs/source/pype.modules.log_viewer.log_view_module.rst new file mode 100644 index 0000000000..8d80170a9c --- /dev/null +++ b/docs/source/pype.modules.log_viewer.log_view_module.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.log\_view\_module module +================================================= + +.. automodule:: pype.modules.log_viewer.log_view_module + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.rst b/docs/source/pype.modules.log_viewer.rst new file mode 100644 index 0000000000..e275d56086 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.rst @@ -0,0 +1,23 @@ +pype.modules.log\_viewer package +================================ + +.. automodule:: pype.modules.log_viewer + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.tray + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.log_view_module diff --git a/docs/source/pype.modules.log_viewer.tray.app.rst b/docs/source/pype.modules.log_viewer.tray.app.rst new file mode 100644 index 0000000000..0948a05594 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.app.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.app module +======================================== + +.. automodule:: pype.modules.log_viewer.tray.app + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.tray.models.rst b/docs/source/pype.modules.log_viewer.tray.models.rst new file mode 100644 index 0000000000..4da3887600 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.models.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.models module +=========================================== + +.. automodule:: pype.modules.log_viewer.tray.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.log_viewer.tray.rst b/docs/source/pype.modules.log_viewer.tray.rst new file mode 100644 index 0000000000..5f4b92f627 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.rst @@ -0,0 +1,17 @@ +pype.modules.log\_viewer.tray package +===================================== + +.. automodule:: pype.modules.log_viewer.tray + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.log_viewer.tray.app + pype.modules.log_viewer.tray.models + pype.modules.log_viewer.tray.widgets diff --git a/docs/source/pype.modules.log_viewer.tray.widgets.rst b/docs/source/pype.modules.log_viewer.tray.widgets.rst new file mode 100644 index 0000000000..cb57c96559 --- /dev/null +++ b/docs/source/pype.modules.log_viewer.tray.widgets.rst @@ -0,0 +1,7 @@ +pype.modules.log\_viewer.tray.widgets module +============================================ + +.. automodule:: pype.modules.log_viewer.tray.widgets + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.settings_action.rst b/docs/source/pype.modules.settings_action.rst new file mode 100644 index 0000000000..10f0881ced --- /dev/null +++ b/docs/source/pype.modules.settings_action.rst @@ -0,0 +1,7 @@ +pype.modules.settings\_action module +==================================== + +.. automodule:: pype.modules.settings_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.standalonepublish_action.rst b/docs/source/pype.modules.standalonepublish_action.rst new file mode 100644 index 0000000000..d51dbcefa0 --- /dev/null +++ b/docs/source/pype.modules.standalonepublish_action.rst @@ -0,0 +1,7 @@ +pype.modules.standalonepublish\_action module +============================================= + +.. automodule:: pype.modules.standalonepublish_action + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.sync_server.rst b/docs/source/pype.modules.sync_server.rst new file mode 100644 index 0000000000..a26dc7e212 --- /dev/null +++ b/docs/source/pype.modules.sync_server.rst @@ -0,0 +1,16 @@ +pype.modules.sync\_server package +================================= + +.. automodule:: pype.modules.sync_server + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.modules.sync_server.sync_server + pype.modules.sync_server.utils diff --git a/docs/source/pype.modules.sync_server.sync_server.rst b/docs/source/pype.modules.sync_server.sync_server.rst new file mode 100644 index 0000000000..36d6aa68ed --- /dev/null +++ b/docs/source/pype.modules.sync_server.sync_server.rst @@ -0,0 +1,7 @@ +pype.modules.sync\_server.sync\_server module +============================================= + +.. automodule:: pype.modules.sync_server.sync_server + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.sync_server.utils.rst b/docs/source/pype.modules.sync_server.utils.rst new file mode 100644 index 0000000000..325d5e435d --- /dev/null +++ b/docs/source/pype.modules.sync_server.utils.rst @@ -0,0 +1,7 @@ +pype.modules.sync\_server.utils module +====================================== + +.. automodule:: pype.modules.sync_server.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst b/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst new file mode 100644 index 0000000000..9f4720ae14 --- /dev/null +++ b/docs/source/pype.modules.websocket_server.hosts.aftereffects.rst @@ -0,0 +1,7 @@ +pype.modules.websocket\_server.hosts.aftereffects module +======================================================== + +.. automodule:: pype.modules.websocket_server.hosts.aftereffects + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst b/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst new file mode 100644 index 0000000000..16ad9666aa --- /dev/null +++ b/docs/source/pype.plugins.maya.publish.validate_vray_referenced_aovs.rst @@ -0,0 +1,7 @@ +pype.plugins.maya.publish.validate\_vray\_referenced\_aovs module +================================================================= + +.. automodule:: pype.plugins.maya.publish.validate_vray_referenced_aovs + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.settings.constants.rst b/docs/source/pype.settings.constants.rst new file mode 100644 index 0000000000..ac652089c8 --- /dev/null +++ b/docs/source/pype.settings.constants.rst @@ -0,0 +1,7 @@ +pype.settings.constants module +============================== + +.. automodule:: pype.settings.constants + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.settings.handlers.rst b/docs/source/pype.settings.handlers.rst new file mode 100644 index 0000000000..60ea0ae952 --- /dev/null +++ b/docs/source/pype.settings.handlers.rst @@ -0,0 +1,7 @@ +pype.settings.handlers module +============================= + +.. automodule:: pype.settings.handlers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tests.test_lib_restructuralization.rst b/docs/source/pype.tests.test_lib_restructuralization.rst new file mode 100644 index 0000000000..8d426fcb6b --- /dev/null +++ b/docs/source/pype.tests.test_lib_restructuralization.rst @@ -0,0 +1,7 @@ +pype.tests.test\_lib\_restructuralization module +================================================ + +.. automodule:: pype.tests.test_lib_restructuralization + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.tray.pype_tray.rst b/docs/source/pype.tools.tray.pype_tray.rst new file mode 100644 index 0000000000..9fc49c5763 --- /dev/null +++ b/docs/source/pype.tools.tray.pype_tray.rst @@ -0,0 +1,7 @@ +pype.tools.tray.pype\_tray module +================================= + +.. automodule:: pype.tools.tray.pype_tray + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.tray.rst b/docs/source/pype.tools.tray.rst new file mode 100644 index 0000000000..b28059d170 --- /dev/null +++ b/docs/source/pype.tools.tray.rst @@ -0,0 +1,15 @@ +pype.tools.tray package +======================= + +.. automodule:: pype.tools.tray + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.tools.tray.pype_tray diff --git a/docs/source/pype.tools.workfiles.app.rst b/docs/source/pype.tools.workfiles.app.rst new file mode 100644 index 0000000000..a3a46b8a07 --- /dev/null +++ b/docs/source/pype.tools.workfiles.app.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.app module +=============================== + +.. automodule:: pype.tools.workfiles.app + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.workfiles.model.rst b/docs/source/pype.tools.workfiles.model.rst new file mode 100644 index 0000000000..44cea32b97 --- /dev/null +++ b/docs/source/pype.tools.workfiles.model.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.model module +================================= + +.. automodule:: pype.tools.workfiles.model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/pype.tools.workfiles.rst b/docs/source/pype.tools.workfiles.rst new file mode 100644 index 0000000000..147c4cebbe --- /dev/null +++ b/docs/source/pype.tools.workfiles.rst @@ -0,0 +1,17 @@ +pype.tools.workfiles package +============================ + +.. automodule:: pype.tools.workfiles + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +.. toctree:: + :maxdepth: 10 + + pype.tools.workfiles.app + pype.tools.workfiles.model + pype.tools.workfiles.view diff --git a/docs/source/pype.tools.workfiles.view.rst b/docs/source/pype.tools.workfiles.view.rst new file mode 100644 index 0000000000..acd32ed250 --- /dev/null +++ b/docs/source/pype.tools.workfiles.view.rst @@ -0,0 +1,7 @@ +pype.tools.workfiles.view module +================================ + +.. automodule:: pype.tools.workfiles.view + :members: + :undoc-members: + :show-inheritance: diff --git a/igniter/__init__.py b/igniter/__init__.py index eda37c5af3..ffac2b023f 100644 --- a/igniter/__init__.py +++ b/igniter/__init__.py @@ -10,11 +10,11 @@ from .bootstrap_repos import BootstrapRepos def run(): """Show Igniter dialog.""" - # app = QtWidgets.QApplication(sys.argv) + app = QtWidgets.QApplication(sys.argv) d = InstallDialog() d.exec_() - #d.show() - #sys.exit(app.exec_()) + d.show() + sys.exit(app.exec_()) __all__ = [ diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 2a71887fbf..f53aa77db7 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -8,7 +8,7 @@ import shutil import sys import tempfile from pathlib import Path -from typing import Union, Callable, List, Tuple +from typing import Union, Callable, List, Tuple, Optional from zipfile import ZipFile, BadZipFile from appdirs import user_data_dir @@ -62,9 +62,10 @@ class PypeVersion: path: Path = None): self.path = path - if major is None or minor is None or subversion is None: - if version is None: - raise ValueError("Need version specified in some way.") + if ( + major is None or minor is None or subversion is None + ) and version is None: + raise ValueError("Need version specified in some way.") if version: values = self._decompose_version(version) self.major = values[0] @@ -133,12 +134,18 @@ class PypeVersion: self.subversion < other.subversion: return True - if self.major == other.major and self.minor == other.minor and \ - self.subversion == other.subversion and \ - self.variant == "staging": + # Directory takes precedence over file + if ( + self.path + and other.path + and other.path.is_dir() + and self.path.is_file() + ): return True - return False + return self.major == other.major and self.minor == other.minor and \ + self.subversion == other.subversion and self.variant == "staging" + def is_staging(self) -> bool: """Test if current version is staging one.""" @@ -435,6 +442,9 @@ class BootstrapRepos: archive (Path): path to archive. """ + if not archive.is_file() and not archive.exists(): + raise ValueError("Archive is not file.") + with ZipFile(archive, "r") as zip_file: name_list = zip_file.namelist() @@ -464,6 +474,9 @@ class BootstrapRepos: directory (Path): path to directory. """ + if not directory.exists() and not directory.is_dir(): + raise ValueError("directory is invalid") + roots = [] for item in directory.iterdir(): if item.is_dir(): @@ -583,8 +596,8 @@ class BootstrapRepos: version_check = PypeVersion( version=zip_version["__version__"]) - version_main = version_check.get_main_version() - detected_main = detected_version.get_main_version() + version_main = version_check.get_main_version() # noqa: E501 + detected_main = detected_version.get_main_version() # noqa: E501 if version_main != detected_main: self._log.error( @@ -762,7 +775,17 @@ class BootstrapRepos: if self._message: self._message.emit(message, error) - def extract_pype(self, version: PypeVersion): + def extract_pype(self, version: PypeVersion) -> Union[Path, None]: + """Extract zipped Pype version to user data directory. + + Args: + version (PypeVersion): Version of Pype. + + Returns: + Path: path to extracted version. + None: if something failed. + + """ if not version.path: raise ValueError( f"version {version} is not associated with any file") @@ -777,7 +800,7 @@ class BootstrapRepos: self._log.error(e.strerror) self._print(msg, True) self._print(e.strerror, True) - return + return None destination.mkdir(parents=True) @@ -788,6 +811,8 @@ class BootstrapRepos: self._print(f"Installed as {version.path.stem}") + return destination + def install_version(self, pype_version: PypeVersion, force: bool = False): """Install Pype version to user data directory. diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index 74d9435815..b4fa68d89a 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -308,14 +308,17 @@ class InstallDialog(QtWidgets.QDialog): options |= QtWidgets.QFileDialog.DontUseNativeDialog options |= QtWidgets.QFileDialog.ShowDirsOnly - filename, _ = QtWidgets.QFileDialog.getExistingDirectory( + result = QtWidgets.QFileDialog.getExistingDirectory( parent=self, caption='Select path', directory=os.getcwd(), options=options) - if filename: - filename = QtCore.QDir.toNativeSeparators(filename) + if not result: + return + + filename = result[0] + filename = QtCore.QDir.toNativeSeparators(filename) if os.path.isdir(filename): self.user_input.setText(filename) diff --git a/start.py b/start.py index 55fb979b8d..75673e63c5 100644 --- a/start.py +++ b/start.py @@ -119,7 +119,7 @@ def set_environments() -> None: env = load_environments(["global"]) except OSError as e: print(f"!!! {e}") - exit() + sys.exit(1) env = acre.merge(env, dict(os.environ)) os.environ.clear() @@ -239,7 +239,7 @@ def boot(): pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") except ValueError: print("!!! Still no DB connection string.") - exit() + sys.exit(1) finally: os.environ["PYPE_MONGO"] = pype_mongo @@ -253,8 +253,13 @@ def boot(): # Find Pype versions # ------------------------------------------------------------------------ + pype_version = None pype_versions = bootstrap.find_pype(include_zips=True) - pype_version = pype_versions[-1] + try: + pype_version = pype_versions[-1] + except IndexError: + # no pype version found + pass if getattr(sys, 'frozen', False): if not pype_versions: @@ -285,6 +290,8 @@ def boot(): "latest available")) # specified version was not found so use latest detected. version_path = pype_version.path + print(f">>> Using version [ {pype_version} ]") + print(f" From {version_path}") # test if latest detected is installed (in user data dir) is_inside = False @@ -301,7 +308,12 @@ def boot(): version_path = bootstrap.install_version( pype_version, force=True) + if pype_version.path.is_file(): + print(">>> Extracting zip file ...") + version_path = bootstrap.extract_pype(pype_version) + # inject version to Python environment (sys.path, ...) + print(">>> Injecting Pype version to running environment ...") bootstrap.add_paths_from_directory(version_path) # add stuff from `/lib` to PYTHONPATH. @@ -355,7 +367,6 @@ def boot(): from pype.lib import terminal as t from pype.version import __version__ print(">>> loading environments ...") - set_environments() set_modules_environments() info = get_info() From 78ca7e8bcff3a47341b8dc8a28eee69fe51dc58b Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 14 Jan 2021 21:30:32 +0100 Subject: [PATCH 27/50] reduce complexity, fix hound --- igniter/bootstrap_repos.py | 8 +- start.py | 255 ++++++++++++++++++++++++------------- 2 files changed, 167 insertions(+), 96 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index f53aa77db7..b485054920 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -8,7 +8,7 @@ import shutil import sys import tempfile from pathlib import Path -from typing import Union, Callable, List, Tuple, Optional +from typing import Union, Callable, List, Tuple from zipfile import ZipFile, BadZipFile from appdirs import user_data_dir @@ -146,7 +146,6 @@ class PypeVersion: return self.major == other.major and self.minor == other.minor and \ self.subversion == other.subversion and self.variant == "staging" - def is_staging(self) -> bool: """Test if current version is staging one.""" return self.variant == "staging" @@ -222,6 +221,7 @@ class BootstrapRepos: # dummy progress reporter def empty_progress(x: int): + """Progress callback dummy.""" return x if not progress_callback: @@ -900,8 +900,8 @@ class BootstrapRepos: # extract zip there self._log.info("extracting zip to destination ...") - with ZipFile(pype_version.path, "r") as zip: - zip.extractall(destination) + with ZipFile(pype_version.path, "r") as zip_ref: + zip_ref.extractall(destination) return destination diff --git a/start.py b/start.py index 75673e63c5..87633afba5 100644 --- a/start.py +++ b/start.py @@ -97,6 +97,7 @@ import re import sys import traceback import subprocess +from pathlib import Path import acre @@ -104,6 +105,7 @@ from igniter import BootstrapRepos from igniter.tools import load_environments +bootstrap = BootstrapRepos() silent_commands = ["run", "igniter"] @@ -114,7 +116,6 @@ def set_environments() -> None: better handling of environments """ - env = {} try: env = load_environments(["global"]) except OSError as e: @@ -187,19 +188,13 @@ def set_modules_environments(): os.environ.update(env) -def boot(): - """Bootstrap Pype.""" +def _process_arguments() -> tuple: + """Process command line arguments. - from pype.lib.terminal_splash import play_animation - bootstrap = BootstrapRepos() - - # ------------------------------------------------------------------------ - # Process arguments - # ------------------------------------------------------------------------ - - # don't play for silenced commands - if all(item not in sys.argv for item in silent_commands): - play_animation() + Returns: + tuple: Return tuple with specific version to use (if any) and flag + to prioritize staging (if set) + """ # check for `--use-version=3.0.0` argument and `--use-staging` use_version = None @@ -219,16 +214,26 @@ def boot(): if "igniter" in sys.argv: import igniter igniter.run() - return + return use_version, use_staging - # ------------------------------------------------------------------------ - # Determine mongodb connection - # ------------------------------------------------------------------------ - # try env variable - if not os.getenv("PYPE_MONGO"): +def _determine_mongodb() -> str: + """Determine mongodb connection string. + + First use ``PYPE_MONGO`` environment variable, then system keyring. + Then try to run **Igniter UI** to let user specify it. + + Returns: + str: mongodb connection URL + + Raises: + RuntimeError: if mongodb connection url cannot by determined. + + """ + + pype_mongo = os.getenv("PYPE_MONGO", None) + if not pype_mongo: # try system keyring - pype_mongo = "" try: pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") except ValueError: @@ -238,10 +243,134 @@ def boot(): try: pype_mongo = bootstrap.registry.get_secure_item("pypeMongo") except ValueError: - print("!!! Still no DB connection string.") - sys.exit(1) - finally: - os.environ["PYPE_MONGO"] = pype_mongo + raise RuntimeError("missing mongodb url") + + return pype_mongo + + +def _find_frozen_pype(use_version: str = None, + use_staging: bool = False) -> Path: + """Find Pype to run from frozen code. + + Args: + use_version (str, optional): Try to use specified version. + use_staging (bool, optional): Prefer *staging* flavor over production. + + Returns: + Path: Path to version to be used. + + Raises: + RuntimeError: If no Pype version are found or no staging version + (if requested). + + """ + + pype_version = None + pype_versions = bootstrap.find_pype(include_zips=True) + try: + # use latest one found (last in the list is latest) + pype_version = pype_versions[-1] + except IndexError: + # no pype version found, run Igniter and ask for them. + print('*** No Pype versions found.') + print("--- launching setup UI ...") + run(["igniter"]) + pype_versions = bootstrap.find_pype() + + if not pype_versions: + raise RuntimeError("No Pype versions found.") + + # find only staging versions + if use_staging: + staging_versions = [v for v in pype_versions if v.is_staging()] + if not staging_versions: + raise RuntimeError("No Pype staging versions found.") + + staging_versions.sort() + # get latest staging version (last in the list is latest) + pype_version = staging_versions[-1] + + # get path of version specified in `--use-version` + version_path = BootstrapRepos.get_version_path_from_list( + use_version, pype_versions) + if not version_path: + if use_version is not None: + print(("!!! Specified version was not found, using " + "latest available")) + # specified version was not found so use latest detected. + version_path = pype_version.path + print(f">>> Using version [ {pype_version} ]") + print(f" From {version_path}") + + # test if latest detected is installed (in user data dir) + is_inside = False + try: + is_inside = pype_version.path.resolve().relative_to( + bootstrap.data_dir) + except ValueError: + # if relative path cannot be calculated, Pype version is not + # inside user data dir + pass + + if not is_inside: + # install latest version to user data dir + version_path = bootstrap.install_version( + pype_version, force=True) + + if pype_version.path.is_file(): + print(">>> Extracting zip file ...") + version_path = bootstrap.extract_pype(pype_version) + + # inject version to Python environment (sys.path, ...) + print(">>> Injecting Pype version to running environment ...") + bootstrap.add_paths_from_directory(version_path) + + # add stuff from `/lib` to PYTHONPATH. + pythonpath = os.getenv("PYTHONPATH", "") + paths = pythonpath.split(os.pathsep) + frozen_libs = os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "lib")) + paths.append(frozen_libs) + os.environ["PYTHONPATH"] = os.pathsep.join(paths) + + # set PYPE_ROOT to point to currently used Pype version. + os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) + + return version_path + + +def boot(): + """Bootstrap Pype.""" + + # ------------------------------------------------------------------------ + # Play animation + # ------------------------------------------------------------------------ + + from pype.lib.terminal_splash import play_animation + + # don't play for silenced commands + if all(item not in sys.argv for item in silent_commands): + play_animation() + + # ------------------------------------------------------------------------ + # Process arguments + # ------------------------------------------------------------------------ + + use_version, use_staging = _process_arguments() + + # ------------------------------------------------------------------------ + # Determine mongodb connection + # ------------------------------------------------------------------------ + + pype_mongo = "" + try: + pype_mongo = _determine_mongodb() + except RuntimeError as e: + # without mongodb url we are done for. + print(f"!!! {e}") + sys.exit(1) + + os.environ["PYPE_MONGO"] = pype_mongo # ------------------------------------------------------------------------ # Load environments from database @@ -253,76 +382,15 @@ def boot(): # Find Pype versions # ------------------------------------------------------------------------ - pype_version = None - pype_versions = bootstrap.find_pype(include_zips=True) - try: - pype_version = pype_versions[-1] - except IndexError: - # no pype version found - pass - if getattr(sys, 'frozen', False): - if not pype_versions: - print('*** No Pype versions found.') - print("--- launching setup UI ...") - run(["igniter"]) - pype_versions = bootstrap.find_pype() - if not pype_versions: - print('!!! Still no Pype versions found.') - return - - # find only staging versions - if use_staging: - staging_versions = [v for v in pype_versions if v.is_staging()] - if not staging_versions: - print("!!! No staging versions detected.") - return - staging_versions.sort() - # get latest - pype_version = staging_versions[-1] - - # get path of version specified in `--use-version` - version_path = BootstrapRepos.get_version_path_from_list( - use_version, pype_versions) - if not version_path: - if use_version is not None: - print(("!!! Specified version was not found, using " - "latest available")) - # specified version was not found so use latest detected. - version_path = pype_version.path - print(f">>> Using version [ {pype_version} ]") - print(f" From {version_path}") - - # test if latest detected is installed (in user data dir) - is_inside = False + # find versions of Pype to be used with frozen code + version_path = None try: - is_inside = pype_version.path.resolve().relative_to( - bootstrap.data_dir) - except ValueError: - # if relative path cannot be calculated, Pype version is not - # inside user data dir - pass - - if not is_inside: - # install latest version to user data dir - version_path = bootstrap.install_version( - pype_version, force=True) - - if pype_version.path.is_file(): - print(">>> Extracting zip file ...") - version_path = bootstrap.extract_pype(pype_version) - - # inject version to Python environment (sys.path, ...) - print(">>> Injecting Pype version to running environment ...") - bootstrap.add_paths_from_directory(version_path) - - # add stuff from `/lib` to PYTHONPATH. - os.environ["PYTHONPATH"] += os.pathsep + os.path.normpath( - os.path.join(os.path.dirname(sys.executable), "lib") - ) - - # set PYPE_ROOT to point to currently used Pype version. - os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) + version_path = _find_frozen_pype(use_version, use_staging) + except RuntimeError as e: + # no version to run + print(f"!!! {e}") + sys.exit(1) else: # run through repos and add them to sys.path and PYTHONPATH # set root @@ -331,6 +399,7 @@ def boot(): # get current version of Pype local_version = bootstrap.get_local_live_version() if use_version and use_version != local_version: + pype_versions = bootstrap.find_pype(include_zips=True) version_path = BootstrapRepos.get_version_path_from_list( use_version, pype_versions) if version_path: @@ -350,6 +419,8 @@ def boot(): paths += repos os.environ["PYTHONPATH"] = os.pathsep.join(paths) + # set this to point either to `python` from venv in case of live code + # or to `pype` or `pype_console` in case of frozen code os.environ["PYPE_EXECUTABLE"] = sys.executable # DEPRECATED: remove when `pype-config` dissolves into Pype for good. From 7f7ca57e5c00a4602cdbfd75ba5c88347af64e82 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 14 Jan 2021 21:34:45 +0100 Subject: [PATCH 28/50] fix return value --- start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/start.py b/start.py index 87633afba5..f9934b68de 100644 --- a/start.py +++ b/start.py @@ -214,7 +214,8 @@ def _process_arguments() -> tuple: if "igniter" in sys.argv: import igniter igniter.run() - return use_version, use_staging + + return use_version, use_staging def _determine_mongodb() -> str: From b1d6036f244424a54c005022c9b875691d406c2c Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 14 Jan 2021 21:39:25 +0100 Subject: [PATCH 29/50] hound --- start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start.py b/start.py index f9934b68de..eb8bb7cd42 100644 --- a/start.py +++ b/start.py @@ -214,7 +214,7 @@ def _process_arguments() -> tuple: if "igniter" in sys.argv: import igniter igniter.run() - + return use_version, use_staging From 2d3bb6a4d9bbcac5eda78174d8c696602bea09cb Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 15 Jan 2021 11:12:31 +0100 Subject: [PATCH 30/50] Fix ftrack server imports --- pype/modules/ftrack/ftrack_server/sub_event_processor.py | 2 +- pype/modules/ftrack/ftrack_server/sub_event_status.py | 2 +- pype/modules/ftrack/ftrack_server/sub_event_storer.py | 2 +- pype/modules/ftrack/ftrack_server/sub_legacy_server.py | 2 +- pype/modules/ftrack/ftrack_server/sub_user_server.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/modules/ftrack/ftrack_server/sub_event_processor.py b/pype/modules/ftrack/ftrack_server/sub_event_processor.py index 77dc85b0a2..f48b2141e6 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_processor.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_processor.py @@ -4,7 +4,7 @@ import signal import socket import datetime -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, ProcessEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_event_status.py b/pype/modules/ftrack/ftrack_server/sub_event_status.py index a398b019eb..07b233282f 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_status.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_status.py @@ -7,7 +7,7 @@ import socket import datetime import ftrack_api -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, StatusEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_event_storer.py b/pype/modules/ftrack/ftrack_server/sub_event_storer.py index 3523e5701f..2fdd3b07f7 100644 --- a/pype/modules/ftrack/ftrack_server/sub_event_storer.py +++ b/pype/modules/ftrack/ftrack_server/sub_event_storer.py @@ -6,7 +6,7 @@ import socket import pymongo import ftrack_api -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, StorerEventHub, diff --git a/pype/modules/ftrack/ftrack_server/sub_legacy_server.py b/pype/modules/ftrack/ftrack_server/sub_legacy_server.py index ecb12830c4..e09bcbb699 100644 --- a/pype/modules/ftrack/ftrack_server/sub_legacy_server.py +++ b/pype/modules/ftrack/ftrack_server/sub_legacy_server.py @@ -4,10 +4,10 @@ import datetime import signal import threading -from ftrack_server import FtrackServer import ftrack_api from pype.api import Logger from pype.modules import ModulesManager +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer log = Logger().get_logger("Event Server Legacy") diff --git a/pype/modules/ftrack/ftrack_server/sub_user_server.py b/pype/modules/ftrack/ftrack_server/sub_user_server.py index 58d5982ac2..b968714faf 100644 --- a/pype/modules/ftrack/ftrack_server/sub_user_server.py +++ b/pype/modules/ftrack/ftrack_server/sub_user_server.py @@ -2,7 +2,7 @@ import sys import signal import socket -from ftrack_server import FtrackServer +from pype.modules.ftrack.ftrack_server.ftrack_server import FtrackServer from pype.modules.ftrack.ftrack_server.lib import ( SocketSession, SocketBaseEventHub From 8cda04861778f90bda9fc427ac1d33db7800c710 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 15 Jan 2021 20:44:37 +0100 Subject: [PATCH 31/50] SyncServer GUI - fix - project propagates into detail model Removed project from Widget, left project on model only --- pype/modules/sync_server/tray/app.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 8b11445157..59543b121f 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,6 +1,5 @@ from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt -from avalon import style from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.modules import ModulesManager @@ -134,7 +133,6 @@ class SyncRepresentationWidget(QtWidgets.QWidget): def __init__(self, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) - self.project = project self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -196,7 +194,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(_id, self.project) + detail_window = SyncServerDetailWindow(_id, + self.table_view.model()._project) detail_window.exec() def _on_context_menu(self, point): @@ -212,9 +211,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 19 REFRESH_SEC = 5000 DEFAULT_SORT = { - "context.asset": 1, - "context.subset": 1, - "context.version": 1, + "updated_dt_remote": -1, "_id": 1 } SORT_BY_COLUMN = [ @@ -740,7 +737,6 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.representation_id = _id self.item = None # set to item that mouse was clicked over - self.project = project manager = ModulesManager() self.sync_server = manager.modules_by_name["sync_server"] @@ -854,17 +850,19 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def _reset_local_site(self): log.info("reset local site: {}".format(self.item._id)) - self.sync_server.reset_provider_for_file(self.project, - self.representation_id, - self.item._id, - 'studio') # TEMP + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'local') def _reset_remote_site(self): log.info("reset remote site: {}".format(self.item._id)) - self.sync_server.reset_provider_for_file(self.project, - self.representation_id, - self.item._id, - 'gdrive') # TEMP + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + self.item._id, + 'remote') class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): @@ -934,7 +932,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.projection = self.get_default_projection() self.query = self.get_default_query() - log.debug("!!! init query: {}".format(self.query)) + import bson.json_util + # log.debug("detail init query:: {}".format( + # bson.json_util.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) From 2f3a1eb9f0bd6797844be12b6d0be881bbc2207a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 15 Jan 2021 20:47:37 +0100 Subject: [PATCH 32/50] SyncServer GUI - implemented saving progress to DB --- pype/modules/sync_server/providers/gdrive.py | 98 ++++++++++++---- pype/modules/sync_server/sync_server.py | 114 ++++++++++++++----- 2 files changed, 163 insertions(+), 49 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 2207fdf3a3..44810b81d1 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -8,6 +8,7 @@ from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload from pype.api import Logger from pype.api import get_system_settings from ..utils import time_function +import time SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly', 'https://www.googleapis.com/auth/drive.file', @@ -42,6 +43,7 @@ class GDriveHandler(AbstractProvider): """ FOLDER_STR = 'application/vnd.google-apps.folder' MY_DRIVE_STR = 'My Drive' # name of root folder of regular Google drive + CHUNK_SIZE = 2097152 # must be divisible by 256! def __init__(self, site_name, tree=None, presets=None): self.presets = None @@ -277,7 +279,9 @@ class GDriveHandler(AbstractProvider): path = new_path_key return folder_id - def upload_file(self, source_path, path, overwrite=False): + def upload_file(self, source_path, path, + server, collection, file, representation, site, + overwrite=False): """ Uploads single file from 'source_path' to destination 'path'. It creates all folders on the path if are not existing. @@ -287,6 +291,13 @@ class GDriveHandler(AbstractProvider): path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -302,8 +313,8 @@ class GDriveHandler(AbstractProvider): path = os.path.dirname(path) else: target_name = os.path.basename(source_path) - file = self.file_path_exists(path + "/" + target_name) - if file and not overwrite: + target_file = self.file_path_exists(path + "/" + target_name) + if target_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -316,23 +327,45 @@ class GDriveHandler(AbstractProvider): } media = MediaFileUpload(source_path, mimetype='application/octet-stream', + chunksize=self.CHUNK_SIZE, resumable=True) try: - if not file: + if not target_file: # update doesnt like parent file_metadata['parents'] = [folder_id] - file = self.service.files().create(body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().create(body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') else: - file = self.service.files().update(fileId=file["id"], - body=file_metadata, - supportsAllDrives=True, - media_body=media, - fields='id').execute() + request = self.service.files().update(fileId=target_file["id"], + body=file_metadata, + supportsAllDrives=True, + media_body=media, + fields='id') + + media.stream() + log.debug("Start Upload! {}".format(source_path)) + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Uploaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = request.next_chunk() except errors.HttpError as ex: if ex.resp['status'] == '404': @@ -344,13 +377,14 @@ class GDriveHandler(AbstractProvider): log.warning("Forbidden received, hit quota. " "Injecting 60s delay.") - import time time.sleep(60) return False raise - return file["id"] + return response['id'] - def download_file(self, source_path, local_path, overwrite=False): + def download_file(self, source_path, local_path, + server, collection, file, representation, site, + overwrite=False): """ Downloads single file from 'source_path' (remote) to 'local_path'. It creates all folders on the local_path if are not existing. @@ -361,6 +395,13 @@ class GDriveHandler(AbstractProvider): local_path (string): absolute path with or without name of the file overwrite (boolean): replace existing file + arguments for saving progress: + server (SyncServer): server instance to call update_db on + collection (str): name of collection + file (dict): info about uploaded file (matches structure from db) + representation (dict): complete repre containing 'file' + site (str): site name + Returns: (string) file_id of created/modified file , throws FileExistsError, FileNotFoundError exceptions @@ -378,9 +419,9 @@ class GDriveHandler(AbstractProvider): else: # just folder, get file name from source target_name = os.path.basename(source_path) - file = os.path.isfile(local_path + "/" + target_name) + local_file = os.path.isfile(local_path + "/" + target_name) - if file and not overwrite: + if local_file and not overwrite: raise FileExistsError("File already exists, " "use 'overwrite' argument") @@ -389,9 +430,24 @@ class GDriveHandler(AbstractProvider): with open(local_path + "/" + target_name, "wb") as fh: downloader = MediaIoBaseDownload(fh, request) - done = False - while done is False: - status, done = downloader.next_chunk() + last_tick = status = response = None + status_val = 0 + while response is None: + if status: + status_val = float(status.progress()) + if not last_tick or \ + time.time() - last_tick >= server.LOG_PROGRESS_SEC: + last_tick = time.time() + log.debug("Downloaded %d%%." % + int(status_val * 100)) + server.update_db(collection=collection, + new_file_id=None, + file=file, + representation=representation, + site=site, + progress=status_val + ) + status, response = downloader.next_chunk() return target_name diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 415f1e7d15..49a572877c 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -92,6 +92,7 @@ class SyncServer(PypeModule, ITrayModule): # set 0 to no limit REPRESENTATION_LIMIT = 100 DEFAULT_SITE = 'studio' + LOG_PROGRESS_SEC = 5 # how often log progress to DB name = "sync_server" label = "Sync Server" @@ -139,13 +140,11 @@ class SyncServer(PypeModule, ITrayModule): try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) - self.sync_server_thread = SyncServerThread(self) - from .tray.app import SyncServerWindow self.widget = SyncServerWindow() except ValueError: - log.info("No system setting for sync. Not syncing.") + log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False except KeyError: log.info(( @@ -266,7 +265,8 @@ class SyncServer(PypeModule, ITrayModule): settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: - log.info("No project setting for Sync Server, not syncing.") + log.info("No project setting for {}, not syncing.". + format(project_name)) return {} if sync_settings.get("enabled"): return sync_settings @@ -427,8 +427,8 @@ class SyncServer(PypeModule, ITrayModule): return SyncStatus.DO_NOTHING - async def upload(self, file, representation, provider_name, site_name, - tree=None, preset=None): + async def upload(self, collection, file, representation, provider_name, + site_name, tree=None, preset=None): """ Upload single 'file' of a 'representation' to 'provider'. Source url is taken from 'file' portion, where {root} placeholder @@ -439,6 +439,7 @@ class SyncServer(PypeModule, ITrayModule): from GDrive), 'created_dt' - time of upload Args: + collection (str): source collection file (dictionary): of file from representation in Mongo representation (dictionary): of representation provider_name (string): gdrive, gdc etc. @@ -468,21 +469,28 @@ class SyncServer(PypeModule, ITrayModule): err = "Folder {} wasn't created. Check permissions.".\ format(target_folder) raise NotADirectoryError(err) - + _, remote_site = self.get_sites_for_project(collection) loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.upload_file, local_file, remote_file, - True) + self, + collection, + file, + representation, + remote_site, + True + ) return file_id - async def download(self, file, representation, provider_name, + async def download(self, collection, file, representation, provider_name, site_name, tree=None, preset=None): """ Downloads file to local folder denoted in representation.Context. Args: + collection (str): source collection file (dictionary) : info about processed file representation (dictionary): repr that 'file' belongs to provider_name (string): 'gdrive' etc @@ -506,26 +514,37 @@ class SyncServer(PypeModule, ITrayModule): local_folder = os.path.dirname(local_file) os.makedirs(local_folder, exist_ok=True) + local_site, _ = self.get_sites_for_project(collection) + loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.download_file, remote_file, local_file, - False) + False, + self, + collection, + file, + representation, + local_site + ) return file_id - def update_db(self, new_file_id, file, representation, provider_name, - error=None): + def update_db(self, collection, new_file_id, file, representation, + site, error=None, progress=None): """ Update 'provider' portion of records in DB with success (file_id) or error (exception) Args: + collection (string): name of project - force to db connection as + each file might come from different collection new_file_id (string): file (dictionary): info about processed file (pulled from DB) representation (dictionary): parent repr of file (from DB) - provider_name (string): label ('gdrive', 'S3') + site (string): label ('gdrive', 'S3') error (string): exception message + progress (float): 0-1 of progress of upload/download Returns: None @@ -539,26 +558,33 @@ class SyncServer(PypeModule, ITrayModule): file_index, _ = self._get_file_info(representation.get('files', []), file_id) site_index, _ = self._get_provider_rec(file.get('sites', []), - provider_name) + site) update = {} if new_file_id: update["$set"] = self._get_success_dict(file_index, site_index, new_file_id) # reset previous errors if any update["$unset"] = self._get_error_dict(file_index, site_index, - "", "") + "", "", "") + elif progress is not None: + update["$set"] = self._get_progress_dict(file_index, site_index, + progress) else: - tries = self._get_tries_count(file, provider_name) + tries = self._get_tries_count(file, site) tries += 1 update["$set"] = self._get_error_dict(file_index, site_index, error, tries) + self.connection.Session["AVALON_PROJECT"] = collection self.connection.update_one( query, update ) + if progress is not None: + return + status = 'failed' error_str = 'with error {}'.format(error) if new_file_id: @@ -574,7 +600,7 @@ class SyncServer(PypeModule, ITrayModule): def _get_file_info(self, files, _id): """ Return record from list of records which name matches to 'provider' - Could be possibly refactored with '_get_file_info' together. + Could be possibly refactored with '_get_provider_rec' together. Args: files (list): of dictionaries with info about published files @@ -611,7 +637,7 @@ class SyncServer(PypeModule, ITrayModule): return -1, None def reset_provider_for_file(self, collection, representation_id, - file_id, site_name): + file_id, side): """ Reset information about synchronization for particular 'file_id' and provider. @@ -620,7 +646,7 @@ class SyncServer(PypeModule, ITrayModule): collection (string): name of project (eg. collection) in DB representation_id(string): _id of representation file_id (string): file _id in representation - site_name (string): 'gdrive', 'S3' etc + side (string): local or remote side Returns: None """ @@ -634,6 +660,12 @@ class SyncServer(PypeModule, ITrayModule): raise ValueError("Representation {} not found in {}". format(representation_id, collection)) + local_site, remote_site = self.get_active_sites(collection) + if side == 'local': + site_name = local_site + else: + site_name = remote_site + files = representation[0].get('files', []) file_index, _ = self._get_file_info(files, file_id) @@ -685,7 +717,8 @@ class SyncServer(PypeModule, ITrayModule): datetime.utcnow()} return val - def _get_error_dict(self, file_index, site_index, error="", tries=""): + def _get_error_dict(self, file_index, site_index, + error="", tries="", progress=""): """ Provide error metadata to be stored in Db. Used for set (error and tries provided) or unset mode. @@ -700,7 +733,9 @@ class SyncServer(PypeModule, ITrayModule): val = {"files.{}.sites.{}.last_failed_dt". format(file_index, site_index): datetime.utcnow(), "files.{}.sites.{}.error".format(file_index, site_index): error, - "files.{}.sites.{}.tries".format(file_index, site_index): tries + "files.{}.sites.{}.tries".format(file_index, site_index): tries, + "files.{}.sites.{}.progress".format(file_index, site_index): + progress } return val @@ -728,6 +763,22 @@ class SyncServer(PypeModule, ITrayModule): _, rec = self._get_provider_rec(file.get("sites", []), provider) return rec.get("tries", 0) + def _get_progress_dict(self, file_index, site_index, progress): + """ + Provide progress metadata to be stored in Db. + Used during upload/download for GUI to show. + Args: + file_index: (int) - index of modified file + site_index: (int) - index of modified site of modified file + progress: (float) - 0-1 progress of upload/download + Returns: + (dictionary) + """ + val = {"files.{}.sites.{}.progress". + format(file_index, site_index): progress + } + return val + def _get_local_file_path(self, file, local_root): """ Auxiliary function for replacing rootless path with real path @@ -873,23 +924,27 @@ class SyncServerThread(threading.Thread): tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.upload(file, + self.module.upload(collection, + file, sync, provider, site, tree, site_preset)) task_files_to_process.append(task) - # store info for exception handling + # store info for exception handlingy files_processed_info.append((file, sync, - site)) + site, + collection + )) processed_file_path.add(file_path) if status == SyncStatus.DO_DOWNLOAD: tree = handler.get_tree() limit -= 1 task = asyncio.create_task( - self.module.download(file, + self.module.download(collection, + file, sync, provider, site, @@ -899,7 +954,9 @@ class SyncServerThread(threading.Thread): files_processed_info.append((file, sync, - local)) + local, + collection + )) processed_file_path.add(file_path) log.debug("Sync tasks count {}". @@ -909,12 +966,13 @@ class SyncServerThread(threading.Thread): return_exceptions=True) for file_id, info in zip(files_created, files_processed_info): - file, representation, site = info + file, representation, site, collection = info error = None if isinstance(file_id, BaseException): error = str(file_id) file_id = None - self.module.update_db(file_id, + self.module.update_db(collection, + file_id, file, representation, site, From 10686e06c5a411bc616afa83194b8acccb0e743c Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 15 Jan 2021 22:42:16 +0100 Subject: [PATCH 33/50] change version comparsion, handling of venv and startup --- igniter/bootstrap_repos.py | 97 ++++-- igniter/splash.txt | 413 +++++++++++++++++++++++ igniter/terminal_splash.py | 43 +++ igniter/tools.py | 145 ++++++-- igniter/user_settings.py | 466 ++++++++++++++++++++++++++ pype/lib/terminal_splash.py | 11 +- pype/lib/user_settings.py | 11 +- setup.py | 3 +- start.py | 59 ++-- tests/igniter/test_bootstrap_repos.py | 35 +- tools/build.ps1 | 2 + tools/build_dependencies.py | 119 +++++++ tools/create_zip.ps1 | 5 +- 13 files changed, 1295 insertions(+), 114 deletions(-) create mode 100644 igniter/splash.txt create mode 100644 igniter/terminal_splash.py create mode 100644 igniter/user_settings.py create mode 100644 tools/build_dependencies.py diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index b485054920..38de3007b4 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -14,8 +14,7 @@ from zipfile import ZipFile, BadZipFile from appdirs import user_data_dir from speedcopy import copyfile -from pype.lib import PypeSettingsRegistry -from pype.version import __version__ +from .user_settings import PypeSettingsRegistry from .tools import load_environments @@ -24,23 +23,23 @@ class PypeVersion: """Class for storing information about Pype version. Attributes: - major (int): [1].2.3-variant-client - minor (int): 1.[2].3-variant-client - subversion (int): 1.2.[3]-variant-client - variant (str): 1.2.3-[variant]-client - client (str): 1.2.3-variant-[client] + major (int): [1].2.3-client-variant + minor (int): 1.[2].3-client-variant + subversion (int): 1.2.[3]-client-variant + client (str): 1.2.3-[client]-variant + variant (str): 1.2.3-client-[variant] path (str): path to Pype """ major = 0 minor = 0 subversion = 0 - variant = "production" + variant = "" client = None path = None _version_regex = re.compile( - r"(?P\d+)\.(?P\d+)\.(?P\d+)(-?((?Pstaging)|(?P.+))(-(?P.+))?)?") # noqa: E501 + r"(?P\d+)\.(?P\d+)\.(?P\d+)(-(?Pstaging)|-(?P.+)(-(?Pstaging)))?") # noqa: E501 @property def version(self): @@ -58,12 +57,12 @@ class PypeVersion: def __init__(self, major: int = None, minor: int = None, subversion: int = None, version: str = None, - variant: str = "production", client: str = None, + variant: str = "", client: str = None, path: Path = None): self.path = path if ( - major is None or minor is None or subversion is None + major is None or minor is None or subversion is None ) and version is None: raise ValueError("Need version specified in some way.") if version: @@ -85,12 +84,13 @@ class PypeVersion: def _compose_version(self): version = "{}.{}.{}".format(self.major, self.minor, self.subversion) - if self.variant == "staging": - version = "{}-{}".format(version, self.variant) if self.client: version = "{}-{}".format(version, self.client) + if self.variant == "staging": + version = "{}-{}".format(version, self.variant) + return version @classmethod @@ -101,10 +101,10 @@ class PypeVersion: "Cannot parse version string: {}".format(version_string)) variant = None - if m.group("variant") == "staging": + if m.group("var1") == "staging" or m.group("var2") == "staging": variant = "staging" - client = m.group("client") or m.group("cli") + client = m.group("client") return (int(m.group("major")), int(m.group("minor")), int(m.group("sub")), variant, client) @@ -125,26 +125,48 @@ class PypeVersion: return hash(self.version) def __lt__(self, other): - if self.major < other.major: + if (self.major, self.minor, self.subversion) < \ + (other.major, other.minor, other.subversion): return True - if self.major <= other.major and self.minor < other.minor: - return True - if self.major <= other.major and self.minor <= other.minor and \ - self.subversion < other.subversion: + # 1.2.3-staging < 1.2.3-client-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and self.variant and \ + other.client and other.variant: return True - # Directory takes precedence over file - if ( - self.path - and other.path - and other.path.is_dir() - and self.path.is_file() - ): + # 1.2.3 < 1.2.3-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and self.variant and \ + not other.client and not other.variant: return True - return self.major == other.major and self.minor == other.minor and \ - self.subversion == other.subversion and self.variant == "staging" + # 1.2.3 < 1.2.3-client + if self.get_main_version() == other.get_main_version() and \ + not self.client and not self.variant and \ + other.client and not other.variant: + return True + + # 1.2.3 < 1.2.3-client-staging + if self.get_main_version() == other.get_main_version() and \ + not self.client and not self.variant and other.client: + return True + + # 1.2.3-client-staging < 1.2.3-client + if self.get_main_version() == other.get_main_version() and \ + self.client and self.variant and \ + other.client and not other.variant: + return True + + # prefer path over no path + if self.version == other.version and \ + not self.path and other.path: + return True + + # prefer path with dir over path with file + return self.version == other.version and self.path and \ + other.path and self.path.is_file() and \ + other.path.is_dir() def is_staging(self) -> bool: """Test if current version is staging one.""" @@ -252,7 +274,12 @@ class BootstrapRepos: @staticmethod def get_local_live_version() -> str: """Get version of local Pype.""" - return __version__ + + version = {} + path = Path(os.path.dirname(__file__)).parent / "pype" / "version.py" + with open(path, "r") as fp: + exec(fp.read(), version) + return version["__version__"] @staticmethod def get_version(repo_dir: Path) -> Union[str, None]: @@ -494,6 +521,7 @@ class BootstrapRepos: def find_pype( self, pype_path: Path = None, + staging: bool = False, include_zips: bool = False) -> Union[List[PypeVersion], None]: """Get ordered dict of detected Pype version. @@ -505,6 +533,8 @@ class BootstrapRepos: Args: pype_path (Path, optional): Try to find Pype on the given path. + staging (bool, optional): Filter only staging version, skip them + otherwise. include_zips (bool, optional): If set True it will try to find Pype in zip files in given directory. @@ -549,6 +579,7 @@ class BootstrapRepos: result = PypeVersion.version_in_str(name) if result[0]: + detected_version: PypeVersion detected_version = result[1] if file.is_dir(): @@ -614,7 +645,11 @@ class BootstrapRepos: continue detected_version.path = file - _pype_versions.append(detected_version) + if staging and detected_version.is_staging(): + _pype_versions.append(detected_version) + + if not staging and not detected_version.is_staging(): + _pype_versions.append(detected_version) return sorted(_pype_versions) diff --git a/igniter/splash.txt b/igniter/splash.txt new file mode 100644 index 0000000000..833bcd4b9c --- /dev/null +++ b/igniter/splash.txt @@ -0,0 +1,413 @@ + + + + * + + + + + + + .* + + + + + + * + .* + * + + + + . + * + .* + * + . + + . + * + .* + .* + .* + * + . + . + * + .* + .* + .* + * + . + _. + /** + \ * + \* + * + * + . + __. + ---* + \ \* + \ * + \* + * + . + \___. + /* * + \ \ * + \ \* + \ * + \* + . + |____. + /* * + \|\ * + \ \ * + \ \ * + \ \* + \/. + _/_____. + /* * + / \ * + \ \ * + \ \ * + \ \__* + \/__. + __________. + --*-- ___* + \ \ \/_* + \ \ __* + \ \ \_* + \ \____\* + \/____/. + \____________ . + /* ___ \* + \ \ \/_\ * + \ \ _____* + \ \ \___/* + \ \____\ * + \/____/ . + |___________ . + /* ___ \ * + \|\ \/_\ \ * + \ \ _____/ * + \ \ \___/ * + \ \____\ / * + \/____/ \. + _/__________ . + /* ___ \ * + / \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---* + \ \____\ / \__* + \/____/ \/__. + ____________ . + --*-- ___ \ * + \ \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\* + \/____/ \/____/. + ____________ + /\ ___ \ . + \ \ \/_\ \ * + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ . + \ \ _____/ * + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ . + \ \ \___/ ---- * + \ \____\ / \____\ . + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ + \ \ \___/ ---- * + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ + \ \ \___/ ---- . + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ _ + \ \ \___/ ---- + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ \ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ __\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ + \ \ \___/ ---- \ \ + \ \____\ / \____\ \__\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___. + \ \ \___/ ---- \ \\ + \ \____\ / \____\ \__\, + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ . + \ \ \___/ ---- \ \\ + \ \____\ / \____\ \__\\, + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ _. + \ \ \___/ ---- \ \\\ + \ \____\ / \____\ \__\\\ + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ __. + \ \ \___/ ---- \ \\ \ + \ \____\ / \____\ \__\\_/. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___. + \ \ \___/ ---- \ \\ \\ + \ \____\ / \____\ \__\\__\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ . + \ \ \___/ ---- \ \\ \\ + \ \____\ / \____\ \__\\__\\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ _. + \ \ \___/ ---- \ \\ \\\ + \ \____\ / \____\ \__\\__\\. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ __. + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\_. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ __. + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__. + \/____/ \/____/ + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ * + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ O* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ ..oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . p.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . Py.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYp.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPe.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE c.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE C1.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE ClU.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE CluB.oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club .oO* + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . .. + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . .. + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . . + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ___ ___ ___ + \ \ \___/ ---- \ \\ \\ \ + \ \____\ / \____\ \__\\__\\__\ + \/____/ \/____/ . PYPE Club . diff --git a/igniter/terminal_splash.py b/igniter/terminal_splash.py new file mode 100644 index 0000000000..1a7645571e --- /dev/null +++ b/igniter/terminal_splash.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""Pype terminal animation.""" +import blessed +from pathlib import Path +from time import sleep + +NO_TERMINAL = False + +try: + term = blessed.Terminal() +except AttributeError: + # this happens when blessed cannot find proper terminal. + # If so, skip printing ascii art animation. + NO_TERMINAL = True + + +def play_animation(): + """Play ASCII art Pype animation.""" + if NO_TERMINAL: + return + print(term.home + term.clear) + frame_size = 7 + splash_file = Path(__file__).parent / "splash.txt" + with splash_file.open("r") as sf: + animation = sf.readlines() + + animation_length = int(len(animation) / frame_size) + current_frame = 0 + for _ in range(animation_length): + frame = "".join( + scanline + for y, scanline in enumerate( + animation[current_frame : current_frame + frame_size] + ) + ) + + with term.location(0, 0): + # term.aquamarine3_bold(frame) + print(f"{term.bold}{term.aquamarine3}{frame}{term.normal}") + + sleep(0.02) + current_frame += frame_size + print(term.move_y(7)) diff --git a/igniter/tools.py b/igniter/tools.py index e7cb99865d..d9a315834a 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -1,13 +1,104 @@ # -*- coding: utf-8 -*- -"""Tools used in **Igniter** GUI.""" +"""Tools used in **Igniter** GUI. + +Functions ``compose_url()`` and ``decompose_url()`` are the same as in +``pype.lib`` and they are here to avoid importing pype module before its +version is decided. + +""" + import os import uuid -from urllib.parse import urlparse +from typing import Dict +from urllib.parse import urlparse, parse_qs from pymongo import MongoClient from pymongo.errors import ServerSelectionTimeoutError, InvalidURI -from pype.lib import decompose_url, compose_url + +def decompose_url(url: str) -> Dict: + """Decompose mongodb url to its separate components. + + Args: + url (str): Mongodb url. + + Returns: + dict: Dictionary of components. + + """ + components = { + "scheme": None, + "host": None, + "port": None, + "username": None, + "password": None, + "auth_db": None + } + + result = urlparse(url) + if result.scheme is None: + _url = "mongodb://{}".format(url) + result = urlparse(_url) + + components["scheme"] = result.scheme + components["host"] = result.hostname + try: + components["port"] = result.port + except ValueError: + raise RuntimeError("invalid port specified") + components["username"] = result.username + components["password"] = result.password + + try: + components["auth_db"] = parse_qs(result.query)['authSource'][0] + except KeyError: + # no auth db provided, mongo will use the one we are connecting to + pass + + return components + + +def compose_url(scheme: str = None, + host: str = None, + username: str = None, + password: str = None, + port: int = None, + auth_db: str = None) -> str: + """Compose mongodb url from its individual components. + + Args: + scheme (str, optional): + host (str, optional): + username (str, optional): + password (str, optional): + port (str, optional): + auth_db (str, optional): + + Returns: + str: mongodb url + + """ + + url = "{scheme}://" + + if username and password: + url += "{username}:{password}@" + + url += "{host}" + if port: + url += ":{port}" + + if auth_db: + url += "?authSource={auth_db}" + + return url.format(**{ + "scheme": scheme, + "host": host, + "username": username, + "password": password, + "port": port, + "auth_db": auth_db + }) def validate_mongo_connection(cnx: str) -> (bool, str): @@ -21,30 +112,29 @@ def validate_mongo_connection(cnx: str) -> (bool, str): """ parsed = urlparse(cnx) - if parsed.scheme in ["mongodb", "mongodb+srv"]: - # we have mongo connection string. Let's try if we can connect. - components = decompose_url(cnx) - mongo_args = { - "host": compose_url(**components), - "serverSelectionTimeoutMS": 1000 - } - port = components.get("port") - if port is not None: - mongo_args["port"] = int(port) - - try: - client = MongoClient(**mongo_args) - client.server_info() - except ServerSelectionTimeoutError as e: - return False, f"Cannot connect to server {cnx} - {e}" - except ValueError: - return False, f"Invalid port specified {parsed.port}" - except InvalidURI as e: - return False, str(e) - else: - return True, "Connection is successful" - else: + if parsed.scheme not in ["mongodb", "mongodb+srv"]: return False, "Not mongodb schema" + # we have mongo connection string. Let's try if we can connect. + components = decompose_url(cnx) + mongo_args = { + "host": compose_url(**components), + "serverSelectionTimeoutMS": 1000 + } + port = components.get("port") + if port is not None: + mongo_args["port"] = int(port) + + try: + client = MongoClient(**mongo_args) + client.server_info() + except ServerSelectionTimeoutError as e: + return False, f"Cannot connect to server {cnx} - {e}" + except ValueError: + return False, f"Invalid port specified {parsed.port}" + except InvalidURI as e: + return False, str(e) + else: + return True, "Connection is successful" def validate_path_string(path: str) -> (bool, str): @@ -109,5 +199,4 @@ def load_environments(sections: list = None) -> dict: continue merged_env = acre.append(merged_env, parsed_env) - env = acre.compute(merged_env, cleanup=True) - return env + return acre.compute(merged_env, cleanup=True) diff --git a/igniter/user_settings.py b/igniter/user_settings.py new file mode 100644 index 0000000000..00ce68cb0b --- /dev/null +++ b/igniter/user_settings.py @@ -0,0 +1,466 @@ +# -*- coding: utf-8 -*- +"""Package to deal with saving and retrieving user specific settings.""" +import os +from datetime import datetime +from abc import ABCMeta, abstractmethod +import json + +# disable lru cache in Python 2 +try: + from functools import lru_cache +except ImportError: + def lru_cache(maxsize): + def max_size(func): + def wrapper(*args, **kwargs): + value = func(*args, **kwargs) + return value + return wrapper + return max_size + +# ConfigParser was renamed in python3 to configparser +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import platform + +import appdirs +import six + + +@six.add_metaclass(ABCMeta) +class ASettingRegistry(): + """Abstract class defining structure of **SettingRegistry** class. + + It is implementing methods to store secure items into keyring, otherwise + mechanism for storing common items must be implemented in abstract + methods. + + Attributes: + _name (str): Registry names. + + """ + + def __init__(self, name): + # type: (str) -> ASettingRegistry + super(ASettingRegistry, self).__init__() + + if six.PY3: + import keyring + # hack for cx_freeze and Windows keyring backend + if platform.system() == "Windows": + from keyring.backends import Windows + keyring.set_keyring(Windows.WinVaultKeyring()) + + self._name = name + self._items = {} + + def set_item(self, name, value): + # type: (str, str) -> None + """Set item to settings registry. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + """ + self._set_item(name, value) + + @abstractmethod + def _set_item(self, name, value): + # type: (str, str) -> None + # Implement it + pass + + def __setitem__(self, name, value): + self._items[name] = value + self._set_item(name, value) + + def get_item(self, name): + # type: (str) -> str + """Get item from settings registry. + + Args: + name (str): Name of the item. + + Returns: + value (str): Value of the item. + + Raises: + ValueError: If item doesn't exist. + + """ + return self._get_item(name) + + @abstractmethod + def _get_item(self, name): + # type: (str) -> str + # Implement it + pass + + def __getitem__(self, name): + return self._get_item(name) + + def delete_item(self, name): + # type: (str) -> None + """Delete item from settings registry. + + Args: + name (str): Name of the item. + + """ + self._delete_item(name) + + @abstractmethod + def _delete_item(self, name): + # type: (str) -> None + """Delete item from settings. + + Note: + see :meth:`pype.lib.user_settings.ARegistrySettings.delete_item` + + """ + pass + + def __delitem__(self, name): + del self._items[name] + self._delete_item(name) + + def set_secure_item(self, name, value): + # type: (str, str) -> None + """Set sensitive item into system's keyring. + + This uses `Keyring module`_ to save sensitive stuff into system's + keyring. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + keyring.set_password(self._name, name, value) + + @lru_cache(maxsize=32) + def get_secure_item(self, name): + # type: (str) -> str + """Get value of sensitive item from system's keyring. + + See also `Keyring module`_ + + Args: + name (str): Name of the item. + + Returns: + value (str): Value of the item. + + Raises: + ValueError: If item doesn't exist. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + value = keyring.get_password(self._name, name) + if not value: + raise ValueError( + "Item {}:{} does not exist in keyring.".format( + self._name, name)) + return value + + def delete_secure_item(self, name): + # type: (str) -> None + """Delete value stored in system's keyring. + + See also `Keyring module`_ + + Args: + name (str): Name of the item to be deleted. + + .. _Keyring module: + https://github.com/jaraco/keyring + + """ + if six.PY2: + raise NotImplementedError( + "Keyring not available on Python 2 hosts") + import keyring + self.get_secure_item.cache_clear() + keyring.delete_password(self._name, name) + + +class IniSettingRegistry(ASettingRegistry): + """Class using :mod:`configparser`. + + This class is using :mod:`configparser` (ini) files to store items. + + """ + + def __init__(self, name, path): + # type: (str, str) -> IniSettingRegistry + super(IniSettingRegistry, self).__init__(name) + # get registry file + version = os.getenv("PYPE_VERSION", "N/A") + self._registry_file = os.path.join(path, "{}.ini".format(name)) + if not os.path.exists(self._registry_file): + with open(self._registry_file, mode="w") as cfg: + print("# Settings registry", cfg) + print("# Generated by Pype {}".format(version), cfg) + now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + print("# {}".format(now), cfg) + + def set_item_section( + self, section, name, value): + # type: (str, str, str) -> None + """Set item to specific section of ini registry. + + If section doesn't exists, it is created. + + Args: + section (str): Name of section. + name (str): Name of the item. + value (str): Value of the item. + + """ + value = str(value) + config = configparser.ConfigParser() + + config.read(self._registry_file) + if not config.has_section(section): + config.add_section(section) + current = config[section] + current[name] = value + + with open(self._registry_file, mode="w") as cfg: + config.write(cfg) + + def _set_item(self, name, value): + # type: (str, str) -> None + self.set_item_section("MAIN", name, value) + + def set_item(self, name, value): + # type: (str, str) -> None + """Set item to settings ini file. + + This saves item to ``DEFAULT`` section of ini as each item there + must reside in some section. + + Args: + name (str): Name of the item. + value (str): Value of the item. + + """ + # this does the some, overridden just for different docstring. + # we cast value to str as ini options values must be strings. + super(IniSettingRegistry, self).set_item(name, str(value)) + + def get_item(self, name): + # type: (str) -> str + """Gets item from settings ini file. + + This gets settings from ``DEFAULT`` section of ini file as each item + there must reside in some section. + + Args: + name (str): Name of the item. + + Returns: + str: Value of item. + + Raises: + ValueError: If value doesn't exist. + + """ + return super(IniSettingRegistry, self).get_item(name) + + @lru_cache(maxsize=32) + def get_item_from_section(self, section, name): + # type: (str, str) -> str + """Get item from section of ini file. + + This will read ini file and try to get item value from specified + section. If that section or item doesn't exist, :exc:`ValueError` + is risen. + + Args: + section (str): Name of ini section. + name (str): Name of the item. + + Returns: + str: Item value. + + Raises: + ValueError: If value doesn't exist. + + """ + config = configparser.ConfigParser() + config.read(self._registry_file) + try: + value = config[section][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}:{}".format(section, name)) + return value + + def _get_item(self, name): + # type: (str) -> str + return self.get_item_from_section("MAIN", name) + + def delete_item_from_section(self, section, name): + # type: (str, str) -> None + """Delete item from section in ini file. + + Args: + section (str): Section name. + name (str): Name of the item. + + Raises: + ValueError: If item doesn't exist. + + """ + self.get_item_from_section.cache_clear() + config = configparser.ConfigParser() + config.read(self._registry_file) + try: + _ = config[section][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}:{}".format(section, name)) + config.remove_option(section, name) + + # if section is empty, delete it + if len(config[section].keys()) == 0: + config.remove_section(section) + + with open(self._registry_file, mode="w") as cfg: + config.write(cfg) + + def _delete_item(self, name): + """Delete item from default section. + + Note: + See :meth:`~pype.lib.IniSettingsRegistry.delete_item_from_section` + + """ + self.delete_item_from_section("MAIN", name) + + +class JSONSettingRegistry(ASettingRegistry): + """Class using json file as storage.""" + + def __init__(self, name, path): + # type: (str, str) -> JSONSettingRegistry + super(JSONSettingRegistry, self).__init__(name) + #: str: name of registry file + self._registry_file = os.path.join(path, "{}.json".format(name)) + now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + header = { + "__metadata__": { + "pype-version": os.getenv("PYPE_VERSION", "N/A"), + "generated": now + }, + "registry": {} + } + + if not os.path.exists(os.path.dirname(self._registry_file)): + os.makedirs(os.path.dirname(self._registry_file), exist_ok=True) + if not os.path.exists(self._registry_file): + with open(self._registry_file, mode="w") as cfg: + json.dump(header, cfg, indent=4) + + @lru_cache(maxsize=32) + def _get_item(self, name): + # type: (str) -> object + """Get item value from registry json. + + Note: + See :meth:`pype.lib.JSONSettingRegistry.get_item` + + """ + with open(self._registry_file, mode="r") as cfg: + data = json.load(cfg) + try: + value = data["registry"][name] + except KeyError: + raise ValueError( + "Registry doesn't contain value {}".format(name)) + return value + + def get_item(self, name): + # type: (str) -> object + """Get item value from registry json. + + Args: + name (str): Name of the item. + + Returns: + value of the item + + Raises: + ValueError: If item is not found in registry file. + + """ + return self._get_item(name) + + def _set_item(self, name, value): + # type: (str, object) -> None + """Set item value to registry json. + + Note: + See :meth:`pype.lib.JSONSettingRegistry.set_item` + + """ + with open(self._registry_file, "r+") as cfg: + data = json.load(cfg) + data["registry"][name] = value + cfg.truncate(0) + cfg.seek(0) + json.dump(data, cfg, indent=4) + + def set_item(self, name, value): + # type: (str, object) -> None + """Set item and its value into json registry file. + + Args: + name (str): name of the item. + value (Any): value of the item. + + """ + self._set_item(name, value) + + def _delete_item(self, name): + # type: (str) -> None + self._get_item.cache_clear() + with open(self._registry_file, "r+") as cfg: + data = json.load(cfg) + del data["registry"][name] + cfg.truncate(0) + cfg.seek(0) + json.dump(data, cfg, indent=4) + + +class PypeSettingsRegistry(JSONSettingRegistry): + """Class handling Pype general settings registry. + + Attributes: + vendor (str): Name used for path construction. + product (str): Additional name used for path construction. + + """ + + def __init__(self): + self.vendor = "pypeclub" + self.product = "pype" + path = appdirs.user_data_dir(self.product, self.vendor) + super(PypeSettingsRegistry, self).__init__("pype_settings", path) diff --git a/pype/lib/terminal_splash.py b/pype/lib/terminal_splash.py index 0d148bd6e2..1a7645571e 100644 --- a/pype/lib/terminal_splash.py +++ b/pype/lib/terminal_splash.py @@ -27,11 +27,12 @@ def play_animation(): animation_length = int(len(animation) / frame_size) current_frame = 0 for _ in range(animation_length): - frame = "" - y = 0 - for scanline in animation[current_frame:current_frame + frame_size]: - frame += scanline - y += 1 + frame = "".join( + scanline + for y, scanline in enumerate( + animation[current_frame : current_frame + frame_size] + ) + ) with term.location(0, 0): # term.aquamarine3_bold(frame) diff --git a/pype/lib/user_settings.py b/pype/lib/user_settings.py index 0b40eccb65..00ce68cb0b 100644 --- a/pype/lib/user_settings.py +++ b/pype/lib/user_settings.py @@ -28,8 +28,6 @@ import platform import appdirs import six -from ..version import __version__ - @six.add_metaclass(ABCMeta) class ASettingRegistry(): @@ -213,11 +211,12 @@ class IniSettingRegistry(ASettingRegistry): # type: (str, str) -> IniSettingRegistry super(IniSettingRegistry, self).__init__(name) # get registry file + version = os.getenv("PYPE_VERSION", "N/A") self._registry_file = os.path.join(path, "{}.ini".format(name)) if not os.path.exists(self._registry_file): with open(self._registry_file, mode="w") as cfg: print("# Settings registry", cfg) - print("# Generated by Pype {}".format(__version__), cfg) + print("# Generated by Pype {}".format(version), cfg) now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") print("# {}".format(now), cfg) @@ -368,7 +367,7 @@ class JSONSettingRegistry(ASettingRegistry): now = datetime.now().strftime("%d/%m/%Y %H:%M:%S") header = { "__metadata__": { - "pype-version": __version__, + "pype-version": os.getenv("PYPE_VERSION", "N/A"), "generated": now }, "registry": {} @@ -459,9 +458,9 @@ class PypeSettingsRegistry(JSONSettingRegistry): product (str): Additional name used for path construction. """ - vendor = "pypeclub" - product = "pype" def __init__(self): + self.vendor = "pypeclub" + self.product = "pype" path = appdirs.user_data_dir(self.product, self.vendor) super(PypeSettingsRegistry, self).__init__("pype_settings", path) diff --git a/setup.py b/setup.py index fbf8f3ec1d..0924f1070d 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,8 @@ build_options = dict( includes=includes, excludes=excludes, bin_includes=bin_includes, - include_files=include_files + include_files=include_files, + optimize=0 ) icon_path = pype_root / "igniter" / "pype.ico" diff --git a/start.py b/start.py index eb8bb7cd42..d47dc1080c 100644 --- a/start.py +++ b/start.py @@ -99,7 +99,16 @@ import traceback import subprocess from pathlib import Path -import acre +# add dependencies folder to sys.pat for frozen code +if getattr(sys, 'frozen', False): + frozen_libs = os.path.normpath( + os.path.join(os.path.dirname(sys.executable), "dependencies")) + sys.path.append(frozen_libs) + # add stuff from `/dependencies` to PYTHONPATH. + pythonpath = os.getenv("PYTHONPATH", "") + paths = pythonpath.split(os.pathsep) + paths.append(frozen_libs) + os.environ["PYTHONPATH"] = os.pathsep.join(paths) from igniter import BootstrapRepos from igniter.tools import load_environments @@ -116,6 +125,15 @@ def set_environments() -> None: better handling of environments """ + try: + import acre + except ImportError: + if getattr(sys, 'frozen', False): + sys.path.append(os.path.join( + os.path.dirname(sys.executable), + "dependencies" + )) + import acre try: env = load_environments(["global"]) except OSError as e: @@ -163,6 +181,7 @@ def set_modules_environments(): """ from pype.modules import ModulesManager + import acre modules_manager = ModulesManager() @@ -267,7 +286,8 @@ def _find_frozen_pype(use_version: str = None, """ pype_version = None - pype_versions = bootstrap.find_pype(include_zips=True) + pype_versions = bootstrap.find_pype(include_zips=True, + staging=use_staging) try: # use latest one found (last in the list is latest) pype_version = pype_versions[-1] @@ -281,16 +301,6 @@ def _find_frozen_pype(use_version: str = None, if not pype_versions: raise RuntimeError("No Pype versions found.") - # find only staging versions - if use_staging: - staging_versions = [v for v in pype_versions if v.is_staging()] - if not staging_versions: - raise RuntimeError("No Pype staging versions found.") - - staging_versions.sort() - # get latest staging version (last in the list is latest) - pype_version = staging_versions[-1] - # get path of version specified in `--use-version` version_path = BootstrapRepos.get_version_path_from_list( use_version, pype_versions) @@ -322,18 +332,11 @@ def _find_frozen_pype(use_version: str = None, print(">>> Extracting zip file ...") version_path = bootstrap.extract_pype(pype_version) + os.environ["PYPE_VERSION"] = pype_version.version # inject version to Python environment (sys.path, ...) print(">>> Injecting Pype version to running environment ...") bootstrap.add_paths_from_directory(version_path) - # add stuff from `/lib` to PYTHONPATH. - pythonpath = os.getenv("PYTHONPATH", "") - paths = pythonpath.split(os.pathsep) - frozen_libs = os.path.normpath( - os.path.join(os.path.dirname(sys.executable), "lib")) - paths.append(frozen_libs) - os.environ["PYTHONPATH"] = os.pathsep.join(paths) - # set PYPE_ROOT to point to currently used Pype version. os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) @@ -347,7 +350,7 @@ def boot(): # Play animation # ------------------------------------------------------------------------ - from pype.lib.terminal_splash import play_animation + from igniter.terminal_splash import play_animation # don't play for silenced commands if all(item not in sys.argv for item in silent_commands): @@ -363,7 +366,6 @@ def boot(): # Determine mongodb connection # ------------------------------------------------------------------------ - pype_mongo = "" try: pype_mongo = _determine_mongodb() except RuntimeError as e: @@ -385,7 +387,6 @@ def boot(): if getattr(sys, 'frozen', False): # find versions of Pype to be used with frozen code - version_path = None try: version_path = _find_frozen_pype(use_version, use_staging) except RuntimeError as e: @@ -399,6 +400,7 @@ def boot(): os.path.dirname(os.path.realpath(__file__))) # get current version of Pype local_version = bootstrap.get_local_live_version() + os.environ["PYPE_VERSION"] = local_version if use_version and use_version != local_version: pype_versions = bootstrap.find_pype(include_zips=True) version_path = BootstrapRepos.get_version_path_from_list( @@ -406,7 +408,9 @@ def boot(): if version_path: # use specified bootstrap.add_paths_from_directory(version_path) - + os.environ["PYPE_VERSION"] = use_version + else: + version_path = pype_root os.environ["PYPE_ROOT"] = pype_root repos = os.listdir(os.path.join(pype_root, "repos")) repos = [os.path.join(pype_root, "repos", repo) for repo in repos] @@ -434,6 +438,8 @@ def boot(): del sys.modules["pype.version"] except AttributeError: pass + except KeyError: + pass from pype import cli from pype.lib import terminal as t @@ -442,10 +448,9 @@ def boot(): set_modules_environments() info = get_info() - info.insert(0, ">>> Using Pype from [ {} ]".format( - os.path.dirname(cli.__file__))) + info.insert(0, f">>> Using Pype from [ {version_path} ]") - t_width = os.get_terminal_size().columns + t_width = os.get_terminal_size().columns - 2 _header = f"*** Pype [{__version__}] " info.insert(0, _header + "-" * (t_width - len(_header))) diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index 34ddc12550..59469b0687 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -33,7 +33,8 @@ def test_pype_version(): assert str(v3) == "1.2.3-staging" v4 = PypeVersion(1, 2, 3, variant="staging", client="client") - assert str(v4) == "1.2.3-staging-client" + assert str(v4) == "1.2.3-client-staging" + assert v3 < v4 v5 = PypeVersion(1, 2, 3, variant="foo", client="x") assert str(v5) == "1.2.3-x" @@ -100,7 +101,7 @@ def test_pype_version(): with pytest.raises(ValueError): _ = PypeVersion(version="booobaa") - v11 = PypeVersion(version="4.6.7-staging-client") + v11 = PypeVersion(version="4.6.7-client-staging") assert v11.major == 4 assert v11.minor == 6 assert v11.subversion == 7 @@ -131,7 +132,7 @@ def test_search_string_for_pype_version(printer): ("foo-3.0", False), ("foo-3.0.1", True), ("3", False), - ("foo-3.0.1-staging-client", True), + ("foo-3.0.1-client-staging", True), ("foo-3.0.1-bar-baz", True) ] for ver_string in strings: @@ -178,7 +179,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): suffix=".zip", type="zip", valid=True), test_pype(prefix="bum-v", version="5.5.4-staging", suffix=".zip", type="zip", valid=True), - test_pype(prefix="zum-v", version="5.5.5-staging-client", + test_pype(prefix="zum-v", version="5.5.5-client-staging", suffix=".zip", type="zip", valid=True), test_pype(prefix="fam-v", version="5.6.3", suffix=".zip", type="zip", valid=True), @@ -201,7 +202,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): suffix=".zip", type="zip", valid=True), test_pype(prefix="woo-v", version="7.2.8-client-strange", suffix=".zip", type="zip", valid=True), - test_pype(prefix="loo-v", version="7.2.10-staging-client", + test_pype(prefix="loo-v", version="7.2.10-client-staging", suffix=".zip", type="zip", valid=True), test_pype(prefix="kok-v", version="7.0.1", suffix=".zip", type="zip", valid=True) @@ -222,7 +223,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): suffix=".zip", type="zip", valid=True), test_pype(prefix="foo-v", version="3.0.1-staging", suffix=".zip", type="zip", valid=True), - test_pype(prefix="foo-v", version="3.0.1-staging-client", + test_pype(prefix="foo-v", version="3.0.1-client-staging", suffix=".zip", type="zip", valid=True), test_pype(prefix="foo-v", version="3.2.0", suffix=".zip", type="zip", valid=True) @@ -254,8 +255,8 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): fp.write("invalid") def _create_valid_dir(path: Path, version: str): - pype_path = path / "pype" - version_path = path / "pype" / "version.py" + pype_path = path / "pype" / "pype" + version_path = pype_path / "version.py" pype_path.mkdir(parents=True, exist_ok=True) with open(version_path, "w") as fp: fp.write(f"__version__ = '{version}'\n\n") @@ -306,7 +307,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): _build_test_item(dir_path, test_file) printer("testing finding Pype in given path ...") - result = fix_bootstrap.find_pype(g_path, True) + result = fix_bootstrap.find_pype(g_path, include_zips=True) # we should have results as file were created assert result is not None, "no Pype version found" # latest item in `result` should be latest version found. @@ -317,6 +318,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): test_versions_3[3].suffix ) ) + assert result, "nothing found" assert result[-1].path == expected_path, "not a latest version of Pype 3" monkeypatch.setenv("PYPE_PATH", e_path.as_posix()) @@ -332,6 +334,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): test_versions_1[5].suffix ) ) + assert result, "nothing found" assert result[-1].path == expected_path, "not a latest version of Pype 1" monkeypatch.delenv("PYPE_PATH", raising=False) @@ -350,14 +353,15 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): # latest item in `result` should be latest version found. expected_path = Path( d_path / "{}{}{}".format( - test_versions_2[4].prefix, - test_versions_2[4].version, - test_versions_2[4].suffix + test_versions_2[3].prefix, + test_versions_2[3].version, + test_versions_2[3].suffix ) ) + assert result, "nothing found" assert result[-1].path == expected_path, "not a latest version of Pype 2" - result = fix_bootstrap.find_pype(e_path, True) + result = fix_bootstrap.find_pype(e_path, include_zips=True) assert result is not None, "no Pype version found" expected_path = Path( e_path / "{}{}{}".format( @@ -368,12 +372,13 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): ) assert result[-1].path == expected_path, "not a latest version of Pype 1" - result = fix_bootstrap.find_pype(dir_path, True) + result = fix_bootstrap.find_pype(dir_path, include_zips=True) assert result is not None, "no Pype versions found" expected_path = Path( - e_path / "{}{}{}".format( + dir_path / "{}{}{}".format( test_versions_4[0].prefix, test_versions_4[0].version, test_versions_4[0].suffix ) ) + assert result[-1].path == expected_path, "not a latest version of Pype 4" diff --git a/tools/build.ps1 b/tools/build.ps1 index 389846d8ea..f2e6b3ba0d 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -168,6 +168,7 @@ catch { Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Cleaning cache files ... " -NoNewline Get-ChildItem $pype_root -Filter "*.pyc" -Force -Recurse | Remove-Item -Force +Get-ChildItem $pype_root -Filter "*.pyo" -Force -Recurse | Remove-Item -Force Get-ChildItem $pype_root -Filter "__pycache__" -Force -Recurse | Remove-Item -Force -Recurse Write-Host "OK" -ForegroundColor green @@ -176,6 +177,7 @@ Write-Host "Building Pype ..." $out = & python setup.py build 2>&1 Set-Content -Path "$($pype_root)\build\build.log" -Value $out +& python -B "$($pype_root)\tools\build_dependencies.py" Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "deactivating venv ..." diff --git a/tools/build_dependencies.py b/tools/build_dependencies.py new file mode 100644 index 0000000000..f85331a787 --- /dev/null +++ b/tools/build_dependencies.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""Script to fix frozen dependencies. + +Because Pype code needs to run under different versions of Python interpreter +(yes, even Python 2) we need to include all dependencies as source code +without Python's system stuff. Cx-freeze puts everything into lib and compile +it as .pyc/.pyo files and that doesn't work for hosts like Maya 2020 with +their own Python interpreter and libraries. + +This script will take ``site-packages`` and copy them to built Pype under +``dependencies`` directory. It will then compare stuff inside with ``lib`` folder +in frozen Pype, removing duplicities from there. + +This must be executed after build finished and it is done by build PowerShell +script. + +Note: Speedcopy can be used for copying if server-side copy is important for +speed. + +""" +import os +import sys +import site +from distutils.util import get_platform +from pathlib import Path +import shutil +import blessed +import time + + +term = blessed.Terminal() + + +def _print(msg: str, type: int = 0) -> None: + """Print message to console. + + Args: + msg (str): message to print + type (int): type of message (0 info, 1 error, 2 note) + + """ + if type == 0: + header = term.aquamarine3(">>> ") + elif type == 1: + header = term.orangered2("!!! ") + elif type == 2: + header = term.tan1("... ") + else: + header = term.darkolivegreen3("--- ") + + print("{}{}".format(header, msg)) + + +_print("Starting dependency cleanup ...") +start_time = time.time_ns() + +# path to venv site packages +sites = site.getsitepackages() + +# WARNING: this assumes that all we've got is path to venv itself and +# another path ending with 'site-packages' as is default. But because +# this must run under different platform, we cannot easily check if this path +# is the one, because under Linux and macOS site-packages are in different +# location. +site_pkg = None +for s in sites: + site_pkg = Path(s) + if site_pkg.name == "site-packages": + break + +_print("Getting venv site-packages ...") +assert site_pkg, "No venv site-packages are found." +_print(f"Working with: {site_pkg}", 2) + +# now, copy it to build directory +build_dir = None +if sys.platform.startswith("linux"): + # TODO: what is it under linux? + raise NotImplementedError("not implemented for linux yet") +elif sys.platform == "darwin": + # TODO: what is it under macOS? + raise NotImplementedError("not implemented for macOS yet") +elif sys.platform == "win32": + # string is formatted as cx_freeze is doing it + build_dir = "exe.{}-{}".format(get_platform(), sys.version[0:3]) + +# create full path +build_dir = Path(os.path.dirname(__file__)).parent / "build" / build_dir + +_print(f"Using build at {build_dir}", 2) +assert build_dir.exists(), "Build directory doesn't exist" + +deps_dir = build_dir / "dependencies" + +# copy all files +_print("Copying dependencies ...") +shutil.copytree(site_pkg.as_posix(), deps_dir.as_posix()) + +# iterate over frozen libs and create list to delete +libs_dir = build_dir / "lib" + +to_delete = [] +_print("Finding duplicates ...") +for d in libs_dir.iterdir(): + if (deps_dir / d.name) in deps_dir.iterdir(): + to_delete.append(d) + _print(f"found {d}", 3) + +# delete duplicates +_print(f"Deleting {len(to_delete)} duplicates ...") +for d in to_delete: + if d.is_dir(): + shutil.rmtree(d) + else: + d.unlink() + +end_time = time.time_ns() +total_time = (end_time - start_time) / 1000000000 +_print(f"Dependency cleanup done in {total_time} secs.") diff --git a/tools/create_zip.ps1 b/tools/create_zip.ps1 index b2d0319534..a78b06bb20 100644 --- a/tools/create_zip.ps1 +++ b/tools/create_zip.ps1 @@ -90,7 +90,7 @@ Write-Host "OK [ $p ]" -ForegroundColor green Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Entering venv ..." try { - . (".\venv\Scripts\Activate.ps1") + . ("$($pype_root)\venv\Scripts\Activate.ps1") } catch { Write-Host "!!! Failed to activate" -ForegroundColor red @@ -99,6 +99,9 @@ catch { } Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Generating zip from current sources ..." +Write-Host "... " -NoNewline -ForegroundColor Magenta +Write-Host "arguments: " -NoNewline -ForegroundColor Gray +Write-Host $ARGS -ForegroundColor White & python "$($pype_root)\start.py" generate-zip $ARGS Write-Host ">>> " -NoNewline -ForegroundColor green From 988e147ae99ab0097b2e0e5b9aadfb9b4f35cbbb Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 15 Jan 2021 22:48:34 +0100 Subject: [PATCH 34/50] hound fixes --- pype/lib/terminal_splash.py | 2 +- start.py | 4 ++-- tools/build_dependencies.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/lib/terminal_splash.py b/pype/lib/terminal_splash.py index 1a7645571e..0ba2706a27 100644 --- a/pype/lib/terminal_splash.py +++ b/pype/lib/terminal_splash.py @@ -30,7 +30,7 @@ def play_animation(): frame = "".join( scanline for y, scanline in enumerate( - animation[current_frame : current_frame + frame_size] + animation[current_frame: current_frame + frame_size] ) ) diff --git a/start.py b/start.py index d47dc1080c..9c648e931c 100644 --- a/start.py +++ b/start.py @@ -110,8 +110,8 @@ if getattr(sys, 'frozen', False): paths.append(frozen_libs) os.environ["PYTHONPATH"] = os.pathsep.join(paths) -from igniter import BootstrapRepos -from igniter.tools import load_environments +from igniter import BootstrapRepos # noqa: E402 +from igniter.tools import load_environments # noqa: E402 bootstrap = BootstrapRepos() diff --git a/tools/build_dependencies.py b/tools/build_dependencies.py index f85331a787..9b3618cd15 100644 --- a/tools/build_dependencies.py +++ b/tools/build_dependencies.py @@ -8,8 +8,8 @@ it as .pyc/.pyo files and that doesn't work for hosts like Maya 2020 with their own Python interpreter and libraries. This script will take ``site-packages`` and copy them to built Pype under -``dependencies`` directory. It will then compare stuff inside with ``lib`` folder -in frozen Pype, removing duplicities from there. +``dependencies`` directory. It will then compare stuff inside with ``lib`` +folder in frozen Pype, removing duplicities from there. This must be executed after build finished and it is done by build PowerShell script. From fd4286e2c6c270e0a58883a532b0a8625e642f28 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 18 Jan 2021 10:58:26 +0100 Subject: [PATCH 35/50] fix(global): import otio python27 workaround --- pype/lib/editorial.py | 49 +++++++++++++++++++++------------------- pype/lib/import_utils.py | 25 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 23 deletions(-) create mode 100644 pype/lib/import_utils.py diff --git a/pype/lib/editorial.py b/pype/lib/editorial.py index 7f29bf00bb..1dbc4d7954 100644 --- a/pype/lib/editorial.py +++ b/pype/lib/editorial.py @@ -1,15 +1,18 @@ import os import re import clique -from opentimelineio import opentime -from opentimelineio.opentime import ( - to_frames, RationalTime, TimeRange) +from .import_utils import discover_host_vendor_module + +try: + from opentimelineio import opentime as _ot +except ImportError: + _ot = discover_host_vendor_module("opentimelineio.opentime") def otio_range_to_frame_range(otio_range): - start = to_frames( + start = _ot.to_frames( otio_range.start_time, otio_range.start_time.rate) - end = start + to_frames( + end = start + _ot.to_frames( otio_range.duration, otio_range.duration.rate) - 1 return start, end @@ -19,12 +22,12 @@ def otio_range_with_handles(otio_range, instance): handle_end = instance.data["handleEnd"] handles_duration = handle_start + handle_end fps = float(otio_range.start_time.rate) - start = to_frames(otio_range.start_time, fps) - duration = to_frames(otio_range.duration, fps) + start = _ot.to_frames(otio_range.start_time, fps) + duration = _ot.to_frames(otio_range.duration, fps) - return TimeRange( - start_time=RationalTime((start - handle_start), fps), - duration=RationalTime((duration + handles_duration), fps) + return _ot.TimeRange( + start_time=_ot.RationalTime((start - handle_start), fps), + duration=_ot.RationalTime((duration + handles_duration), fps) ) @@ -80,22 +83,22 @@ def trim_media_range(media_range, source_range): Trim input media range with clip source range. Args: - media_range (otio.opentime.TimeRange): available range of media - source_range (otio.opentime.TimeRange): clip required range + media_range (otio._ot._ot.TimeRange): available range of media + source_range (otio._ot._ot.TimeRange): clip required range Returns: - otio.opentime.TimeRange: trimmed media range + otio._ot._ot.TimeRange: trimmed media range """ - rw_media_start = RationalTime( + rw_media_start = _ot.RationalTime( media_range.start_time.value + source_range.start_time.value, media_range.start_time.rate ) - rw_media_duration = RationalTime( + rw_media_duration = _ot.RationalTime( source_range.duration.value, media_range.duration.rate ) - return TimeRange( + return _ot.TimeRange( rw_media_start, rw_media_duration) @@ -109,12 +112,12 @@ def range_from_frames(start, duration, fps): fps (float): frame range Returns: - otio.opentime.TimeRange: crated range + otio._ot._ot.TimeRange: crated range """ - return TimeRange( - RationalTime(start, fps), - RationalTime(duration, fps) + return _ot.TimeRange( + _ot.RationalTime(start, fps), + _ot.RationalTime(duration, fps) ) @@ -130,8 +133,8 @@ def frames_to_secons(frames, framerate): float: second value """ - rt = opentime.from_frames(frames, framerate) - return opentime.to_seconds(rt) + rt = _ot.from_frames(frames, framerate) + return _ot.to_seconds(rt) def make_sequence_collection(path, otio_range, metadata): @@ -140,7 +143,7 @@ def make_sequence_collection(path, otio_range, metadata): Args: path (str): path to image sequence with `%d` - otio_range (otio.opentime.TimeRange): range to be used + otio_range (otio._ot._ot.TimeRange): range to be used metadata (dict): data where padding value can be found Returns: diff --git a/pype/lib/import_utils.py b/pype/lib/import_utils.py new file mode 100644 index 0000000000..5c832a925c --- /dev/null +++ b/pype/lib/import_utils.py @@ -0,0 +1,25 @@ +import os +import sys +import importlib +from .log import PypeLogger as Logger + +log = Logger().get_logger(__name__) + + +def discover_host_vendor_module(module_name): + host = os.environ["AVALON_APP"] + pype_root = os.environ["PYPE_ROOT"] + main_module = module_name.split(".")[0] + module_path = os.path.join( + pype_root, "hosts", host, "vendor", main_module) + + log.debug( + "Importing moduel from host vendor path: `{}`".format(module_path)) + + if not os.path.exists(module_path): + log.warning( + "Path not existing: `{}`".format(module_path)) + return None + + sys.path.insert(1, module_path) + return importlib.import_module(module_name) From 4e132607c32f8b26f6aa9af2465ca197f3b7c78b Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 18 Jan 2021 12:20:03 +0100 Subject: [PATCH 36/50] suggestions for environment handling --- start.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/start.py b/start.py index 9c648e931c..d9b825307b 100644 --- a/start.py +++ b/start.py @@ -424,14 +424,24 @@ def boot(): paths += repos os.environ["PYTHONPATH"] = os.pathsep.join(paths) + # TODO: add venv when running from source + # set this to point either to `python` from venv in case of live code # or to `pype` or `pype_console` in case of frozen code os.environ["PYPE_EXECUTABLE"] = sys.executable - # DEPRECATED: remove when `pype-config` dissolves into Pype for good. + # TODO: DEPRECATE remove when `pype-config` dissolves into Pype for good. + # PYPE_MODULE_ROOT should be changed to PYPE_REPOS_ROOT + # This needs to replace environment building in hosts # .-=-----------------------=-=. ^ .=-=--------------------------=-. os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"] + # TODO: add pype tools and vendor to environment + os.environ["PYTHONPATH"] = os.pathsep.join( + [os.environ["PYTHONPATH"], + os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"), + os.path.join(os.environ["PYPE_ROOT"], "pype", "vendor")]) + # delete Pype module from cache so it is used from specific version try: del sys.modules["pype"] From f7d943bec719c9fd14eaf0f20457f05ed3b2a25b Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 18 Jan 2021 12:20:16 +0100 Subject: [PATCH 37/50] move setup scripts to pype/pype --- {setup => pype/setup}/blender/init.py | 0 .../32bit/backgrounds_selected_to32bit.py | 0 .../Comp/colorbleed/32bit/backgrounds_to32bit.py | 0 .../colorbleed/32bit/loaders_selected_to32bit.py | 0 .../Comp/colorbleed/32bit/loaders_to32bit.py | 0 .../colorbleed/duplicate_with_input_connections.py | 0 .../scripts/Comp/colorbleed/set_rendermode.py | 0 .../fusion/scripts/Comp/colorbleed/switch_ui.py | 0 .../colorbleed/update_selected_loader_ranges.py | 0 .../HieroPlayer/PlayerPresets.hrox | 0 .../hiero_plugin_path/Icons/1_add_handles_end.png | Bin .../hiero/hiero_plugin_path/Icons/2_add_handles.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/3D.png | Bin .../hiero_plugin_path/Icons/3_add_handles_start.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/4_2D.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/edit.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/fusion.png | Bin .../hiero/hiero_plugin_path/Icons/hierarchy.png | Bin .../hiero/hiero_plugin_path/Icons/houdini.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/layers.psd | Bin .../setup}/hiero/hiero_plugin_path/Icons/lense.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/lense1.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/maya.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/nuke.png | Bin .../hiero/hiero_plugin_path/Icons/resolution.png | Bin .../hiero/hiero_plugin_path/Icons/resolution.psd | Bin .../hiero/hiero_plugin_path/Icons/retiming.png | Bin .../hiero/hiero_plugin_path/Icons/retiming.psd | Bin .../setup}/hiero/hiero_plugin_path/Icons/review.png | Bin .../setup}/hiero/hiero_plugin_path/Icons/review.psd | Bin .../setup}/hiero/hiero_plugin_path/Icons/volume.png | Bin .../hiero/hiero_plugin_path/Icons/z_layer_bg.png | Bin .../hiero/hiero_plugin_path/Icons/z_layer_fg.png | Bin .../hiero/hiero_plugin_path/Icons/z_layer_main.png | Bin .../Python/Startup/SpreadsheetExport.py | 0 .../hiero_plugin_path/Python/Startup/Startup.py | 0 .../Python/Startup/otioexporter/OTIOExportTask.py | 0 .../Python/Startup/otioexporter/OTIOExportUI.py | 0 .../Python/Startup/otioexporter/__init__.py | 0 .../Python/Startup/project_helpers.py | 0 .../Python/Startup/selection_tracker.py | 0 .../Python/Startup/setFrameRate.py | 0 .../Python/Startup/version_everywhere.py | 0 .../Python/StartupUI/PimpMySpreadsheet.py | 0 .../hiero_plugin_path/Python/StartupUI/Purge.py | 0 .../Python/StartupUI/nukeStyleKeyboardShortcuts.py | 0 .../Python/StartupUI/otioimporter/OTIOImport.py | 0 .../Python/StartupUI/otioimporter/__init__.py | 0 .../Python/StartupUI/setPosterFrame.py | 0 .../pipeline.xml | 0 .../pipeline.xml | 0 .../pipeline.xml | 0 {setup => pype/setup}/houdini/MainMenuCommon.XML | 0 {setup => pype/setup}/houdini/scripts/123.py | 0 {setup => pype/setup}/maya/userSetup.py | 0 .../setup}/nuke/nuke_path/KnobScripter/__init__.py | 0 .../KnobScripter/icons/icon_clearConsole.png | Bin .../nuke_path/KnobScripter/icons/icon_download.png | Bin .../nuke_path/KnobScripter/icons/icon_exitnode.png | Bin .../nuke/nuke_path/KnobScripter/icons/icon_pick.png | Bin .../nuke_path/KnobScripter/icons/icon_prefs.png | Bin .../nuke_path/KnobScripter/icons/icon_prefs2.png | Bin .../nuke_path/KnobScripter/icons/icon_refresh.png | Bin .../nuke/nuke_path/KnobScripter/icons/icon_run.png | Bin .../nuke/nuke_path/KnobScripter/icons/icon_save.png | Bin .../nuke_path/KnobScripter/icons/icon_search.png | Bin .../nuke_path/KnobScripter/icons/icon_snippets.png | Bin .../nuke/nuke_path/KnobScripter/knob_scripter.py | 0 {setup => pype/setup}/nuke/nuke_path/init.py | 0 {setup => pype/setup}/nuke/nuke_path/menu.py | 0 .../setup}/nuke/nuke_path/write_to_read.py | 0 71 files changed, 0 insertions(+), 0 deletions(-) rename {setup => pype/setup}/blender/init.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/set_rendermode.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/switch_ui.py (100%) rename {setup => pype/setup}/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/1_add_handles_end.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/2_add_handles.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/3D.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/3_add_handles_start.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/4_2D.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/edit.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/fusion.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/hierarchy.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/houdini.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/layers.psd (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/lense.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/lense1.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/maya.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/nuke.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/resolution.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/resolution.psd (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/retiming.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/retiming.psd (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/review.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/review.psd (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/volume.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/z_layer_bg.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/z_layer_fg.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Icons/z_layer_main.png (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/Startup.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/project_helpers.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/Purge.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml (100%) rename {setup => pype/setup}/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml (100%) rename {setup => pype/setup}/houdini/MainMenuCommon.XML (100%) rename {setup => pype/setup}/houdini/scripts/123.py (100%) rename {setup => pype/setup}/maya/userSetup.py (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/__init__.py (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_clearConsole.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_download.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_exitnode.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_pick.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_prefs.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_prefs2.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_refresh.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_run.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_save.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_search.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/icons/icon_snippets.png (100%) rename {setup => pype/setup}/nuke/nuke_path/KnobScripter/knob_scripter.py (100%) rename {setup => pype/setup}/nuke/nuke_path/init.py (100%) rename {setup => pype/setup}/nuke/nuke_path/menu.py (100%) rename {setup => pype/setup}/nuke/nuke_path/write_to_read.py (100%) diff --git a/setup/blender/init.py b/pype/setup/blender/init.py similarity index 100% rename from setup/blender/init.py rename to pype/setup/blender/init.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_selected_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/backgrounds_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_selected_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py b/pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py rename to pype/setup/fusion/scripts/Comp/colorbleed/32bit/loaders_to32bit.py diff --git a/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py b/pype/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py rename to pype/setup/fusion/scripts/Comp/colorbleed/duplicate_with_input_connections.py diff --git a/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py b/pype/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/set_rendermode.py rename to pype/setup/fusion/scripts/Comp/colorbleed/set_rendermode.py diff --git a/setup/fusion/scripts/Comp/colorbleed/switch_ui.py b/pype/setup/fusion/scripts/Comp/colorbleed/switch_ui.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/switch_ui.py rename to pype/setup/fusion/scripts/Comp/colorbleed/switch_ui.py diff --git a/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py b/pype/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py similarity index 100% rename from setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py rename to pype/setup/fusion/scripts/Comp/colorbleed/update_selected_loader_ranges.py diff --git a/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox b/pype/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox similarity index 100% rename from setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox rename to pype/setup/hiero/hiero_plugin_path/HieroPlayer/PlayerPresets.hrox diff --git a/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png b/pype/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png rename to pype/setup/hiero/hiero_plugin_path/Icons/1_add_handles_end.png diff --git a/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png b/pype/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/2_add_handles.png rename to pype/setup/hiero/hiero_plugin_path/Icons/2_add_handles.png diff --git a/setup/hiero/hiero_plugin_path/Icons/3D.png b/pype/setup/hiero/hiero_plugin_path/Icons/3D.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/3D.png rename to pype/setup/hiero/hiero_plugin_path/Icons/3D.png diff --git a/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png b/pype/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png rename to pype/setup/hiero/hiero_plugin_path/Icons/3_add_handles_start.png diff --git a/setup/hiero/hiero_plugin_path/Icons/4_2D.png b/pype/setup/hiero/hiero_plugin_path/Icons/4_2D.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/4_2D.png rename to pype/setup/hiero/hiero_plugin_path/Icons/4_2D.png diff --git a/setup/hiero/hiero_plugin_path/Icons/edit.png b/pype/setup/hiero/hiero_plugin_path/Icons/edit.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/edit.png rename to pype/setup/hiero/hiero_plugin_path/Icons/edit.png diff --git a/setup/hiero/hiero_plugin_path/Icons/fusion.png b/pype/setup/hiero/hiero_plugin_path/Icons/fusion.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/fusion.png rename to pype/setup/hiero/hiero_plugin_path/Icons/fusion.png diff --git a/setup/hiero/hiero_plugin_path/Icons/hierarchy.png b/pype/setup/hiero/hiero_plugin_path/Icons/hierarchy.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/hierarchy.png rename to pype/setup/hiero/hiero_plugin_path/Icons/hierarchy.png diff --git a/setup/hiero/hiero_plugin_path/Icons/houdini.png b/pype/setup/hiero/hiero_plugin_path/Icons/houdini.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/houdini.png rename to pype/setup/hiero/hiero_plugin_path/Icons/houdini.png diff --git a/setup/hiero/hiero_plugin_path/Icons/layers.psd b/pype/setup/hiero/hiero_plugin_path/Icons/layers.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/layers.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/layers.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/lense.png b/pype/setup/hiero/hiero_plugin_path/Icons/lense.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/lense.png rename to pype/setup/hiero/hiero_plugin_path/Icons/lense.png diff --git a/setup/hiero/hiero_plugin_path/Icons/lense1.png b/pype/setup/hiero/hiero_plugin_path/Icons/lense1.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/lense1.png rename to pype/setup/hiero/hiero_plugin_path/Icons/lense1.png diff --git a/setup/hiero/hiero_plugin_path/Icons/maya.png b/pype/setup/hiero/hiero_plugin_path/Icons/maya.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/maya.png rename to pype/setup/hiero/hiero_plugin_path/Icons/maya.png diff --git a/setup/hiero/hiero_plugin_path/Icons/nuke.png b/pype/setup/hiero/hiero_plugin_path/Icons/nuke.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/nuke.png rename to pype/setup/hiero/hiero_plugin_path/Icons/nuke.png diff --git a/setup/hiero/hiero_plugin_path/Icons/resolution.png b/pype/setup/hiero/hiero_plugin_path/Icons/resolution.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/resolution.png rename to pype/setup/hiero/hiero_plugin_path/Icons/resolution.png diff --git a/setup/hiero/hiero_plugin_path/Icons/resolution.psd b/pype/setup/hiero/hiero_plugin_path/Icons/resolution.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/resolution.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/resolution.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/retiming.png b/pype/setup/hiero/hiero_plugin_path/Icons/retiming.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/retiming.png rename to pype/setup/hiero/hiero_plugin_path/Icons/retiming.png diff --git a/setup/hiero/hiero_plugin_path/Icons/retiming.psd b/pype/setup/hiero/hiero_plugin_path/Icons/retiming.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/retiming.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/retiming.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/review.png b/pype/setup/hiero/hiero_plugin_path/Icons/review.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/review.png rename to pype/setup/hiero/hiero_plugin_path/Icons/review.png diff --git a/setup/hiero/hiero_plugin_path/Icons/review.psd b/pype/setup/hiero/hiero_plugin_path/Icons/review.psd similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/review.psd rename to pype/setup/hiero/hiero_plugin_path/Icons/review.psd diff --git a/setup/hiero/hiero_plugin_path/Icons/volume.png b/pype/setup/hiero/hiero_plugin_path/Icons/volume.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/volume.png rename to pype/setup/hiero/hiero_plugin_path/Icons/volume.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_bg.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_fg.png diff --git a/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png b/pype/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png similarity index 100% rename from setup/hiero/hiero_plugin_path/Icons/z_layer_main.png rename to pype/setup/hiero/hiero_plugin_path/Icons/z_layer_main.png diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/SpreadsheetExport.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/Startup.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/Startup.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportTask.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/OTIOExportUI.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/otioexporter/__init__.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/project_helpers.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/selection_tracker.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/setFrameRate.py diff --git a/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py b/pype/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py rename to pype/setup/hiero/hiero_plugin_path/Python/Startup/version_everywhere.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/PimpMySpreadsheet.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/Purge.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/nukeStyleKeyboardShortcuts.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/OTIOImport.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/otioimporter/__init__.py diff --git a/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py b/pype/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py similarity index 100% rename from setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py rename to pype/setup/hiero/hiero_plugin_path/Python/StartupUI/setPosterFrame.py diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/10.5/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/11.1/Processors/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml b/pype/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml similarity index 100% rename from setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml rename to pype/setup/hiero/hiero_plugin_path/TaskPresets/11.2/hiero.exporters.FnShotProcessor.ShotProcessor/pipeline.xml diff --git a/setup/houdini/MainMenuCommon.XML b/pype/setup/houdini/MainMenuCommon.XML similarity index 100% rename from setup/houdini/MainMenuCommon.XML rename to pype/setup/houdini/MainMenuCommon.XML diff --git a/setup/houdini/scripts/123.py b/pype/setup/houdini/scripts/123.py similarity index 100% rename from setup/houdini/scripts/123.py rename to pype/setup/houdini/scripts/123.py diff --git a/setup/maya/userSetup.py b/pype/setup/maya/userSetup.py similarity index 100% rename from setup/maya/userSetup.py rename to pype/setup/maya/userSetup.py diff --git a/setup/nuke/nuke_path/KnobScripter/__init__.py b/pype/setup/nuke/nuke_path/KnobScripter/__init__.py similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/__init__.py rename to pype/setup/nuke/nuke_path/KnobScripter/__init__.py diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_clearConsole.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_clearConsole.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_clearConsole.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_clearConsole.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_download.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_download.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_download.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_download.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_exitnode.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_exitnode.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_exitnode.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_exitnode.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_pick.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_pick.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_pick.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_pick.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_prefs.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs2.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs2.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_prefs2.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_prefs2.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_refresh.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_refresh.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_refresh.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_refresh.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_run.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_run.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_run.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_run.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_save.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_save.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_save.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_save.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_search.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_search.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_search.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_search.png diff --git a/setup/nuke/nuke_path/KnobScripter/icons/icon_snippets.png b/pype/setup/nuke/nuke_path/KnobScripter/icons/icon_snippets.png similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/icons/icon_snippets.png rename to pype/setup/nuke/nuke_path/KnobScripter/icons/icon_snippets.png diff --git a/setup/nuke/nuke_path/KnobScripter/knob_scripter.py b/pype/setup/nuke/nuke_path/KnobScripter/knob_scripter.py similarity index 100% rename from setup/nuke/nuke_path/KnobScripter/knob_scripter.py rename to pype/setup/nuke/nuke_path/KnobScripter/knob_scripter.py diff --git a/setup/nuke/nuke_path/init.py b/pype/setup/nuke/nuke_path/init.py similarity index 100% rename from setup/nuke/nuke_path/init.py rename to pype/setup/nuke/nuke_path/init.py diff --git a/setup/nuke/nuke_path/menu.py b/pype/setup/nuke/nuke_path/menu.py similarity index 100% rename from setup/nuke/nuke_path/menu.py rename to pype/setup/nuke/nuke_path/menu.py diff --git a/setup/nuke/nuke_path/write_to_read.py b/pype/setup/nuke/nuke_path/write_to_read.py similarity index 100% rename from setup/nuke/nuke_path/write_to_read.py rename to pype/setup/nuke/nuke_path/write_to_read.py From c7c3d6b78d54b7f78221899ea88be1b4dba21585 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Jan 2021 14:14:11 +0100 Subject: [PATCH 38/50] SyncServer GUI - fixes for style --- pype/modules/sync_server/tray/app.py | 13 ++++++- pype/tools/settings/settings/style/style.css | 36 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 59543b121f..f83d90fd3b 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -6,6 +6,7 @@ from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style +#from avalon import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -1269,6 +1270,16 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): self.icons = {} def paint(self, painter, option, index): + option = QtWidgets.QStyleOptionViewItem(option) + option.showDecorationSelected = True + + if (option.showDecorationSelected and + (option.state & QtWidgets.QStyle.State_Selected)): + painter.setOpacity(0.20) # highlight color is a bit off + painter.fillRect(option.rect, + option.palette.highlight()) + painter.setOpacity(1) + d = index.data(QtCore.Qt.DisplayRole) if d: provider, value = d.split() @@ -1296,7 +1307,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): overlay_rect.setHeight(overlay_rect.height() * (1.0 - float(value))) painter.fillRect(overlay_rect, QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) - + painter.setOpacity(1) class SyncRepresentationErrorWindow(QtWidgets.QDialog): def __init__(self, _id, project, dt, tries, msg, parent=None): diff --git a/pype/tools/settings/settings/style/style.css b/pype/tools/settings/settings/style/style.css index f3eb3a258e..3ce9837a8b 100644 --- a/pype/tools/settings/settings/style/style.css +++ b/pype/tools/settings/settings/style/style.css @@ -353,3 +353,39 @@ QScrollBar::up-arrow:vertical, QScrollBar::down-arrow:vertical { QScrollBar::add-page:vertical, QScrollBar::sub-page:vertical { background: none; } + +QTableView +{ + border: 1px solid #444; + gridline-color: #6c6c6c; + background-color: #201F1F; + alternate-background-color:#21252B; +} + +QHeaderView +{ + border: 1px transparent; + border-radius: 2px; + margin: 0px; + padding: 0px; +} + +QHeaderView::section { + background-color: #21252B; + /*color: silver;*/ + padding: 4px; + border: 1px solid #6c6c6c; + border-radius: 0px; + text-align: center; + color: #969b9e; + font-weight: bold; +} + +QTableView::item:pressed, QListView::item:pressed, QTreeView::item:pressed { + background: #78879b; + color: #FFFFFF; +} + +QTableView::item:selected:active, QTreeView::item:selected:active, QListView::item:selected:active { + background: #3d8ec9; +} \ No newline at end of file From 21e285c2cc2d72b05186e96de5867ad17cdc56c2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Jan 2021 19:06:57 +0100 Subject: [PATCH 39/50] SyncServer GUI - fix - reset_provider_for_file Removed pulling sync_server from ModulesManager (not a singleton!) to injecting object and passing everywhere --- pype/modules/sync_server/sync_server.py | 14 ++-- pype/modules/sync_server/tray/app.py | 92 +++++++++++++------------ 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 49a572877c..b999518103 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -118,7 +118,6 @@ class SyncServer(PypeModule, ITrayModule): self.sync_server_thread = None # asyncio requires new thread self.action_show_widget = None - self.connection = AvalonMongoDB() def connect_with_modules(self, *_a, **kw): return @@ -136,13 +135,14 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() self.connection = AvalonMongoDB() + self.connection.install() try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) from .tray.app import SyncServerWindow - self.widget = SyncServerWindow() + self.widget = SyncServerWindow(self) except ValueError: log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False @@ -654,13 +654,13 @@ class SyncServer(PypeModule, ITrayModule): query = { "_id": ObjectId(representation_id) } - self.connection.Session["AVALON_PROJECT"] = collection - representation = list(self.connection.find(query)) + + representation = list(self.connection.database[collection].find(query)) if not representation: raise ValueError("Representation {} not found in {}". format(representation_id, collection)) - local_site, remote_site = self.get_active_sites(collection) + local_site, remote_site = self.get_sites_for_project(collection) if side == 'local': site_name = local_site else: @@ -672,7 +672,7 @@ class SyncServer(PypeModule, ITrayModule): site_index, _ = self._get_provider_rec(files[file_index]. get('sites', []), site_name) - if file_index > 0 and site_index > 0: + if file_index >= 0 and site_index >= 0: elem = {"name": site_name} update = { "$set": {"files.{}.sites.{}".format(file_index, site_index): @@ -680,7 +680,7 @@ class SyncServer(PypeModule, ITrayModule): } } - self.connection.update_one( + self.connection.database[collection].update_one( query, update ) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index f83d90fd3b..09f7b9e3af 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,12 +1,9 @@ from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt -from avalon.api import AvalonMongoDB from pype.tools.settings.settings.widgets.base import ProjectListWidget -from pype.modules import ModulesManager import attr import os from pype.tools.settings.settings import style -#from avalon import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp from pype.lib import PypeLogger @@ -30,7 +27,7 @@ class SyncServerWindow(QtWidgets.QDialog): Main window that contains list of synchronizable projects and summary view with all synchronizable representations for first project """ - def __init__(self, parent=None): + def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) @@ -39,14 +36,15 @@ class SyncServerWindow(QtWidgets.QDialog): self.setWindowIcon(QtGui.QIcon(style.app_icon_path())) self.resize(1400, 800) - body = QtWidgets.QWidget() - footer = QtWidgets.QWidget() + body = QtWidgets.QWidget(self) + footer = QtWidgets.QWidget(self) footer.setFixedHeight(20) container = QtWidgets.QWidget() - projects = SyncProjectListWidget(parent=self) + projects = SyncProjectListWidget(sync_server, self) projects.refresh() # force selection of default - repres = SyncRepresentationWidget(project=projects.current_project, + repres = SyncRepresentationWidget(sync_server, + project=projects.current_project, parent=self) container_layout = QtWidgets.QHBoxLayout(container) @@ -63,7 +61,7 @@ class SyncServerWindow(QtWidgets.QDialog): body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) - message = QtWidgets.QLabel() + message = QtWidgets.QLabel(footer) message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) @@ -86,6 +84,9 @@ class SyncProjectListWidget(ProjectListWidget): """ Lists all projects that are synchronized to choose from """ + def __init__(self, sync_server, parent): + super(SyncProjectListWidget, self).__init__(parent) + self.sync_server = sync_server def validate_context_change(self): return True @@ -93,13 +94,11 @@ class SyncProjectListWidget(ProjectListWidget): def refresh(self): model = self.project_list.model() model.clear() - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] - for project_name in sync_server.get_synced_presets().keys(): + for project_name in self.sync_server.get_synced_presets().keys(): model.appendRow(QtGui.QStandardItem(project_name)) - if len(sync_server.get_synced_presets().keys()) == 0: + if len(self.sync_server.get_synced_presets().keys()) == 0: model.appendRow(QtGui.QStandardItem("No project configured")) self.current_project = self.project_list.currentIndex().data( @@ -132,9 +131,11 @@ class SyncRepresentationWidget(QtWidgets.QWidget): ("state", 50) ) - def __init__(self, project=None, parent=None): + def __init__(self, sync_server, project=None, parent=None): super(SyncRepresentationWidget, self).__init__(parent) + self.sync_server = sync_server + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -144,7 +145,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - model = SyncRepresentationModel(headers, project) + model = SyncRepresentationModel(sync_server, headers, project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( @@ -185,17 +186,23 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addWidget(self.table_view) self.table_view.doubleClicked.connect(self._doubleClicked) + self.table_view.clicked.connect(self._clicked) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) + def _clicked(self, index): + print("clicked") + self.table_view.model()._selected_id = self.table_view.model().\ + data(index, Qt.UserRole) + def _doubleClicked(self, index): """ Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(_id, + detail_window = SyncServerDetailWindow(self.sync_server, _id, self.table_view.model()._project) detail_window.exec() @@ -253,7 +260,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): priority = attr.ib(default=None) state = attr.ib(default=None) - def __init__(self, header, project=None): + def __init__(self, sync_server, header, project=None): super(SyncRepresentationModel, self).__init__() self._header = header self._data = [] @@ -261,19 +268,15 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) self.filter = None + self._selected_id = None self._initialized = False - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project - - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project(self._project) + self.sync_server.get_sites_for_project(self._project) self.projection = self.get_default_projection() @@ -290,6 +293,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.timer.timeout.connect(self.tick) self.timer.start(self.REFRESH_SEC) + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + def data(self, index, role): item = self._data[index.row()] @@ -456,7 +463,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): project (str): name of project """ self._project = project - self.dbcon.Session["AVALON_PROJECT"] = self._project self.refresh() def get_default_query(self, limit=0): @@ -672,7 +678,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, _id, project, parent=None): + def __init__(self, sync_server, _id, project, parent=None): log.debug( "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) @@ -687,11 +693,8 @@ class SyncServerDetailWindow(QtWidgets.QDialog): footer = QtWidgets.QWidget() footer.setFixedHeight(20) - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = None - - container = SyncRepresentationDetailWidget(_id, project, parent=self) + container = SyncRepresentationDetailWidget(sync_server, _id, project, + parent=self) body_layout = QtWidgets.QHBoxLayout(body) body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) @@ -730,17 +733,16 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): ("remote_site", 60), ("size", 60), ("priority", 20), - ("state", 50) + ("state", 90) ) - def __init__(self, _id=None, project=None, parent=None): + def __init__(self, sync_server, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) self.representation_id = _id self.item = None # set to item that mouse was clicked over - manager = ModulesManager() - self.sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -751,7 +753,8 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view = QtWidgets.QTableView() headers = [item[0] for item in self.default_widths] - model = SyncRepresentationDetailModel(headers, _id, project) + model = SyncRepresentationDetailModel(sync_server, headers, _id, + project) self.table_view.setModel(model) self.table_view.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) self.table_view.setSelectionMode( @@ -905,7 +908,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): tries = attr.ib(default=None) error = attr.ib(default=None) - def __init__(self, header, _id, project=None): + def __init__(self, sync_server, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() self._header = header self._data = [] @@ -916,16 +919,11 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self._id = _id self._initialized = False - self.dbcon = AvalonMongoDB() - self.dbcon.install() - self.dbcon.Session["AVALON_PROJECT"] = self._project - - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] + self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote self.local_site, self.remote_site = \ - sync_server.get_sites_for_project(self._project) + self.sync_server.get_sites_for_project(self._project) self.sort = self.DEFAULT_SORT @@ -943,6 +941,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.timer.timeout.connect(self.tick) self.timer.start(SyncRepresentationModel.REFRESH_SEC) + @property + def dbcon(self): + return self.sync_server.connection.database[self._project] + def tick(self): self.refresh(representations=None, load_records=self._rec_loaded) self.timer.start(SyncRepresentationModel.REFRESH_SEC) @@ -1026,7 +1028,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): errors.append(repre.get('failed_local_error')) item = self.SyncRepresentationDetail( - repre.get("_id"), + file.get("_id"), os.path.basename(file["path"]), local_updated, remote_updated, From 51ae46bd6d7d41fe4bd52d546ee90a4e9d135c93 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 12:21:08 +0100 Subject: [PATCH 40/50] SyncServer GUI - fix - keep selection during model refresh --- pype/modules/sync_server/sync_server.py | 5 +- pype/modules/sync_server/tray/app.py | 109 +++++++++++++++++++++--- 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index b999518103..984e5e34a6 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -117,6 +117,9 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread + self.connection = AvalonMongoDB() + self.connection.install() + self.action_show_widget = None def connect_with_modules(self, *_a, **kw): @@ -134,8 +137,6 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() - self.connection = AvalonMongoDB() - self.connection.install() try: self.presets = self.get_synced_presets() diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 09f7b9e3af..80680af749 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -136,6 +136,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.sync_server = sync_server + self._selected_id = None # keep last selected _id + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -185,19 +187,37 @@ class SyncRepresentationWidget(QtWidgets.QWidget): layout.addLayout(top_bar_layout) layout.addWidget(self.table_view) - self.table_view.doubleClicked.connect(self._doubleClicked) - self.table_view.clicked.connect(self._clicked) + self.table_view.doubleClicked.connect(self._double_clicked) self.filter.textChanged.connect(lambda: model.set_filter( self.filter.text())) self.table_view.customContextMenuRequested.connect( self._on_context_menu) - def _clicked(self, index): - print("clicked") - self.table_view.model()._selected_id = self.table_view.model().\ - data(index, Qt.UserRole) + self.table_view.model().modelReset.connect(self._set_selection) - def _doubleClicked(self, index): + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self, new_selection): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index and index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + + def _double_clicked(self, index): """ Opens representation dialog with all files after doubleclick """ @@ -268,7 +288,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._rec_loaded = 0 self._buffer = [] # stash one page worth of records (actually cursor) self.filter = None - self._selected_id = None self._initialized = False @@ -465,6 +484,25 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._project = project self.refresh() + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + def get_default_query(self, limit=0): """ Returns basic aggregate query for main table. @@ -744,6 +782,8 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.sync_server = sync_server + self._selected_id = None + self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representation..") @@ -799,6 +839,30 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.table_view.customContextMenuRequested.connect( self._on_context_menu) + self.table_view.model().modelReset.connect(self._set_selection) + + self.selection_model = self.table_view.selectionModel() + self.selection_model.selectionChanged.connect(self._selection_changed) + + def _selection_changed(self): + index = self.selection_model.currentIndex() + self._selected_id = self.table_view.model().data(index, Qt.UserRole) + + def _set_selection(self): + """ + Sets selection to 'self._selected_id' if exists. + + Keep selection during model refresh. + """ + if self._selected_id: + index = self.table_view.model().get_index(self._selected_id) + if index.isValid(): + mode = QtCore.QItemSelectionModel.Select | \ + QtCore.QItemSelectionModel.Rows + self.selection_model.setCurrentIndex(index, mode) + else: + self._selected_id = None + def _show_detail(self): """ Shows windows with error message for failed sync of a file. @@ -853,7 +917,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): to_run() def _reset_local_site(self): - log.info("reset local site: {}".format(self.item._id)) + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, @@ -861,7 +928,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): 'local') def _reset_remote_site(self): - log.info("reset remote site: {}".format(self.item._id)) + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, @@ -1093,6 +1163,25 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.filter = filter self.refresh() + def get_index(self, id): + """ + Get index of 'id' value. + + Used for keeping selection after refresh. + + Args: + id (str): MongoDB _id + Returns: + (QModelIndex) + """ + index = None + for i in range(self.rowCount(None)): + index = self.index(i, 0) + value = self.data(index, Qt.UserRole) + if value == id: + return index + return index + def get_default_query(self, limit=0): """ Gets query that gets used when no extra sorting, filtering or From 2eed46847a6cf4001802919d46deb144697ae7f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 13:09:10 +0100 Subject: [PATCH 41/50] SyncServer GUI - changed order by status Currently processed sorted as first now --- pype/modules/sync_server/tray/app.py | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 80680af749..313cb80c6a 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -13,9 +13,9 @@ import json log = PypeLogger().get_logger("SyncServer") STATUS = { - 0: 'Queued', + 0: 'In Progress', 1: 'Failed', - 2: 'In Progress', + 2: 'Queued', 3: 'Paused', 4: 'Synced OK', -1: 'Not available' @@ -516,10 +516,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'local_site' - progress of repr on local side, 1 = finished 'remote_site' - progress on remote side, calculates from files 'state' - - 0 - queued + 0 - in progress 1 - failed - 2 - paused (not implemented yet) - 3 - in progress + 2 - queued + 3 - paused (not implemented yet) 4 - finished on both sides are calculated and must be calculated in DB because of @@ -603,10 +603,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): , 'failed_local': {'$sum': '$failed_local'} , 'updated_dt_local': {'$max': "$updated_dt_local"} }}, - {"$sort": self.sort}, {"$limit": limit}, {"$skip": self._rec_loaded}, - {"$project": self.projection} + {"$project": self.projection}, + {"$sort": self.sort} ] def _get_match_part(self): @@ -678,12 +678,12 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'case': { '$or': [{'$eq': ['$avg_progress_remote', 0]}, {'$eq': ['$avg_progress_local', 0]}]}, - 'then': 0 + 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 + 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ @@ -695,18 +695,18 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): {'$lt': ['$avg_progress_local', 1]} ]} ]}, - 'then': 2 + 'then': 0 # In progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 + 'then': 3 # Paused }, { 'case': {'$and': [ {'$eq': ['$avg_progress_remote', 1]}, {'$eq': ['$avg_progress_local', 1]} ]}, - 'then': 4 + 'then': 4 # Synced OK }, ], 'default': -1 @@ -1262,10 +1262,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): []] }]}} }}, - {"$sort": self.sort}, {"$limit": limit}, {"$skip": self._rec_loaded}, - {"$project": self.projection} + {"$project": self.projection}, + {"$sort": self.sort} ] def _get_match_part(self): @@ -1315,12 +1315,12 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'case': { '$or': [{'$eq': ['$progress_remote', 0]}, {'$eq': ['$progress_local', 0]}]}, - 'then': 0 + 'then': 2 # Queued }, { 'case': { '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 + 'then': 1 # Failed }, { 'case': {'$or': [{'$and': [ @@ -1332,7 +1332,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$lt': ['$progress_local', 1]} ]} ]}, - 'then': 2 + 'then': 0 # In Progress }, { 'case': {'$eq': ['dummy_placeholder', 'paused']}, @@ -1343,7 +1343,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): {'$eq': ['$progress_remote', 1]}, {'$eq': ['$progress_local', 1]} ]}, - 'then': 4 + 'then': 4 # Synced OK }, ], 'default': -1 From 01344854bea40b8acfc7a901b7a7a4b4b9f9b68b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 13:35:10 +0100 Subject: [PATCH 42/50] SyncServer GUI - safer initialization of DB --- pype/modules/sync_server/sync_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 984e5e34a6..84637a1d62 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -117,9 +117,6 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None # settings for all enabled projects for sync self.sync_server_thread = None # asyncio requires new thread - self.connection = AvalonMongoDB() - self.connection.install() - self.action_show_widget = None def connect_with_modules(self, *_a, **kw): @@ -138,6 +135,9 @@ class SyncServer(PypeModule, ITrayModule): self.presets = None self.lock = threading.Lock() + self.connection = AvalonMongoDB() + self.connection.install() + try: self.presets = self.get_synced_presets() self.set_active_sites(self.presets) From 83a8c6e3a1be6f03ee6390141dc622f6eb0b0e9c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 14:01:51 +0100 Subject: [PATCH 43/50] SyncServer GUI - Hound --- pype/modules/sync_server/providers/gdrive.py | 4 +- pype/modules/sync_server/tray/app.py | 52 +++++++++++--------- pype/plugins/global/publish/integrate_new.py | 16 +++--- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 44810b81d1..5bc6f21b38 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -383,8 +383,8 @@ class GDriveHandler(AbstractProvider): return response['id'] def download_file(self, source_path, local_path, - server, collection, file, representation, site, - overwrite=False): + server, collection, file, representation, site, + overwrite=False): """ Downloads single file from 'source_path' (remote) to 'local_path'. It creates all folders on the local_path if are not existing. diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 313cb80c6a..31dc8744e1 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -212,7 +212,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): index = self.table_view.model().get_index(self._selected_id) if index and index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None @@ -716,7 +716,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): class SyncServerDetailWindow(QtWidgets.QDialog): - def __init__(self, sync_server, _id, project, parent=None): + def __init__(self, sync_server, _id, project, parent=None): log.debug( "!!! SyncServerDetailWindow _id:: {}".format(_id)) super(SyncServerDetailWindow, self).__init__(parent) @@ -1001,9 +1001,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.projection = self.get_default_projection() self.query = self.get_default_query() - import bson.json_util - # log.debug("detail init query:: {}".format( - # bson.json_util.dumps(self.query, indent=4))) representations = self.dbcon.aggregate(self.query) self.refresh(representations) @@ -1226,23 +1223,31 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): [0]]}]}} # file might be successfully created or failed, not both , 'updated_dt_remote': {'$first': { - '$cond': [{'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - } - ]}} + '$cond': [ + {'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + { + '$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ] + } + ] + }} , 'updated_dt_local': {'$first': { - '$cond': [{'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - } - ]}} + '$cond': [ + {'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + { + '$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ] + } + ] + }} , 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} , 'failed_local': { @@ -1286,8 +1291,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): return { "type": "representation", "_id": self._id, - '$or': [{'files.path': {'$regex': regex_str, - '$options': 'i'}}] + '$or': [{'files.path': {'$regex': regex_str, '$options': 'i'}}] } def get_default_projection(self): @@ -1400,6 +1404,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): QtGui.QBrush(QtGui.QColor(0, 0, 0, 200))) painter.setOpacity(1) + class SyncRepresentationErrorWindow(QtWidgets.QDialog): def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWindow, self).__init__(parent) @@ -1471,4 +1476,3 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): return "%3.1f%s%s" % (value, unit, suffix) value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) - diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 2b867e0116..5ba92435fd 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -929,7 +929,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dest += '.{}'.format(self.TMP_FILE_EXT) return dest - def prepare_file_info(self, path, size=None, file_hash=None, sites=None, instance=None): + def prepare_file_info(self, path, size=None, file_hash=None, + sites=None, instance=None): """ Prepare information for one file (asset or resource) Arguments: @@ -939,6 +940,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sites(optional): array of published locations, [ {'name':'studio', 'created_dt':date} by default keys expected ['studio', 'site1', 'gdrive1'] + instance(dict, optional): to get collected settings Returns: rec: dictionary with filled info """ @@ -947,17 +949,17 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): sync_server_presets = None if (instance.context.data["system_settings"] - ["modules"] - ["sync_server"] - ["enabled"]): + ["modules"] + ["sync_server"] + ["enabled"]): sync_server_presets = (instance.context.data["project_settings"] ["global"] ["sync_server"]) if sync_server_presets["enabled"]: - local_site = sync_server_presets["config"].get("active_site", - "studio").strip() - remote_site = sync_server_presets["config"].get("remote_site") + local_site = sync_server_presets["config"].\ + get("active_site", "studio").strip() + remote_site = sync_server_presets["config"].get("remote_site") rec = { "_id": io.ObjectId(), From 46f09c8cb2f069d932b4626ed3f2d5e8d313cd34 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Jan 2021 16:46:33 +0100 Subject: [PATCH 44/50] SyncServer GUI - Hound 2. --- pype/modules/sync_server/tray/app.py | 261 +++++++++++++++------------ 1 file changed, 146 insertions(+), 115 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 31dc8744e1..afd103f9d5 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -27,6 +27,7 @@ class SyncServerWindow(QtWidgets.QDialog): Main window that contains list of synchronizable projects and summary view with all synchronizable representations for first project """ + def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) self.setWindowFlags(QtCore.Qt.Window) @@ -84,6 +85,7 @@ class SyncProjectListWidget(ProjectListWidget): """ Lists all projects that are synchronized to choose from """ + def __init__(self, sync_server, parent): super(SyncProjectListWidget, self).__init__(parent) self.sync_server = sync_server @@ -105,7 +107,7 @@ class SyncProjectListWidget(ProjectListWidget): QtCore.Qt.DisplayRole ) if not self.current_project: - self.current_project = self.project_list.model().item(0).\ + self.current_project = self.project_list.model().item(0). \ data(QtCore.Qt.DisplayRole) @@ -114,7 +116,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Summary dialog with list of representations that matches current settings 'local_site' and 'remote_site'. """ - active_changed = QtCore.Signal() # active index changed + active_changed = QtCore.Signal() # active index changed default_widths = ( ("asset", 210), @@ -222,8 +224,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): Opens representation dialog with all files after doubleclick """ _id = self.table_view.model().data(index, Qt.UserRole) - detail_window = SyncServerDetailWindow(self.sync_server, _id, - self.table_view.model()._project) + detail_window = SyncServerDetailWindow( + self.sync_server, _id, self.table_view.model()._project) detail_window.exec() def _on_context_menu(self, point): @@ -243,18 +245,18 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "_id": 1 } SORT_BY_COLUMN = [ - "context.asset", # asset - "context.subset", # subset - "context.version", # version - "context.representation", # representation - "updated_dt_local", # local created_dt - "updated_dt_remote", # remote created_dt - "avg_progress_local", # local progress - "avg_progress_remote", # remote progress - "files_count", # count of files - "files_size", # file size of all files - "context.asset", # priority TODO - "status" # state + "context.asset", # asset + "context.subset", # subset + "context.version", # version + "context.representation", # representation + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "avg_progress_local", # local progress + "avg_progress_remote", # remote progress + "files_count", # count of files + "files_size", # file size of all files + "context.asset", # priority TODO + "status" # state ] numberPopulated = QtCore.Signal(int) @@ -543,8 +545,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { + }}, + 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} @@ -554,54 +556,68 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, [1], - [0]]}]}} - , 'progress_local': {'$first': { + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, [1], - [0]]}]}} + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { + 'updated_dt_remote': {'$first': { '$cond': [{'$size': "$order_remote.created_dt"}, "$order_remote.created_dt", {'$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - []] - }]}} - , 'updated_dt_local': {'$first': { + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ]} + ]}}, + 'updated_dt_local': {'$first': { '$cond': [{'$size': "$order_local.created_dt"}, "$order_local.created_dt", {'$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - []] - }]}} - , 'files_size': {'$ifNull': ["$files.size", 0]} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ]} + ]}}, + 'files_size': {'$ifNull': ["$files.size", 0]}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]} }}, {'$group': { - '_id': '$_id' + '_id': '$_id', # pass through context - same for representation - , 'context': {'$addToSet': '$context'} + 'context': {'$addToSet': '$context'}, # pass through files as a list - , 'files': {'$addToSet': '$files'} + 'files': {'$addToSet': '$files'}, # count how many files - , 'files_count': {'$sum': 1} - , 'files_size': {'$sum': '$files_size'} + 'files_count': {'$sum': 1}, + 'files_size': {'$sum': '$files_size'}, # sum avg progress, finished = 1 - , 'avg_progress_remote': {'$avg': "$progress_remote"} - , 'avg_progress_local': {'$avg': "$progress_local"} + 'avg_progress_remote': {'$avg': "$progress_remote"}, + 'avg_progress_local': {'$avg': "$progress_local"}, # select last touch of file - , 'updated_dt_remote': {'$max': "$updated_dt_remote"} - , 'failed_remote': {'$sum': '$failed_remote'} - , 'failed_local': {'$sum': '$failed_local'} - , 'updated_dt_local': {'$max': "$updated_dt_local"} + 'updated_dt_remote': {'$max': "$updated_dt_remote"}, + 'failed_remote': {'$sum': '$failed_remote'}, + 'failed_local': {'$sum': '$failed_local'}, + 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, @@ -634,12 +650,11 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): regex_str = '.*{}.*'.format(self.filter) return { "type": "representation", - '$or': [{'context.subset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.asset': {'$regex': regex_str, - '$options': 'i'}}, - {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], + '$or': [ + {'context.subset': {'$regex': regex_str, '$options': 'i'}}, + {'context.asset': {'$regex': regex_str, '$options': 'i'}}, + {'context.representation': {'$regex': regex_str, + '$options': 'i'}}], 'files.sites': { '$elemMatch': { '$or': [ @@ -687,9 +702,9 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): }, { 'case': {'$or': [{'$and': [ - {'$gt': ['$avg_progress_remote', 0]}, - {'$lt': ['$avg_progress_remote', 1]} - ]}, + {'$gt': ['$avg_progress_remote', 0]}, + {'$lt': ['$avg_progress_remote', 1]} + ]}, {'$and': [ {'$gt': ['$avg_progress_local', 0]}, {'$lt': ['$avg_progress_local', 1]} @@ -761,7 +776,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): project (str): name of project with repre parent (QDialog): SyncServerDetailWindow """ - active_changed = QtCore.Signal() # active index changed + active_changed = QtCore.Signal() # active index changed default_widths = ( ("file", 290), @@ -858,7 +873,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): index = self.table_view.model().get_index(self._selected_id) if index.isValid(): mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.selection_model.setCurrentIndex(index, mode) else: self._selected_id = None @@ -950,13 +965,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): } SORT_BY_COLUMN = [ "files.path", - "updated_dt_local", # local created_dt - "updated_dt_remote", # remote created_dt - "progress_local", # local progress - "progress_remote", # remote progress - "size", # remote progress - "context.asset", # priority TODO - "status" # state + "updated_dt_local", # local created_dt + "updated_dt_remote", # remote created_dt + "progress_local", # local progress + "progress_remote", # remote progress + "size", # remote progress + "context.asset", # priority TODO + "status" # state ] @attr.s @@ -1200,8 +1215,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'order_remote': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.remote_site]} - }} - , 'order_local': { + }}, + 'order_local': { '$filter': {'input': '$files.sites', 'as': 'p', 'cond': {'$eq': ['$$p.name', self.local_site]} }} @@ -1211,61 +1226,74 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): # successfully finished load/download 'progress_remote': {'$first': { '$cond': [{'$size': "$order_remote.progress"}, - "$order_remote.progress", {'$cond': [ - {'$size': "$order_remote.created_dt"}, - [1], - [0]]}]}} - , 'progress_local': {'$first': { + "$order_remote.progress", + {'$cond': [ + {'$size': "$order_remote.created_dt"}, + [1], + [0] + ]} + ]}}, + 'progress_local': {'$first': { '$cond': [{'$size': "$order_local.progress"}, - "$order_local.progress", {'$cond': [ - {'$size': "$order_local.created_dt"}, - [1], - [0]]}]}} + "$order_local.progress", + {'$cond': [ + {'$size': "$order_local.created_dt"}, + [1], + [0] + ]} + ]}}, # file might be successfully created or failed, not both - , 'updated_dt_remote': {'$first': { + 'updated_dt_remote': {'$first': { '$cond': [ - {'$size': "$order_remote.created_dt"}, - "$order_remote.created_dt", - { - '$cond': [ - {'$size': "$order_remote.last_failed_dt"}, - "$order_remote.last_failed_dt", - [] - ] - } - ] - }} - , 'updated_dt_local': {'$first': { + {'$size': "$order_remote.created_dt"}, + "$order_remote.created_dt", + { + '$cond': [ + {'$size': "$order_remote.last_failed_dt"}, + "$order_remote.last_failed_dt", + [] + ] + } + ] + }}, + 'updated_dt_local': {'$first': { '$cond': [ - {'$size': "$order_local.created_dt"}, - "$order_local.created_dt", - { - '$cond': [ - {'$size': "$order_local.last_failed_dt"}, - "$order_local.last_failed_dt", - [] - ] - } - ] - }} - , 'failed_remote': { - '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, 0]} - , 'failed_local': { - '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, 0]} - , 'failed_remote_error': {'$first': { + {'$size': "$order_local.created_dt"}, + "$order_local.created_dt", + { + '$cond': [ + {'$size': "$order_local.last_failed_dt"}, + "$order_local.last_failed_dt", + [] + ] + } + ] + }}, + 'failed_remote': { + '$cond': [{'$size': "$order_remote.last_failed_dt"}, + 1, + 0]}, + 'failed_local': { + '$cond': [{'$size': "$order_local.last_failed_dt"}, + 1, + 0]}, + 'failed_remote_error': {'$first': { '$cond': [{'$size': "$order_remote.error"}, - "$order_remote.error", [""]]}} - , 'failed_local_error': {'$first': { + "$order_remote.error", + [""]]}}, + 'failed_local_error': {'$first': { '$cond': [{'$size': "$order_local.error"}, - "$order_local.error", [""]]}} - , 'tries': {'$first': { + "$order_local.error", + [""]]}}, + 'tries': {'$first': { '$cond': [{'$size': "$order_local.tries"}, "$order_local.tries", {'$cond': [ {'$size': "$order_remote.tries"}, "$order_remote.tries", - []] - }]}} + [] + ]} + ]}} }}, {"$limit": limit}, {"$skip": self._rec_loaded}, @@ -1360,6 +1388,7 @@ class ImageDelegate(QtWidgets.QStyledItemDelegate): """ Prints icon of site and progress of synchronization """ + def __init__(self, parent=None): super(ImageDelegate, self).__init__(parent) self.icons = {} @@ -1444,6 +1473,7 @@ class SyncRepresentationErrorWidget(QtWidgets.QWidget): """ Dialog to show when sync error happened, prints error message """ + def __init__(self, _id, project, dt, tries, msg, parent=None): super(SyncRepresentationErrorWidget, self).__init__(parent) @@ -1460,6 +1490,7 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): """ Pretty print for file size """ + def __init__(self, parent=None): super(SizeDelegate, self).__init__(parent) From 8df840995eddf0a5b528d48c0ee9bcb20f2f1406 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 19 Jan 2021 19:32:04 +0100 Subject: [PATCH 45/50] proper python environments --- pype/lib/__init__.py | 11 + pype/vendor/python/common/README.md | 2 + pype/vendor/{ => python/common}/capture.py | 0 .../common}/capture_gui/__init__.py | 0 .../common}/capture_gui/accordion.py | 0 .../{ => python/common}/capture_gui/app.py | 0 .../common}/capture_gui/colorpicker.py | 0 .../{ => python/common}/capture_gui/lib.py | 0 .../{ => python/common}/capture_gui/plugin.py | 0 .../capture_gui/plugins/cameraplugin.py | 0 .../capture_gui/plugins/codecplugin.py | 0 .../plugins/defaultoptionsplugin.py | 0 .../capture_gui/plugins/displayplugin.py | 0 .../capture_gui/plugins/genericplugin.py | 0 .../common}/capture_gui/plugins/ioplugin.py | 0 .../capture_gui/plugins/panzoomplugin.py | 0 .../capture_gui/plugins/rendererplugin.py | 0 .../capture_gui/plugins/resolutionplugin.py | 0 .../common}/capture_gui/plugins/timeplugin.py | 0 .../capture_gui/plugins/viewportplugin.py | 0 .../common}/capture_gui/presets.py | 0 .../common}/capture_gui/resources/config.png | Bin .../common}/capture_gui/resources/import.png | Bin .../common}/capture_gui/resources/reset.png | Bin .../common}/capture_gui/resources/save.png | Bin .../{ => python/common}/capture_gui/tokens.py | 0 .../common}/capture_gui/vendor/Qt.py | 0 .../common}/capture_gui/vendor/__init__.py | 0 .../common}/capture_gui/version.py | 0 pype/vendor/{ => python/common}/pysync.py | 0 pype/vendor/python/python_2/README.md | 3 + pype/vendor/python/python_3/README.md | 2 + setup.py | 1 - start.py | 215 ++++++++++++------ tools/build.ps1 | 8 + tools/build_dependencies.py | 5 +- tools/create_env.ps1 | 2 +- tools/create_zip.ps1 | 4 +- tools/run_mongo.ps1 | 90 +++++++- tools/run_settings.ps1 | 13 ++ tools/run_tests.ps1 | 15 ++ tools/run_tray.ps1 | 12 + 42 files changed, 302 insertions(+), 81 deletions(-) create mode 100644 pype/vendor/python/common/README.md rename pype/vendor/{ => python/common}/capture.py (100%) rename pype/vendor/{ => python/common}/capture_gui/__init__.py (100%) rename pype/vendor/{ => python/common}/capture_gui/accordion.py (100%) rename pype/vendor/{ => python/common}/capture_gui/app.py (100%) rename pype/vendor/{ => python/common}/capture_gui/colorpicker.py (100%) rename pype/vendor/{ => python/common}/capture_gui/lib.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/cameraplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/codecplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/defaultoptionsplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/displayplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/genericplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/ioplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/panzoomplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/rendererplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/resolutionplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/timeplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/plugins/viewportplugin.py (100%) rename pype/vendor/{ => python/common}/capture_gui/presets.py (100%) rename pype/vendor/{ => python/common}/capture_gui/resources/config.png (100%) rename pype/vendor/{ => python/common}/capture_gui/resources/import.png (100%) rename pype/vendor/{ => python/common}/capture_gui/resources/reset.png (100%) rename pype/vendor/{ => python/common}/capture_gui/resources/save.png (100%) rename pype/vendor/{ => python/common}/capture_gui/tokens.py (100%) rename pype/vendor/{ => python/common}/capture_gui/vendor/Qt.py (100%) rename pype/vendor/{ => python/common}/capture_gui/vendor/__init__.py (100%) rename pype/vendor/{ => python/common}/capture_gui/version.py (100%) rename pype/vendor/{ => python/common}/pysync.py (100%) create mode 100644 pype/vendor/python/python_2/README.md create mode 100644 pype/vendor/python/python_3/README.md diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 691c105b76..d10f3d199d 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -1,5 +1,16 @@ # -*- coding: utf-8 -*- +# flake8: noqa E402 """Pype module API.""" +# add vendor to sys path based on Python version +import sys +import os +import site + +# add Python version specific vendor folder +site.addsitedir( + os.path.join( + os.getenv("PYPE_ROOT", ""), + "vendor", "python", "python_{}".format(sys.version[0]))) from .terminal import Terminal from .execute import ( diff --git a/pype/vendor/python/common/README.md b/pype/vendor/python/common/README.md new file mode 100644 index 0000000000..52effb5a63 --- /dev/null +++ b/pype/vendor/python/common/README.md @@ -0,0 +1,2 @@ +### Info +Here are modules that can run under both Python 2 and Python 3 hosts. \ No newline at end of file diff --git a/pype/vendor/capture.py b/pype/vendor/python/common/capture.py similarity index 100% rename from pype/vendor/capture.py rename to pype/vendor/python/common/capture.py diff --git a/pype/vendor/capture_gui/__init__.py b/pype/vendor/python/common/capture_gui/__init__.py similarity index 100% rename from pype/vendor/capture_gui/__init__.py rename to pype/vendor/python/common/capture_gui/__init__.py diff --git a/pype/vendor/capture_gui/accordion.py b/pype/vendor/python/common/capture_gui/accordion.py similarity index 100% rename from pype/vendor/capture_gui/accordion.py rename to pype/vendor/python/common/capture_gui/accordion.py diff --git a/pype/vendor/capture_gui/app.py b/pype/vendor/python/common/capture_gui/app.py similarity index 100% rename from pype/vendor/capture_gui/app.py rename to pype/vendor/python/common/capture_gui/app.py diff --git a/pype/vendor/capture_gui/colorpicker.py b/pype/vendor/python/common/capture_gui/colorpicker.py similarity index 100% rename from pype/vendor/capture_gui/colorpicker.py rename to pype/vendor/python/common/capture_gui/colorpicker.py diff --git a/pype/vendor/capture_gui/lib.py b/pype/vendor/python/common/capture_gui/lib.py similarity index 100% rename from pype/vendor/capture_gui/lib.py rename to pype/vendor/python/common/capture_gui/lib.py diff --git a/pype/vendor/capture_gui/plugin.py b/pype/vendor/python/common/capture_gui/plugin.py similarity index 100% rename from pype/vendor/capture_gui/plugin.py rename to pype/vendor/python/common/capture_gui/plugin.py diff --git a/pype/vendor/capture_gui/plugins/cameraplugin.py b/pype/vendor/python/common/capture_gui/plugins/cameraplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/cameraplugin.py rename to pype/vendor/python/common/capture_gui/plugins/cameraplugin.py diff --git a/pype/vendor/capture_gui/plugins/codecplugin.py b/pype/vendor/python/common/capture_gui/plugins/codecplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/codecplugin.py rename to pype/vendor/python/common/capture_gui/plugins/codecplugin.py diff --git a/pype/vendor/capture_gui/plugins/defaultoptionsplugin.py b/pype/vendor/python/common/capture_gui/plugins/defaultoptionsplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/defaultoptionsplugin.py rename to pype/vendor/python/common/capture_gui/plugins/defaultoptionsplugin.py diff --git a/pype/vendor/capture_gui/plugins/displayplugin.py b/pype/vendor/python/common/capture_gui/plugins/displayplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/displayplugin.py rename to pype/vendor/python/common/capture_gui/plugins/displayplugin.py diff --git a/pype/vendor/capture_gui/plugins/genericplugin.py b/pype/vendor/python/common/capture_gui/plugins/genericplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/genericplugin.py rename to pype/vendor/python/common/capture_gui/plugins/genericplugin.py diff --git a/pype/vendor/capture_gui/plugins/ioplugin.py b/pype/vendor/python/common/capture_gui/plugins/ioplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/ioplugin.py rename to pype/vendor/python/common/capture_gui/plugins/ioplugin.py diff --git a/pype/vendor/capture_gui/plugins/panzoomplugin.py b/pype/vendor/python/common/capture_gui/plugins/panzoomplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/panzoomplugin.py rename to pype/vendor/python/common/capture_gui/plugins/panzoomplugin.py diff --git a/pype/vendor/capture_gui/plugins/rendererplugin.py b/pype/vendor/python/common/capture_gui/plugins/rendererplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/rendererplugin.py rename to pype/vendor/python/common/capture_gui/plugins/rendererplugin.py diff --git a/pype/vendor/capture_gui/plugins/resolutionplugin.py b/pype/vendor/python/common/capture_gui/plugins/resolutionplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/resolutionplugin.py rename to pype/vendor/python/common/capture_gui/plugins/resolutionplugin.py diff --git a/pype/vendor/capture_gui/plugins/timeplugin.py b/pype/vendor/python/common/capture_gui/plugins/timeplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/timeplugin.py rename to pype/vendor/python/common/capture_gui/plugins/timeplugin.py diff --git a/pype/vendor/capture_gui/plugins/viewportplugin.py b/pype/vendor/python/common/capture_gui/plugins/viewportplugin.py similarity index 100% rename from pype/vendor/capture_gui/plugins/viewportplugin.py rename to pype/vendor/python/common/capture_gui/plugins/viewportplugin.py diff --git a/pype/vendor/capture_gui/presets.py b/pype/vendor/python/common/capture_gui/presets.py similarity index 100% rename from pype/vendor/capture_gui/presets.py rename to pype/vendor/python/common/capture_gui/presets.py diff --git a/pype/vendor/capture_gui/resources/config.png b/pype/vendor/python/common/capture_gui/resources/config.png similarity index 100% rename from pype/vendor/capture_gui/resources/config.png rename to pype/vendor/python/common/capture_gui/resources/config.png diff --git a/pype/vendor/capture_gui/resources/import.png b/pype/vendor/python/common/capture_gui/resources/import.png similarity index 100% rename from pype/vendor/capture_gui/resources/import.png rename to pype/vendor/python/common/capture_gui/resources/import.png diff --git a/pype/vendor/capture_gui/resources/reset.png b/pype/vendor/python/common/capture_gui/resources/reset.png similarity index 100% rename from pype/vendor/capture_gui/resources/reset.png rename to pype/vendor/python/common/capture_gui/resources/reset.png diff --git a/pype/vendor/capture_gui/resources/save.png b/pype/vendor/python/common/capture_gui/resources/save.png similarity index 100% rename from pype/vendor/capture_gui/resources/save.png rename to pype/vendor/python/common/capture_gui/resources/save.png diff --git a/pype/vendor/capture_gui/tokens.py b/pype/vendor/python/common/capture_gui/tokens.py similarity index 100% rename from pype/vendor/capture_gui/tokens.py rename to pype/vendor/python/common/capture_gui/tokens.py diff --git a/pype/vendor/capture_gui/vendor/Qt.py b/pype/vendor/python/common/capture_gui/vendor/Qt.py similarity index 100% rename from pype/vendor/capture_gui/vendor/Qt.py rename to pype/vendor/python/common/capture_gui/vendor/Qt.py diff --git a/pype/vendor/capture_gui/vendor/__init__.py b/pype/vendor/python/common/capture_gui/vendor/__init__.py similarity index 100% rename from pype/vendor/capture_gui/vendor/__init__.py rename to pype/vendor/python/common/capture_gui/vendor/__init__.py diff --git a/pype/vendor/capture_gui/version.py b/pype/vendor/python/common/capture_gui/version.py similarity index 100% rename from pype/vendor/capture_gui/version.py rename to pype/vendor/python/common/capture_gui/version.py diff --git a/pype/vendor/pysync.py b/pype/vendor/python/common/pysync.py similarity index 100% rename from pype/vendor/pysync.py rename to pype/vendor/python/common/pysync.py diff --git a/pype/vendor/python/python_2/README.md b/pype/vendor/python/python_2/README.md new file mode 100644 index 0000000000..f101ddbf54 --- /dev/null +++ b/pype/vendor/python/python_2/README.md @@ -0,0 +1,3 @@ +## Info + +Only **Python 2** specific modules are here. \ No newline at end of file diff --git a/pype/vendor/python/python_3/README.md b/pype/vendor/python/python_3/README.md new file mode 100644 index 0000000000..1b8ab4ce69 --- /dev/null +++ b/pype/vendor/python/python_3/README.md @@ -0,0 +1,2 @@ +## Info +Only **Python 3** modules are here. \ No newline at end of file diff --git a/setup.py b/setup.py index 0924f1070d..3a782a8d72 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,6 @@ include_files = [ "pype", "repos", "schema", - "setup", "vendor", "LICENSE", "README.md", diff --git a/start.py b/start.py index d9b825307b..2b450c0c3f 100644 --- a/start.py +++ b/start.py @@ -46,7 +46,7 @@ So, bootstrapping Pype looks like this:: YES NO | | | +------v--------------+ - | | Fire up Igniter GUI |<---------\ + | | Fire up Igniter GUI |<---------+ | | and ask User | | | +---------------------+ | | | @@ -72,9 +72,9 @@ So, bootstrapping Pype looks like this:: | | to user data dir. | | | +--------------|------------------+ | | .-- Is Pype found? --. | - | YES NO ---------/ + | YES NO ---------+ | | - |<--------/ + |<---------+ | +-------------v------------+ | Run Pype | @@ -97,6 +97,7 @@ import re import sys import traceback import subprocess +import site from pathlib import Path # add dependencies folder to sys.pat for frozen code @@ -112,7 +113,7 @@ if getattr(sys, 'frozen', False): from igniter import BootstrapRepos # noqa: E402 from igniter.tools import load_environments # noqa: E402 - +from igniter.bootstrap_repos import PypeVersion # noqa: E402 bootstrap = BootstrapRepos() silent_commands = ["run", "igniter"] @@ -214,7 +215,6 @@ def _process_arguments() -> tuple: tuple: Return tuple with specific version to use (if any) and flag to prioritize staging (if set) """ - # check for `--use-version=3.0.0` argument and `--use-staging` use_version = None use_staging = False @@ -268,10 +268,35 @@ def _determine_mongodb() -> str: return pype_mongo +def _initialize_environment(pype_version: PypeVersion) -> None: + version_path = pype_version.path + os.environ["PYPE_VERSION"] = pype_version.version + # inject version to Python environment (sys.path, ...) + print(">>> Injecting Pype version to running environment ...") + bootstrap.add_paths_from_directory(version_path) + + # add venv 'site-packages' to PYTHONPATH + python_path = os.getenv("PYTHONPATH", "") + split_paths = python_path.split(os.pathsep) + # add pype tools + split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) + # add common pype vendor + # (common for multiple Python interpreter versions) + split_paths.append(os.path.join( + os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) + + # set PYPE_ROOT to point to currently used Pype version. + os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) + + def _find_frozen_pype(use_version: str = None, use_staging: bool = False) -> Path: """Find Pype to run from frozen code. + This will process and modify environment variables: + ``PYTHONPATH``, ``PYPE_VERSION``, ``PYPE_ROOT`` + Args: use_version (str, optional): Try to use specified version. use_staging (bool, optional): Prefer *staging* flavor over production. @@ -284,7 +309,6 @@ def _find_frozen_pype(use_version: str = None, (if requested). """ - pype_version = None pype_versions = bootstrap.find_pype(include_zips=True, staging=use_staging) @@ -299,15 +323,26 @@ def _find_frozen_pype(use_version: str = None, pype_versions = bootstrap.find_pype() if not pype_versions: - raise RuntimeError("No Pype versions found.") + # no Pype versions found anyway, lets use then the one + # shipped with frozen Pype + version_path = _bootstrap_from_code(use_version) + pype_version = PypeVersion( + version=BootstrapRepos.get_version(version_path), + path=version_path) + _initialize_environment(pype_version) + return version_path # get path of version specified in `--use-version` version_path = BootstrapRepos.get_version_path_from_list( use_version, pype_versions) + if not version_path: if use_version is not None: - print(("!!! Specified version was not found, using " - "latest available")) + if not pype_version: + ... + else: + print(("!!! Specified version was not found, using " + "latest available")) # specified version was not found so use latest detected. version_path = pype_version.path print(f">>> Using version [ {pype_version} ]") @@ -331,20 +366,76 @@ def _find_frozen_pype(use_version: str = None, if pype_version.path.is_file(): print(">>> Extracting zip file ...") version_path = bootstrap.extract_pype(pype_version) + pype_version.path = version_path - os.environ["PYPE_VERSION"] = pype_version.version - # inject version to Python environment (sys.path, ...) - print(">>> Injecting Pype version to running environment ...") - bootstrap.add_paths_from_directory(version_path) - - # set PYPE_ROOT to point to currently used Pype version. - os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) - + _initialize_environment(pype_version) return version_path +def _bootstrap_from_code(use_version): + """Bootstrap live code (or the one coming with frozen Pype. + + Args: + use_version: (str): specific version to use. + + Returns: + Path: path to sourced version. + + """ + # run through repos and add them to `sys.path` and `PYTHONPATH` + # set root + if getattr(sys, 'frozen', False): + pype_root = os.path.normpath( + os.path.dirname(sys.executable)) + local_version = bootstrap.get_version(Path(pype_root)) + else: + pype_root = os.path.normpath( + os.path.dirname(os.path.realpath(__file__))) + # get current version of Pype + local_version = bootstrap.get_local_live_version() + + os.environ["PYPE_VERSION"] = local_version + if use_version and use_version != local_version: + pype_versions = bootstrap.find_pype(include_zips=True) + version_path = BootstrapRepos.get_version_path_from_list( + use_version, pype_versions) + if version_path: + # use specified + bootstrap.add_paths_from_directory(version_path) + os.environ["PYPE_VERSION"] = use_version + else: + version_path = pype_root + os.environ["PYPE_ROOT"] = pype_root + repos = os.listdir(os.path.join(pype_root, "repos")) + repos = [os.path.join(pype_root, "repos", repo) for repo in repos] + # add self to python paths + repos.insert(0, pype_root) + for repo in repos: + sys.path.append(repo) + + # add venv 'site-packages' to PYTHONPATH + python_path = os.getenv("PYTHONPATH", "") + split_paths = python_path.split(os.pathsep) + split_paths += repos + # add pype tools + split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) + # last one should be venv site-packages + # this is slightly convoluted as we can get here from frozen code too + # in case when we are running without any version installed. + if not getattr(sys, 'frozen', False): + split_paths.append(site.getsitepackages()[-1]) + # add common pype vendor + # (common for multiple Python interpreter versions) + split_paths.append(os.path.join( + os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) + + return Path(version_path) + + def boot(): """Bootstrap Pype.""" + version_path = None # ------------------------------------------------------------------------ # Play animation @@ -378,7 +469,12 @@ def boot(): # ------------------------------------------------------------------------ # Load environments from database # ------------------------------------------------------------------------ - + # set PYPE_ROOT to running location until proper version can be + # determined. + if getattr(sys, 'frozen', False): + os.environ["PYPE_ROOT"] = os.path.dirname(sys.executable) + else: + os.environ["PYPE_ROOT"] = os.path.dirname(__file__) set_environments() # ------------------------------------------------------------------------ @@ -394,37 +490,7 @@ def boot(): print(f"!!! {e}") sys.exit(1) else: - # run through repos and add them to sys.path and PYTHONPATH - # set root - pype_root = os.path.normpath( - os.path.dirname(os.path.realpath(__file__))) - # get current version of Pype - local_version = bootstrap.get_local_live_version() - os.environ["PYPE_VERSION"] = local_version - if use_version and use_version != local_version: - pype_versions = bootstrap.find_pype(include_zips=True) - version_path = BootstrapRepos.get_version_path_from_list( - use_version, pype_versions) - if version_path: - # use specified - bootstrap.add_paths_from_directory(version_path) - os.environ["PYPE_VERSION"] = use_version - else: - version_path = pype_root - os.environ["PYPE_ROOT"] = pype_root - repos = os.listdir(os.path.join(pype_root, "repos")) - repos = [os.path.join(pype_root, "repos", repo) for repo in repos] - # add self to python paths - repos.insert(0, pype_root) - for repo in repos: - sys.path.append(repo) - - pythonpath = os.getenv("PYTHONPATH", "") - paths = pythonpath.split(os.pathsep) - paths += repos - os.environ["PYTHONPATH"] = os.pathsep.join(paths) - - # TODO: add venv when running from source + version_path = _bootstrap_from_code(use_version) # set this to point either to `python` from venv in case of live code # or to `pype` or `pype_console` in case of frozen code @@ -433,14 +499,16 @@ def boot(): # TODO: DEPRECATE remove when `pype-config` dissolves into Pype for good. # PYPE_MODULE_ROOT should be changed to PYPE_REPOS_ROOT # This needs to replace environment building in hosts + # .-=-----------------------=-=. ^ .=-=--------------------------=-. os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"] + # -=------------------------=-=. v .=-=--------------------------=-. - # TODO: add pype tools and vendor to environment - os.environ["PYTHONPATH"] = os.pathsep.join( - [os.environ["PYTHONPATH"], - os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"), - os.path.join(os.environ["PYPE_ROOT"], "pype", "vendor")]) + if getattr(sys, 'frozen', False): + os.environ["PYPE_REPOS_ROOT"] = os.environ["PYPE_ROOT"] + else: + os.environ["PYPE_REPOS_ROOT"] = os.path.join( + os.environ["PYPE_ROOT"], "repos") # delete Pype module from cache so it is used from specific version try: @@ -457,6 +525,7 @@ def boot(): print(">>> loading environments ...") set_modules_environments() + assert version_path, "Version path not defined." info = get_info() info.insert(0, f">>> Using Pype from [ {version_path} ]") @@ -485,42 +554,42 @@ def get_info() -> list: components = get_default_components() - infos = [] + inf = [] if not getattr(sys, 'frozen', False): - infos.append(("Pype variant", "staging")) + inf.append(("Pype variant", "staging")) else: - infos.append(("Pype variant", "production")) - infos.append(("Running pype from", os.environ.get('PYPE_ROOT'))) - infos.append(("Using mongodb", components["host"])) + inf.append(("Pype variant", "production")) + inf.append(("Running pype from", os.environ.get('PYPE_ROOT'))) + inf.append(("Using mongodb", components["host"])) if os.environ.get("FTRACK_SERVER"): - infos.append(("Using FTrack at", - os.environ.get("FTRACK_SERVER"))) + inf.append(("Using FTrack at", + os.environ.get("FTRACK_SERVER"))) if os.environ.get('DEADLINE_REST_URL'): - infos.append(("Using Deadline webservice at", - os.environ.get("DEADLINE_REST_URL"))) + inf.append(("Using Deadline webservice at", + os.environ.get("DEADLINE_REST_URL"))) if os.environ.get('MUSTER_REST_URL'): - infos.append(("Using Muster at", - os.environ.get("MUSTER_REST_URL"))) + inf.append(("Using Muster at", + os.environ.get("MUSTER_REST_URL"))) # Reinitialize PypeLogger.initialize() log_components = PypeLogger.log_mongo_url_components if log_components["host"]: - infos.append(("Logging to MongoDB", log_components["host"])) - infos.append((" - port", log_components["port"] or "")) - infos.append((" - database", PypeLogger.log_database_name)) - infos.append((" - collection", PypeLogger.log_collection_name)) - infos.append((" - user", log_components["username"] or "")) + inf.append(("Logging to MongoDB", log_components["host"])) + inf.append((" - port", log_components["port"] or "")) + inf.append((" - database", PypeLogger.log_database_name)) + inf.append((" - collection", PypeLogger.log_collection_name)) + inf.append((" - user", log_components["username"] or "")) if log_components["auth_db"]: - infos.append((" - auth source", log_components["auth_db"])) + inf.append((" - auth source", log_components["auth_db"])) - maximum = max(len(i[0]) for i in infos) + maximum = max(len(i[0]) for i in inf) formatted = [] - for info in infos: + for info in inf: padding = (maximum - len(info[0])) + 1 formatted.append( "... {}:{}[ {} ]".format(info[0], " " * padding, info[1])) diff --git a/tools/build.ps1 b/tools/build.ps1 index f2e6b3ba0d..9e067acb34 100644 --- a/tools/build.ps1 +++ b/tools/build.ps1 @@ -175,6 +175,14 @@ Write-Host "OK" -ForegroundColor green Write-Host ">>> " -NoNewline -ForegroundColor green Write-Host "Building Pype ..." $out = & python setup.py build 2>&1 +if ($LASTEXITCODE -ne 0) +{ + Write-Host "!!! " -NoNewLine -ForegroundColor Red + Write-Host "Build failed. Check the log: " -NoNewline + Write-Host ".\build\build.log" -ForegroundColor Yellow + deactivate + Exit-WithCode $LASTEXITCODE +} Set-Content -Path "$($pype_root)\build\build.log" -Value $out & python -B "$($pype_root)\tools\build_dependencies.py" diff --git a/tools/build_dependencies.py b/tools/build_dependencies.py index 9b3618cd15..d28b631eca 100644 --- a/tools/build_dependencies.py +++ b/tools/build_dependencies.py @@ -88,7 +88,10 @@ elif sys.platform == "win32": build_dir = Path(os.path.dirname(__file__)).parent / "build" / build_dir _print(f"Using build at {build_dir}", 2) -assert build_dir.exists(), "Build directory doesn't exist" +if not build_dir.exists(): + _print("Build directory doesn't exist", 1) + _print("Probably freezing of code failed. Check ./build/build.log", 3) + sys.exit(1) deps_dir = build_dir / "dependencies" diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 index bb04368964..38737a23a9 100644 --- a/tools/create_env.ps1 +++ b/tools/create_env.ps1 @@ -9,7 +9,7 @@ .EXAMPLE -PS> .\build.ps1 +PS> .\create_env.ps1 #> diff --git a/tools/create_zip.ps1 b/tools/create_zip.ps1 index a78b06bb20..35bddfeb4e 100644 --- a/tools/create_zip.ps1 +++ b/tools/create_zip.ps1 @@ -1,6 +1,6 @@ <# .SYNOPSIS - Helper script create virtual env. + Helper script create distributable Pype zip. .DESCRIPTION This script will detect Python installation, create venv and install @@ -9,7 +9,7 @@ .EXAMPLE -PS> .\build.ps1 +PS> .\create_zip.ps1 #> diff --git a/tools/run_mongo.ps1 b/tools/run_mongo.ps1 index 45d4679940..1b6d95ca57 100644 --- a/tools/run_mongo.ps1 +++ b/tools/run_mongo.ps1 @@ -1,7 +1,91 @@ +<# +.SYNOPSIS + Helper script to run mongodb. + +.DESCRIPTION + This script will detect mongodb, add it to the PATH and launch it on specified port and db location. + +.EXAMPLE + +PS> .\run_mongo.ps1 + +#> + +$art = @" + + + ____________ + /\ ___ \ + \ \ \/_\ \ + \ \ _____/ ______ ___ ___ ___ + \ \ \___/ /\ \ \ \\ \\ \ + \ \____\ \ \_____\ \__\\__\\__\ + \/____/ \/_____/ . PYPE Club . + +"@ + +Write-Host $art -ForegroundColor DarkGreen + +function Exit-WithCode($exitcode) { + # Only exit this host process if it's a child of another PowerShell parent process... + $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId + $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name + if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) } + + exit $exitcode +} + + +function Find-Mongo { + Write-Host ">>> " -NoNewLine -ForegroundColor Green + Write-Host "Detecting MongoDB ... " -NoNewline + if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) { + if(Test-Path 'C:\Program Files\MongoDB\Server\*\bin\mongod.exe' -PathType Leaf) { + # we have mongo server installed on standard Windows location + # so we can inject it to the PATH. We'll use latest version available. + $mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]} + if(Test-Path "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) { + $env:PATH="$($env:PATH);C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" + Write-Host "OK" -ForegroundColor Green + Write-Host " - auto-added from [ " -NoNewline + Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewLine -ForegroundColor Cyan + Write-Host " ]" + } else { + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected" -ForegroundColor Yellow + Write-Host "Tried to find it on standard location [ " -NoNewline -ForegroundColor Gray + Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewline -ForegroundColor White + Write-Host " ] but failed." -ForegroundColor Gray + Exit-WithCode 1 + } + } else { + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow + Exit-WithCode 1 + } + + } else { + Write-Host "OK" -ForegroundColor Green + } + <# + .SYNOPSIS + Function to detect mongod in path. + .DESCRIPTION + This will test presence of mongod in PATH. If it's not there, it will try + to find it in default install location. It support different mongo versions + (using latest if found). When mongod is found, path to it is added to PATH + #> +} + $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName -& "$($pype_root)\venv\Scripts\Activate.ps1" +# mongodb port +$port = 2707 -python "$($pype_root)\start.py" mongodb -deactivate +# path to database +$dbpath = (Get-Item $pype_root).parent.FullName + "\mongo_db_data" + +Find-Mongo +$mongo = Get-Command "mongod" | Select-Object -ExpandProperty Definition +Start-Process -FilePath $mongo "--dbpath $($dbpath) --port $($port)" diff --git a/tools/run_settings.ps1 b/tools/run_settings.ps1 index 64d35a4b81..725e0cd8a0 100644 --- a/tools/run_settings.ps1 +++ b/tools/run_settings.ps1 @@ -1,3 +1,16 @@ +<# +.SYNOPSIS + Helper script to Pype Settings UI + +.DESCRIPTION + This script will run Pype and open Settings UI. + +.EXAMPLE + +PS> .\run_settings.ps1 + +#> + $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName diff --git a/tools/run_tests.ps1 b/tools/run_tests.ps1 index 498a3c3d7d..4159114ad3 100644 --- a/tools/run_tests.ps1 +++ b/tools/run_tests.ps1 @@ -1,3 +1,18 @@ +<# +.SYNOPSIS + Helper script to build Pype. + +.DESCRIPTION + This script will detect Python installation, create venv and install + all necessary packages from `requirements.txt` needed by Pype to be + included during application freeze on Windows. + +.EXAMPLE + +PS> .\run_test.ps1 + +#> + function Exit-WithCode($exitcode) { # Only exit this host process if it's a child of another PowerShell parent process... $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId diff --git a/tools/run_tray.ps1 b/tools/run_tray.ps1 index f794f85929..0b75354756 100644 --- a/tools/run_tray.ps1 +++ b/tools/run_tray.ps1 @@ -1,3 +1,15 @@ +<# +.SYNOPSIS + Helper script Pype Tray. + +.DESCRIPTION + + +.EXAMPLE + +PS> .\run_tray.ps1 + +#> $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent $pype_root = (Get-Item $script_dir).parent.FullName From f2ccc5043891e702c0c6eba42ae2e7d1a2fd9abe Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 20 Jan 2021 12:18:29 +0100 Subject: [PATCH 46/50] clean deprecated commands --- pype/cli.py | 55 +++++++---------------------------------------------- start.py | 5 +++++ 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/pype/cli.py b/pype/cli.py index 1f7834a91e..62975bff31 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -11,6 +11,10 @@ from .pype_commands import PypeCommands @click.group(invoke_without_command=True) @click.pass_context +@click.option("--use-version", + expose_value=False, help="use specified version") +@click.option("--use-staging", is_flag=True, + expose_value=False, help="use staging variants") def main(ctx): """Pype is main command serving as entry point to pipeline system. @@ -42,12 +46,6 @@ def tray(debug=False): PypeCommands().launch_tray(debug) -@main.command() -def mongodb(): - """Launch local mongodb server. Useful for development.""" - PypeCommands().launch_local_mongodb() - - @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("--ftrack-url", envvar="FTRACK_SERVER", @@ -60,7 +58,7 @@ def mongodb(): envvar="FTRACK_EVENTS_PATH", help=("path to ftrack event handlers")) @click.option("--no-stored-credentials", is_flag=True, - help="dont use stored credentials") + help="don't use stored credentials") @click.option("--store-credentials", is_flag=True, help="store provided credentials") @click.option("--legacy", is_flag=True, @@ -169,41 +167,6 @@ def texturecopy(debug, project, asset, path): PypeCommands().texture_copy(project, asset, path) -@main.command() -@click.option("-k", "--keyword", help="select tests by keyword to run", - type=click.STRING) -@click.argument("id", nargs=-1, type=click.STRING) -def test(pype, keyword, id): - """Run test suite.""" - if pype: - PypeCommands().run_pype_tests(keyword, id) - - -@main.command() -def make_docs(): - """Generate documentation with Sphinx into `docs/build`.""" - PypeCommands().make_docs() - - -@main.command() -def coverage(): - """Generate code coverage report.""" - PypeCommands().pype_setup_coverage() - - -@main.command() -def clean(): - """Delete python bytecode files. - - Working throughout Pype directory, it will remove all pyc bytecode files. - This is normally not needed but there are cases when update of repostories - caused errors thanks to these files. If you encounter errors complaining - about `magic number`, run this command. - """ - # TODO: reimplement in Python - pass - - @main.command(context_settings={"ignore_unknown_options": True}) @click.option("--app", help="Registered application name") @click.option("--project", help="Project name", @@ -237,7 +200,9 @@ def launch(app, project, asset, task, Optionally you can specify ftrack credentials if needed. ARGUMENTS are passed to launched application. + """ + # TODO: this needs to switch for Settings if ftrack_server: os.environ["FTRACK_SERVER"] = ftrack_server @@ -258,12 +223,6 @@ def launch(app, project, asset, task, PypeCommands().run_application(app, project, asset, task, tools, arguments) -@main.command() -def validate_config(): - """Validate all json configuration files for errors.""" - PypeCommands().validate_jsons() - - @main.command() @click.option("-p", "--path", help="Path to zip file", default=None) def generate_zip(path): diff --git a/start.py b/start.py index 2b450c0c3f..94b1675e23 100644 --- a/start.py +++ b/start.py @@ -219,6 +219,11 @@ def _process_arguments() -> tuple: use_version = None use_staging = False for arg in sys.argv: + if arg == "--use-version": + print("!!! Please use option --use-version like:") + print(" --use-version=3.0.0") + sys.exit(1) + m = re.search(r"--use-version=(?P\d+\.\d+\.\d*.+?)", arg) if m and m.group('version'): use_version = m.group('version') From 9392dbc6a66220f476cb49d7acc09987334fce46 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 20 Jan 2021 14:10:36 +0100 Subject: [PATCH 47/50] handle no terminal situation --- start.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/start.py b/start.py index 94b1675e23..53384ff4ef 100644 --- a/start.py +++ b/start.py @@ -534,7 +534,16 @@ def boot(): info = get_info() info.insert(0, f">>> Using Pype from [ {version_path} ]") - t_width = os.get_terminal_size().columns - 2 + t_width = 20 + try: + t_width = os.get_terminal_size().columns - 2 + except ValueError: + # running without terminal + pass + except OSError: + # running without terminal + pass + _header = f"*** Pype [{__version__}] " info.insert(0, _header + "-" * (t_width - len(_header))) From 90e6666f1809dd82f08d746654840dd921ecfaf7 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 20 Jan 2021 15:21:18 +0100 Subject: [PATCH 48/50] cleanup default environments --- .../system_settings/applications.json | 52 ++++++++----------- .../defaults/system_settings/general.json | 19 ++----- .../defaults/system_settings/modules.json | 4 +- 3 files changed, 27 insertions(+), 48 deletions(-) diff --git a/pype/settings/defaults/system_settings/applications.json b/pype/settings/defaults/system_settings/applications.json index 639b52e423..e1f0f029a5 100644 --- a/pype/settings/defaults/system_settings/applications.json +++ b/pype/settings/defaults/system_settings/applications.json @@ -13,14 +13,13 @@ "MAYA_DISABLE_CER", "PYMEL_SKIP_MEL_INIT", "LC_ALL", - "PYPE_LOG_NO_COLORS", - "MAYA_TEST" + "PYPE_LOG_NO_COLORS" ] }, "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/maya", - "{PYPE_MODULE_ROOT}/repos/maya-look-assigner", - "{PYTHON_ENV}/python2/Lib/site-packages", + "{PYPE_ROOT}/pype/setup/maya", + "{PYPE_REPOS_PATH}/avalon-core/setup/maya", + "{PYPE_REPOS_PATH}/maya-look-assigner", "{PYTHONPATH}" ], "MAYA_DISABLE_CLIC_IPM": "Yes", @@ -28,8 +27,7 @@ "MAYA_DISABLE_CER": "Yes", "PYMEL_SKIP_MEL_INIT": "Yes", "LC_ALL": "C", - "PYPE_LOG_NO_COLORS": "Yes", - "MAYA_TEST": "{MAYA_VERSION}" + "PYPE_LOG_NO_COLORS": "Yes" }, "variants": { "maya_2020": { @@ -241,8 +239,8 @@ ] }, "NUKE_PATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/nuke/nuke_path", - "{PYPE_MODULE_ROOT}/setup/nuke/nuke_path", + "{PYPE_REPOS_PATH}/avalon-core/setup/nuke/nuke_path", + "{PYPE_ROOT}/setup/nuke/nuke_path", "{PYPE_STUDIO_PLUGINS}/nuke" ], "PATH": { @@ -364,8 +362,8 @@ ] }, "NUKE_PATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/nuke/nuke_path", - "{PYPE_MODULE_ROOT}/setup/nuke/nuke_path", + "{PYPE_REPOS_PATH}/avalon-core/setup/nuke/nuke_path", + "{PYPE_ROOT}/setup/nuke/nuke_path", "{PYPE_STUDIO_PLUGINS}/nuke" ], "PATH": { @@ -490,7 +488,7 @@ ] }, "HIERO_PLUGIN_PATH": [ - "{PYPE_MODULE_ROOT}/setup/nukestudio/hiero_plugin_path" + "{PYPE_ROOT}/setup/nukestudio/hiero_plugin_path" ], "PATH": { "windows": "C:/Program Files (x86)/QuickTime/QTSystem/;{PATH}" @@ -612,7 +610,7 @@ ] }, "HIERO_PLUGIN_PATH": [ - "{PYPE_MODULE_ROOT}/setup/nukestudio/hiero_plugin_path" + "{PYPE_ROOT}/setup/nukestudio/hiero_plugin_path" ], "PATH": { "windows": "C:/Program Files (x86)/QuickTime/QTSystem/;{PATH}" @@ -811,8 +809,6 @@ }, "PYTHONPATH": [ "{PYTHON36_RESOLVE}/Lib/site-packages", - "{VIRTUAL_ENV}/Lib/site-packages", - "{PYTHONPATH}", "{RESOLVE_SCRIPT_API}/Modules", "{PYTHONPATH}" ], @@ -821,7 +817,7 @@ "{PYTHON36_RESOLVE}/Scripts", "{PATH}" ], - "PRE_PYTHON_SCRIPT": "{PYPE_MODULE_ROOT}/pype/resolve/preload_console.py", + "PRE_PYTHON_SCRIPT": "{PYPE_ROOT}/pype/resolve/preload_console.py", "PYPE_LOG_NO_COLORS": "True", "RESOLVE_DEV": "True" }, @@ -862,14 +858,14 @@ ] }, "HOUDINI_PATH": { - "darwin": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "linux": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "windows": "{PYPE_MODULE_ROOT}/setup/houdini;&" + "darwin": "{PYPE_ROOT}/setup/houdini:&", + "linux": "{PYPE_ROOT}/setup/houdini:&", + "windows": "{PYPE_ROOT}/setup/houdini;&" }, "HOUDINI_MENU_PATH": { - "darwin": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "linux": "{PYPE_MODULE_ROOT}/setup/houdini:&", - "windows": "{PYPE_MODULE_ROOT}/setup/houdini;&" + "darwin": "{PYPE_ROOT}/setup/houdini:&", + "linux": "{PYPE_ROOT}/setup/houdini:&", + "windows": "{PYPE_ROOT}/setup/houdini;&" } }, "variants": { @@ -920,9 +916,9 @@ "CREATE_NEW_CONSOLE" ] }, - "BLENDER_USER_SCRIPTS": "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/blender", + "BLENDER_USER_SCRIPTS": "{PYPE_REPOS_PATH}/avalon-core/setup/blender", "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/blender", + "{PYPE_REPOS_PATH}/avalon-core/setup/blender", "{PYTHONPATH}" ], "CREATE_NEW_CONSOLE": "yes" @@ -1098,14 +1094,12 @@ "__environment_keys__": { "photoshop": [ "AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH", - "PYTHONPATH", "PYPE_LOG_NO_COLORS", "WEBSOCKET_URL", "WORKFILES_SAVE_AS" ] }, "AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH": "1", - "PYTHONPATH": "{PYTHONPATH}", "PYPE_LOG_NO_COLORS": "Yes", "WEBSOCKET_URL": "ws://localhost:8099/ws/", "WORKFILES_SAVE_AS": "Yes" @@ -1164,14 +1158,12 @@ "__environment_keys__": { "aftereffects": [ "AVALON_AFTEREFFECTS_WORKFILES_ON_LAUNCH", - "PYTHONPATH", "PYPE_LOG_NO_COLORS", "WEBSOCKET_URL", "WORKFILES_SAVE_AS" ] }, "AVALON_AFTEREFFECTS_WORKFILES_ON_LAUNCH": "1", - "PYTHONPATH": "{PYTHONPATH}", "PYPE_LOG_NO_COLORS": "Yes", "WEBSOCKET_URL": "ws://localhost:8097/ws/", "WORKFILES_SAVE_AS": "Yes" @@ -1232,7 +1224,7 @@ "CELACTION_TEMPLATE" ] }, - "CELACTION_TEMPLATE": "{PYPE_MODULE_ROOT}/pype/hosts/celaction/celaction_template_scene.scn" + "CELACTION_TEMPLATE": "{PYPE_ROOT}/pype/hosts/celaction/celaction_template_scene.scn" }, "variants": { "celation_Local": { @@ -1280,7 +1272,7 @@ "QT_PREFERRED_BINDING" ] }, - "AVALON_UNREAL_PLUGIN": "{PYPE_MODULE_ROOT}/repos/avalon-unreal-integration", + "AVALON_UNREAL_PLUGIN": "{PYPE_REPOS_PATH}/avalon-unreal-integration", "PYPE_LOG_NO_COLORS": "True", "QT_PREFERRED_BINDING": "PySide" }, diff --git a/pype/settings/defaults/system_settings/general.json b/pype/settings/defaults/system_settings/general.json index bae0ed4e87..d1810148cd 100644 --- a/pype/settings/defaults/system_settings/general.json +++ b/pype/settings/defaults/system_settings/general.json @@ -15,9 +15,6 @@ "__environment_keys__": { "global": [ "FFMPEG_PATH", - "PATH", - "PYTHONPATH", - "PYPE_PROJECT_CONFIGS", "PYPE_PYTHON_EXE", "PYPE_OCIO_CONFIG", "PYBLISH_GUI", @@ -25,20 +22,10 @@ ] }, "FFMPEG_PATH": { - "windows": "{VIRTUAL_ENV}/localized/ffmpeg_exec/windows/bin;{PYPE_SETUP_PATH}/vendor/bin/ffmpeg_exec/windows/bin", - "darwin": "{VIRTUAL_ENV}/localized/ffmpeg_exec/darwin/bin:{PYPE_SETUP_PATH}/vendor/bin/ffmpeg_exec/darwin/bin", - "linux": "{VIRTUAL_ENV}/localized/ffmpeg_exec/linux:{PYPE_SETUP_PATH}/vendor/bin/ffmpeg_exec/linux" + "windows": "{PYPE_ROOT}/vendor/bin/ffmpeg_exec/windows/bin", + "darwin": "{PYPE_ROOT}/vendor/bin/ffmpeg_exec/darwin/bin", + "linux": ":{PYPE_ROOT}/vendor/bin/ffmpeg_exec/linux" }, - "PATH": [ - "{FFMPEG_PATH}", - "{PATH}" - ], - "PYTHONPATH": { - "windows": "{VIRTUAL_ENV}/Lib/site-packages;{PYPE_MODULE_ROOT}/pype/tools;{PYPE_MODULE_ROOT}/pype/vendor;{PYTHONPATH}", - "linux": "{VIRTUAL_ENV}/lib/python{PYTHON_VERSION}/site-packages:{PYPE_MODULE_ROOT}/pype/tools:{PYPE_MODULE_ROOT}/pype/vendor:{PYTHONPATH}", - "darwin": "{VIRTUAL_ENV}/lib/python{PYTHON_VERSION}/site-packages:{PYPE_MODULE_ROOT}/pype/tools:{PYPE_MODULE_ROOT}/pype/vendor:{PYTHONPATH}" - }, - "PYPE_PROJECT_CONFIGS": "{PYPE_SETUP_PATH}/../studio-project-configs", "PYPE_PYTHON_EXE": { "windows": "{VIRTUAL_ENV}/Scripts/python.exe", "linux": "{VIRTUAL_ENV}/Scripts/python", diff --git a/pype/settings/defaults/system_settings/modules.json b/pype/settings/defaults/system_settings/modules.json index 6493901bac..eb794b5f9d 100644 --- a/pype/settings/defaults/system_settings/modules.json +++ b/pype/settings/defaults/system_settings/modules.json @@ -61,7 +61,7 @@ } }, "is_hierarchical": { - "avalon_mongo_id": { + "tools_env": { "write_security_roles": [ "API", "Administrator", @@ -73,7 +73,7 @@ "Pypeclub" ] }, - "tools_env": { + "avalon_mongo_id": { "write_security_roles": [ "API", "Administrator", From ba2eaf31ab513758abdc9f39d447c2bc59f8b765 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 20 Jan 2021 15:51:20 +0100 Subject: [PATCH 49/50] fix wrong variable in settings --- .../defaults/system_settings/applications.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pype/settings/defaults/system_settings/applications.json b/pype/settings/defaults/system_settings/applications.json index e1f0f029a5..bb0ae7f468 100644 --- a/pype/settings/defaults/system_settings/applications.json +++ b/pype/settings/defaults/system_settings/applications.json @@ -18,8 +18,8 @@ }, "PYTHONPATH": [ "{PYPE_ROOT}/pype/setup/maya", - "{PYPE_REPOS_PATH}/avalon-core/setup/maya", - "{PYPE_REPOS_PATH}/maya-look-assigner", + "{PYPE_REPOS_ROOT}/avalon-core/setup/maya", + "{PYPE_REPOS_ROOT}/maya-look-assigner", "{PYTHONPATH}" ], "MAYA_DISABLE_CLIC_IPM": "Yes", @@ -239,7 +239,7 @@ ] }, "NUKE_PATH": [ - "{PYPE_REPOS_PATH}/avalon-core/setup/nuke/nuke_path", + "{PYPE_REPOS_ROOT}/avalon-core/setup/nuke/nuke_path", "{PYPE_ROOT}/setup/nuke/nuke_path", "{PYPE_STUDIO_PLUGINS}/nuke" ], @@ -362,7 +362,7 @@ ] }, "NUKE_PATH": [ - "{PYPE_REPOS_PATH}/avalon-core/setup/nuke/nuke_path", + "{PYPE_REPOS_ROOT}/avalon-core/setup/nuke/nuke_path", "{PYPE_ROOT}/setup/nuke/nuke_path", "{PYPE_STUDIO_PLUGINS}/nuke" ], @@ -916,9 +916,9 @@ "CREATE_NEW_CONSOLE" ] }, - "BLENDER_USER_SCRIPTS": "{PYPE_REPOS_PATH}/avalon-core/setup/blender", + "BLENDER_USER_SCRIPTS": "{PYPE_REPOS_ROOT}/avalon-core/setup/blender", "PYTHONPATH": [ - "{PYPE_REPOS_PATH}/avalon-core/setup/blender", + "{PYPE_REPOS_ROOT}/avalon-core/setup/blender", "{PYTHONPATH}" ], "CREATE_NEW_CONSOLE": "yes" @@ -1272,7 +1272,7 @@ "QT_PREFERRED_BINDING" ] }, - "AVALON_UNREAL_PLUGIN": "{PYPE_REPOS_PATH}/avalon-unreal-integration", + "AVALON_UNREAL_PLUGIN": "{PYPE_REPOS_ROOT}/avalon-unreal-integration", "PYPE_LOG_NO_COLORS": "True", "QT_PREFERRED_BINDING": "PySide" }, From 21dd9cbf56f984f2a7b7a09979031bd743eb85ea Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 20 Jan 2021 17:01:55 +0100 Subject: [PATCH 50/50] remove PYPE_MODULE_ROOT traces --- pype/plugins/global/publish/extract_burnin.py | 4 ++-- pype/settings/defaults/system_settings/applications.json | 4 ++-- start.py | 8 -------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index c14dfba50a..8d1b9c81e3 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -814,9 +814,9 @@ class ExtractBurnin(pype.api.Extractor): """Return path to python script for burnin processing.""" # TODO maybe convert to Plugin's attribute # Get script path. - module_path = os.environ["PYPE_MODULE_ROOT"] + module_path = os.environ["PYPE_ROOT"] - # There can be multiple paths in PYPE_MODULE_ROOT, in which case + # There can be multiple paths in PYPE_ROOT, in which case # we just take first one. if os.pathsep in module_path: module_path = module_path.split(os.pathsep)[0] diff --git a/pype/settings/defaults/system_settings/applications.json b/pype/settings/defaults/system_settings/applications.json index bb0ae7f468..5ad20291cd 100644 --- a/pype/settings/defaults/system_settings/applications.json +++ b/pype/settings/defaults/system_settings/applications.json @@ -138,8 +138,8 @@ ] }, "PYTHONPATH": [ - "{PYPE_MODULE_ROOT}/repos/avalon-core/setup/maya", - "{PYPE_MODULE_ROOT}/repos/maya-look-assigner", + "{PYPE_REPOS_ROOT}/avalon-core/setup/maya", + "{PYPE_REPOS_ROOT}/maya-look-assigner", "{PYTHON_ENV}/python2/Lib/site-packages", "{PYTHONPATH}" ], diff --git a/start.py b/start.py index 53384ff4ef..ad863481ff 100644 --- a/start.py +++ b/start.py @@ -501,14 +501,6 @@ def boot(): # or to `pype` or `pype_console` in case of frozen code os.environ["PYPE_EXECUTABLE"] = sys.executable - # TODO: DEPRECATE remove when `pype-config` dissolves into Pype for good. - # PYPE_MODULE_ROOT should be changed to PYPE_REPOS_ROOT - # This needs to replace environment building in hosts - - # .-=-----------------------=-=. ^ .=-=--------------------------=-. - os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"] - # -=------------------------=-=. v .=-=--------------------------=-. - if getattr(sys, 'frozen', False): os.environ["PYPE_REPOS_ROOT"] = os.environ["PYPE_ROOT"] else: