Merge pull request #1939 from pypeclub/enhancement/headless-and-validation

OpenPype: Add version validation and `--headless` mode and update progress 🔄
This commit is contained in:
Ondřej Samohel 2021-09-03 17:23:32 +02:00 committed by GitHub
commit 80841fde06
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 506 additions and 40 deletions

2
.gitmodules vendored
View file

@ -9,4 +9,4 @@
url = https://github.com/arrow-py/arrow.git
[submodule "openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api"]
path = openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api
url = https://bitbucket.org/ftrack/ftrack-python-api.git
url = https://bitbucket.org/ftrack/ftrack-python-api.git

View file

@ -12,6 +12,9 @@ from .version import __version__ as version
def open_dialog():
"""Show Igniter dialog."""
if os.getenv("OPENPYPE_HEADLESS_MODE"):
print("!!! Can't open dialog in headless mode. Exiting.")
sys.exit(1)
from Qt import QtWidgets, QtCore
from .install_dialog import InstallDialog
@ -28,8 +31,31 @@ def open_dialog():
return d.result()
def open_update_window(openpype_version):
"""Open update window."""
if os.getenv("OPENPYPE_HEADLESS_MODE"):
print("!!! Can't open dialog in headless mode. Exiting.")
sys.exit(1)
from Qt import QtWidgets, QtCore
from .update_window import UpdateWindow
scale_attr = getattr(QtCore.Qt, "AA_EnableHighDpiScaling", None)
if scale_attr is not None:
QtWidgets.QApplication.setAttribute(scale_attr)
app = QtWidgets.QApplication(sys.argv)
d = UpdateWindow(version=openpype_version)
d.open()
app.exec_()
version_path = d.get_version_path()
return version_path
__all__ = [
"BootstrapRepos",
"open_dialog",
"open_update_window",
"version"
]

View file

@ -9,6 +9,7 @@ import sys
import tempfile
from pathlib import Path
from typing import Union, Callable, List, Tuple
import hashlib
from zipfile import ZipFile, BadZipFile
@ -28,6 +29,25 @@ LOG_WARNING = 1
LOG_ERROR = 3
def sha256sum(filename):
"""Calculate sha256 for content of the file.
Args:
filename (str): Path to file.
Returns:
str: hex encoded sha256
"""
h = hashlib.sha256()
b = bytearray(128 * 1024)
mv = memoryview(b)
with open(filename, 'rb', buffering=0) as f:
for n in iter(lambda: f.readinto(mv), 0):
h.update(mv[:n])
return h.hexdigest()
class OpenPypeVersion(semver.VersionInfo):
"""Class for storing information about OpenPype version.
@ -261,7 +281,8 @@ class BootstrapRepos:
self.live_repo_dir = Path(Path(__file__).parent / ".." / "repos")
@staticmethod
def get_version_path_from_list(version: str, version_list: list) -> Path:
def get_version_path_from_list(
version: str, version_list: list) -> Union[Path, None]:
"""Get path for specific version in list of OpenPype versions.
Args:
@ -275,6 +296,7 @@ class BootstrapRepos:
for v in version_list:
if str(v) == version:
return v.path
return None
@staticmethod
def get_local_live_version() -> str:
@ -487,6 +509,7 @@ class BootstrapRepos:
openpype_root = openpype_path.resolve()
# generate list of filtered paths
dir_filter = [openpype_root / f for f in self.openpype_filter]
checksums = []
file: Path
for file in openpype_list:
@ -508,12 +531,119 @@ class BootstrapRepos:
processed_path = file
self._print(f"- processing {processed_path}")
zip_file.write(file, file.resolve().relative_to(openpype_root))
checksums.append(
(
sha256sum(file.as_posix()),
file.resolve().relative_to(openpype_root)
)
)
zip_file.write(
file, file.resolve().relative_to(openpype_root))
checksums_str = ""
for c in checksums:
checksums_str += "{}:{}\n".format(c[0], c[1])
zip_file.writestr("checksums", checksums_str)
# test if zip is ok
zip_file.testzip()
self._progress_callback(100)
def validate_openpype_version(self, path: Path) -> tuple:
"""Validate version directory or zip file.
This will load `checksums` file if present, calculate checksums
of existing files in given path and compare. It will also compare
lists of files together for missing files.
Args:
path (Path): Path to OpenPype version to validate.
Returns:
tuple(bool, str): with version validity as first item
and string with reason as second.
"""
if not path.exists():
return False, "Path doesn't exist"
if path.is_file():
return self._validate_zip(path)
return self._validate_dir(path)
@staticmethod
def _validate_zip(path: Path) -> tuple:
"""Validate content of zip file."""
with ZipFile(path, "r") as zip_file:
# read checksums
try:
checksums_data = str(zip_file.read("checksums"))
except IOError:
# FIXME: This should be set to False sometimes in the future
return True, "Cannot read checksums for archive."
# split it to the list of tuples
checksums = [
tuple(line.split(":"))
for line in checksums_data.split("\n") if line
]
# calculate and compare checksums in the zip file
for file in checksums:
h = hashlib.sha256()
try:
h.update(zip_file.read(file[1]))
except FileNotFoundError:
return False, f"Missing file [ {file[1]} ]"
if h.hexdigest() != file[0]:
return False, f"Invalid checksum on {file[1]}"
# get list of files in zip minus `checksums` file itself
# and turn in to set to compare against list of files
# from checksum file. If difference exists, something is
# wrong
files_in_zip = zip_file.namelist()
files_in_zip.remove("checksums")
files_in_zip = set(files_in_zip)
files_in_checksum = set([file[1] for file in checksums])
diff = files_in_zip.difference(files_in_checksum)
if diff:
return False, f"Missing files {diff}"
return True, "All ok"
@staticmethod
def _validate_dir(path: Path) -> tuple:
checksums_file = Path(path / "checksums")
if not checksums_file.exists():
# FIXME: This should be set to False sometimes in the future
return True, "Cannot read checksums for archive."
checksums_data = checksums_file.read_text()
checksums = [
tuple(line.split(":"))
for line in checksums_data.split("\n") if line
]
files_in_dir = [
file.relative_to(path).as_posix()
for file in path.iterdir() if file.is_file()
]
files_in_dir.remove("checksums")
files_in_dir = set(files_in_dir)
files_in_checksum = set([file[1] for file in checksums])
for file in checksums:
try:
current = sha256sum((path / file[1]).as_posix())
except FileNotFoundError:
return False, f"Missing file [ {file[1]} ]"
if file[0] != current:
return False, f"Invalid checksum on {file[1]}"
diff = files_in_dir.difference(files_in_checksum)
if diff:
return False, f"Missing files {diff}"
return True, "All ok"
@staticmethod
def add_paths_from_archive(archive: Path) -> None:
"""Add first-level directory and 'repos' as paths to :mod:`sys.path`.
@ -837,6 +967,7 @@ class BootstrapRepos:
# test if destination directory already exist, if so lets delete it.
if destination.exists() and force:
self._print("removing existing directory")
try:
shutil.rmtree(destination)
except OSError as e:
@ -846,6 +977,7 @@ class BootstrapRepos:
raise OpenPypeVersionIOError(
f"cannot remove existing {destination}") from e
elif destination.exists() and not force:
self._print("destination directory already exists")
raise OpenPypeVersionExists(f"{destination} already exist.")
else:
# create destination parent directories even if they don't exist.
@ -855,6 +987,7 @@ class BootstrapRepos:
if openpype_version.path.is_dir():
# create zip inside temporary directory.
self._print("Creating zip from directory ...")
self._progress_callback(0)
with tempfile.TemporaryDirectory() as temp_dir:
temp_zip = \
Path(temp_dir) / f"openpype-v{openpype_version}.zip"
@ -880,13 +1013,16 @@ class BootstrapRepos:
raise OpenPypeVersionInvalid("Invalid file format")
if not self.is_inside_user_data(openpype_version.path):
self._progress_callback(35)
openpype_version.path = self._copy_zip(
openpype_version.path, destination)
# extract zip there
self._print("extracting zip to destination ...")
with ZipFile(openpype_version.path, "r") as zip_ref:
self._progress_callback(75)
zip_ref.extractall(destination)
self._progress_callback(100)
return destination

View file

@ -14,21 +14,13 @@ from .tools import (
validate_mongo_connection,
get_openpype_path_from_db
)
from .nice_progress_bar import NiceProgressBar
from .user_settings import OpenPypeSecureRegistry
from .tools import load_stylesheet
from .version import __version__
def load_stylesheet():
stylesheet_path = os.path.join(
os.path.dirname(__file__),
"stylesheet.css"
)
with open(stylesheet_path, "r") as file_stream:
stylesheet = file_stream.read()
return stylesheet
class ButtonWithOptions(QtWidgets.QFrame):
option_clicked = QtCore.Signal(str)
@ -91,25 +83,6 @@ class ButtonWithOptions(QtWidgets.QFrame):
self.option_clicked.emit(self._default_value)
class NiceProgressBar(QtWidgets.QProgressBar):
def __init__(self, parent=None):
super(NiceProgressBar, self).__init__(parent)
self._real_value = 0
def setValue(self, value):
self._real_value = value
if value != 0 and value < 11:
value = 11
super(NiceProgressBar, self).setValue(value)
def value(self):
return self._real_value
def text(self):
return "{} %".format(self._real_value)
class ConsoleWidget(QtWidgets.QWidget):
def __init__(self, parent=None):
super(ConsoleWidget, self).__init__(parent)

View file

@ -0,0 +1,20 @@
from Qt import QtCore, QtGui, QtWidgets # noqa
class NiceProgressBar(QtWidgets.QProgressBar):
def __init__(self, parent=None):
super(NiceProgressBar, self).__init__(parent)
self._real_value = 0
def setValue(self, value):
self._real_value = value
if value != 0 and value < 11:
value = 11
super(NiceProgressBar, self).setValue(value)
def value(self):
return self._real_value
def text(self):
return "{} %".format(self._real_value)

View file

@ -248,3 +248,15 @@ def get_openpype_path_from_db(url: str) -> Union[str, None]:
if os.path.exists(path):
return path
return None
def load_stylesheet() -> str:
"""Load css style sheet.
Returns:
str: content of the stylesheet
"""
stylesheet_path = Path(__file__).parent.resolve() / "stylesheet.css"
return stylesheet_path.read_text()

61
igniter/update_thread.py Normal file
View file

@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
"""Working thread for update."""
from Qt.QtCore import QThread, Signal, QObject # noqa
from .bootstrap_repos import (
BootstrapRepos,
OpenPypeVersion
)
class UpdateThread(QThread):
"""Install Worker thread.
This class takes care of finding OpenPype version on user entered path
(or loading this path from database). If nothing is entered by user,
OpenPype will create its zip files from repositories that comes with it.
If path contains plain repositories, they are zipped and installed to
user data dir.
"""
progress = Signal(int)
message = Signal((str, bool))
def __init__(self, parent=None):
self._result = None
self._openpype_version = None
QThread.__init__(self, parent)
def set_version(self, openpype_version: OpenPypeVersion):
self._openpype_version = openpype_version
def result(self):
"""Result of finished installation."""
return self._result
def _set_result(self, value):
if self._result is not None:
raise AssertionError("BUG: Result was set more than once!")
self._result = value
def run(self):
"""Thread entry point.
Using :class:`BootstrapRepos` to either install OpenPype as zip files
or copy them from location specified by user or retrieved from
database.
"""
bs = BootstrapRepos(
progress_callback=self.set_progress, message=self.message)
version_path = bs.install_version(self._openpype_version)
self._set_result(version_path)
def set_progress(self, progress: int) -> None:
"""Helper to set progress bar.
Args:
progress (int): Progress in percents.
"""
self.progress.emit(progress)

136
igniter/update_window.py Normal file
View file

@ -0,0 +1,136 @@
# -*- coding: utf-8 -*-
"""Progress window to show when OpenPype is updating/installing locally."""
import os
from .update_thread import UpdateThread
from Qt import QtCore, QtGui, QtWidgets # noqa
from .bootstrap_repos import OpenPypeVersion
from .nice_progress_bar import NiceProgressBar
from .tools import load_stylesheet
class UpdateWindow(QtWidgets.QDialog):
"""OpenPype update window."""
_width = 500
_height = 100
def __init__(self, version: OpenPypeVersion, parent=None):
super(UpdateWindow, self).__init__(parent)
self._openpype_version = version
self._result_version_path = None
self.setWindowTitle(
f"OpenPype is updating ..."
)
self.setModal(True)
self.setWindowFlags(
QtCore.Qt.WindowMinimizeButtonHint
)
current_dir = os.path.dirname(os.path.abspath(__file__))
roboto_font_path = os.path.join(current_dir, "RobotoMono-Regular.ttf")
poppins_font_path = os.path.join(current_dir, "Poppins")
icon_path = os.path.join(current_dir, "openpype_icon.png")
# Install roboto font
QtGui.QFontDatabase.addApplicationFont(roboto_font_path)
for filename in os.listdir(poppins_font_path):
if os.path.splitext(filename)[1] == ".ttf":
QtGui.QFontDatabase.addApplicationFont(filename)
# Load logo
pixmap_openpype_logo = QtGui.QPixmap(icon_path)
# Set logo as icon of window
self.setWindowIcon(QtGui.QIcon(pixmap_openpype_logo))
self._pixmap_openpype_logo = pixmap_openpype_logo
self._update_thread = None
self.resize(QtCore.QSize(self._width, self._height))
self._init_ui()
# Set stylesheet
self.setStyleSheet(load_stylesheet())
self._run_update()
def _init_ui(self):
# Main info
# --------------------------------------------------------------------
main_label = QtWidgets.QLabel(
f"<b>OpenPype</b> is updating to {self._openpype_version}", self)
main_label.setWordWrap(True)
main_label.setObjectName("MainLabel")
# Progress bar
# --------------------------------------------------------------------
progress_bar = NiceProgressBar(self)
progress_bar.setAlignment(QtCore.Qt.AlignCenter)
progress_bar.setTextVisible(False)
# add all to main
main = QtWidgets.QVBoxLayout(self)
main.addSpacing(15)
main.addWidget(main_label, 0)
main.addSpacing(15)
main.addWidget(progress_bar, 0)
main.addSpacing(15)
self._progress_bar = progress_bar
def _run_update(self):
"""Start install process.
This will once again validate entered path and mongo if ok, start
working thread that will do actual job.
"""
# Check if install thread is not already running
if self._update_thread and self._update_thread.isRunning():
return
self._progress_bar.setRange(0, 0)
update_thread = UpdateThread(self)
update_thread.set_version(self._openpype_version)
update_thread.message.connect(self.update_console)
update_thread.progress.connect(self._update_progress)
update_thread.finished.connect(self._installation_finished)
self._update_thread = update_thread
update_thread.start()
def get_version_path(self):
return self._result_version_path
def _installation_finished(self):
status = self._update_thread.result()
self._result_version_path = status
self._progress_bar.setRange(0, 1)
self._update_progress(100)
QtWidgets.QApplication.processEvents()
self.done(0)
def _update_progress(self, progress: int):
# not updating progress as we are not able to determine it
# correctly now. Progress bar is set to un-deterministic mode
# until we are able to get progress in better way.
"""
self._progress_bar.setRange(0, 0)
self._progress_bar.setValue(progress)
text_visible = self._progress_bar.isTextVisible()
if progress == 0:
if text_visible:
self._progress_bar.setTextVisible(False)
elif not text_visible:
self._progress_bar.setTextVisible(True)
"""
return
def update_console(self, msg: str, error: bool = False) -> None:
"""Display message in console.
Args:
msg (str): message.
error (bool): if True, print it red.
"""
print(msg)

View file

@ -18,6 +18,8 @@ from .pype_commands import PypeCommands
@click.option("--list-versions", is_flag=True, expose_value=False,
help=("list all detected versions. Use With `--use-staging "
"to list staging versions."))
@click.option("--validate-version", expose_value=False,
help="validate given version integrity")
def main(ctx):
"""Pype is main command serving as entry point to pipeline system.

View file

@ -179,6 +179,12 @@ else:
ssl_cert_file = certifi.where()
os.environ["SSL_CERT_FILE"] = ssl_cert_file
if "--headless" in sys.argv:
os.environ["OPENPYPE_HEADLESS_MODE"] = "1"
sys.argv.remove("--headless")
else:
if os.getenv("OPENPYPE_HEADLESS_MODE") != "1":
os.environ.pop("OPENPYPE_HEADLESS_MODE", None)
import igniter # noqa: E402
from igniter import BootstrapRepos # noqa: E402
@ -343,7 +349,7 @@ def _process_arguments() -> tuple:
# check for `--use-version=3.0.0` argument and `--use-staging`
use_version = None
use_staging = False
print_versions = False
commands = []
for arg in sys.argv:
if arg == "--use-version":
_print("!!! Please use option --use-version like:")
@ -366,17 +372,38 @@ def _process_arguments() -> tuple:
" proper version string."))
sys.exit(1)
if arg == "--validate-version":
_print("!!! Please use option --validate-version like:")
_print(" --validate-version=3.0.0")
sys.exit(1)
if arg.startswith("--validate-version="):
m = re.search(
r"--validate-version=(?P<version>\d+\.\d+\.\d+(?:\S*)?)", arg)
if m and m.group('version'):
use_version = m.group('version')
sys.argv.remove(arg)
commands.append("validate")
else:
_print("!!! Requested version isn't in correct format.")
_print((" Use --list-versions to find out"
" proper version string."))
sys.exit(1)
if "--use-staging" in sys.argv:
use_staging = True
sys.argv.remove("--use-staging")
if "--list-versions" in sys.argv:
print_versions = True
commands.append("print_versions")
sys.argv.remove("--list-versions")
# handle igniter
# this is helper to run igniter before anything else
if "igniter" in sys.argv:
if os.getenv("OPENPYPE_HEADLESS_MODE") == "1":
_print("!!! Cannot open Igniter dialog in headless mode.")
sys.exit(1)
import igniter
return_code = igniter.open_dialog()
@ -389,7 +416,7 @@ def _process_arguments() -> tuple:
sys.argv.pop(idx)
sys.argv.insert(idx, "tray")
return use_version, use_staging, print_versions
return use_version, use_staging, commands
def _determine_mongodb() -> str:
@ -424,6 +451,11 @@ def _determine_mongodb() -> str:
if not openpype_mongo:
_print("*** No DB connection string specified.")
if os.getenv("OPENPYPE_HEADLESS_MODE") == "1":
_print("!!! Cannot open Igniter dialog in headless mode.")
_print(
"!!! Please use `OPENPYPE_MONGO` to specify server address.")
sys.exit(1)
_print("--- launching setup UI ...")
result = igniter.open_dialog()
@ -527,6 +559,9 @@ def _find_frozen_openpype(use_version: str = None,
except IndexError:
# no OpenPype version found, run Igniter and ask for them.
_print('*** No OpenPype versions found.')
if os.getenv("OPENPYPE_HEADLESS_MODE") == "1":
_print("!!! Cannot open Igniter dialog in headless mode.")
sys.exit(1)
_print("--- launching setup UI ...")
import igniter
return_code = igniter.open_dialog()
@ -590,8 +625,16 @@ def _find_frozen_openpype(use_version: str = None,
if not is_inside:
# install latest version to user data dir
version_path = bootstrap.install_version(
openpype_version, force=True)
if os.getenv("OPENPYPE_HEADLESS_MODE", "0") != "1":
import igniter
version_path = igniter.open_update_window(openpype_version)
else:
version_path = bootstrap.install_version(
openpype_version, force=True)
openpype_version.path = version_path
_initialize_environment(openpype_version)
return openpype_version.path
if openpype_version.path.is_file():
_print(">>> Extracting zip file ...")
@ -738,7 +781,7 @@ def boot():
# Process arguments
# ------------------------------------------------------------------------
use_version, use_staging, print_versions = _process_arguments()
use_version, use_staging, commands = _process_arguments()
if os.getenv("OPENPYPE_VERSION"):
if use_version:
@ -766,13 +809,47 @@ def boot():
# Get openpype path from database and set it to environment so openpype can
# find its versions there and bootstrap them.
openpype_path = get_openpype_path_from_db(openpype_mongo)
if getattr(sys, 'frozen', False):
local_version = bootstrap.get_version(Path(OPENPYPE_ROOT))
else:
local_version = bootstrap.get_local_live_version()
if "validate" in commands:
_print(f">>> Validating version [ {use_version} ]")
openpype_versions = bootstrap.find_openpype(include_zips=True,
staging=True)
openpype_versions += bootstrap.find_openpype(include_zips=True,
staging=False)
v: OpenPypeVersion
found = [v for v in openpype_versions if str(v) == use_version]
if not found:
_print(f"!!! Version [ {use_version} ] not found.")
list_versions(openpype_versions, local_version)
sys.exit(1)
# print result
result = bootstrap.validate_openpype_version(
bootstrap.get_version_path_from_list(
use_version, openpype_versions))
_print("{}{}".format(
">>> " if result[0] else "!!! ",
bootstrap.validate_openpype_version(
bootstrap.get_version_path_from_list(
use_version, openpype_versions)
)[1])
)
sys.exit(1)
if not openpype_path:
_print("*** Cannot get OpenPype path from database.")
if not os.getenv("OPENPYPE_PATH") and openpype_path:
os.environ["OPENPYPE_PATH"] = openpype_path
if print_versions:
if "print_versions" in commands:
if not use_staging:
_print("--- This will list only non-staging versions detected.")
_print(" To see staging versions, use --use-staging argument.")
@ -803,6 +880,13 @@ def boot():
# no version to run
_print(f"!!! {e}")
sys.exit(1)
# validate version
_print(f">>> Validating version [ {str(version_path)} ]")
result = bootstrap.validate_openpype_version(version_path)
if not result[0]:
_print(f"!!! Invalid version: {result[1]}")
sys.exit(1)
_print(f"--- version is valid")
else:
version_path = _bootstrap_from_code(use_version, use_staging)

View file

@ -18,11 +18,14 @@ Running OpenPype without any commands will default to `tray`.
```shell
openpype_console --use-version=3.0.0-foo+bar
```
`--headless` - to run OpenPype in headless mode (without using graphical UI)
`--use-staging` - to use staging versions of OpenPype.
`--list-versions [--use-staging]` - to list available versions.
`--validate-version` to validate integrity of given version
For more information [see here](admin_use#run-openpype).
## Commands

View file

@ -56,6 +56,19 @@ openpype_console --list-versions
You can add `--use-staging` to list staging versions.
:::
If you want to validate integrity of some available version, you can use:
```shell
openpype_console --validate-version=3.3.0
```
This will go through the version and validate file content against sha 256 hashes
stored in `checksums` file.
:::tip Headless mode
Add `--headless` to run OpenPype without graphical UI (useful on server or on automated tasks, etc.)
:::
### Details
When you run OpenPype from executable, few check are made: