diff --git a/.gitmodules b/.gitmodules index 28f164726d..e1b0917e9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,4 +9,4 @@ url = https://github.com/arrow-py/arrow.git [submodule "openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api"] path = openpype/modules/default_modules/ftrack/python2_vendor/ftrack-python-api - url = https://bitbucket.org/ftrack/ftrack-python-api.git + url = https://bitbucket.org/ftrack/ftrack-python-api.git \ No newline at end of file diff --git a/igniter/__init__.py b/igniter/__init__.py index 20bf9be106..defd45e233 100644 --- a/igniter/__init__.py +++ b/igniter/__init__.py @@ -12,6 +12,9 @@ from .version import __version__ as version def open_dialog(): """Show Igniter dialog.""" + if os.getenv("OPENPYPE_HEADLESS_MODE"): + print("!!! Can't open dialog in headless mode. Exiting.") + sys.exit(1) from Qt import QtWidgets, QtCore from .install_dialog import InstallDialog @@ -28,8 +31,31 @@ def open_dialog(): return d.result() +def open_update_window(openpype_version): + """Open update window.""" + if os.getenv("OPENPYPE_HEADLESS_MODE"): + print("!!! Can't open dialog in headless mode. Exiting.") + sys.exit(1) + from Qt import QtWidgets, QtCore + from .update_window import UpdateWindow + + scale_attr = getattr(QtCore.Qt, "AA_EnableHighDpiScaling", None) + if scale_attr is not None: + QtWidgets.QApplication.setAttribute(scale_attr) + + app = QtWidgets.QApplication(sys.argv) + + d = UpdateWindow(version=openpype_version) + d.open() + + app.exec_() + version_path = d.get_version_path() + return version_path + + __all__ = [ "BootstrapRepos", "open_dialog", + "open_update_window", "version" ] diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index b49a2f6e7f..f7f35824c8 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -9,6 +9,7 @@ import sys import tempfile from pathlib import Path from typing import Union, Callable, List, Tuple +import hashlib from zipfile import ZipFile, BadZipFile @@ -28,6 +29,25 @@ LOG_WARNING = 1 LOG_ERROR = 3 +def sha256sum(filename): + """Calculate sha256 for content of the file. + + Args: + filename (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + + class OpenPypeVersion(semver.VersionInfo): """Class for storing information about OpenPype version. @@ -261,7 +281,8 @@ class BootstrapRepos: self.live_repo_dir = Path(Path(__file__).parent / ".." / "repos") @staticmethod - def get_version_path_from_list(version: str, version_list: list) -> Path: + def get_version_path_from_list( + version: str, version_list: list) -> Union[Path, None]: """Get path for specific version in list of OpenPype versions. Args: @@ -275,6 +296,7 @@ class BootstrapRepos: for v in version_list: if str(v) == version: return v.path + return None @staticmethod def get_local_live_version() -> str: @@ -487,6 +509,7 @@ class BootstrapRepos: openpype_root = openpype_path.resolve() # generate list of filtered paths dir_filter = [openpype_root / f for f in self.openpype_filter] + checksums = [] file: Path for file in openpype_list: @@ -508,12 +531,119 @@ class BootstrapRepos: processed_path = file self._print(f"- processing {processed_path}") - zip_file.write(file, file.resolve().relative_to(openpype_root)) + checksums.append( + ( + sha256sum(file.as_posix()), + file.resolve().relative_to(openpype_root) + ) + ) + zip_file.write( + file, file.resolve().relative_to(openpype_root)) + checksums_str = "" + for c in checksums: + checksums_str += "{}:{}\n".format(c[0], c[1]) + zip_file.writestr("checksums", checksums_str) # test if zip is ok zip_file.testzip() self._progress_callback(100) + def validate_openpype_version(self, path: Path) -> tuple: + """Validate version directory or zip file. + + This will load `checksums` file if present, calculate checksums + of existing files in given path and compare. It will also compare + lists of files together for missing files. + + Args: + path (Path): Path to OpenPype version to validate. + + Returns: + tuple(bool, str): with version validity as first item + and string with reason as second. + + """ + if not path.exists(): + return False, "Path doesn't exist" + + if path.is_file(): + return self._validate_zip(path) + return self._validate_dir(path) + + @staticmethod + def _validate_zip(path: Path) -> tuple: + """Validate content of zip file.""" + with ZipFile(path, "r") as zip_file: + # read checksums + try: + checksums_data = str(zip_file.read("checksums")) + except IOError: + # FIXME: This should be set to False sometimes in the future + return True, "Cannot read checksums for archive." + + # split it to the list of tuples + checksums = [ + tuple(line.split(":")) + for line in checksums_data.split("\n") if line + ] + + # calculate and compare checksums in the zip file + for file in checksums: + h = hashlib.sha256() + try: + h.update(zip_file.read(file[1])) + except FileNotFoundError: + return False, f"Missing file [ {file[1]} ]" + if h.hexdigest() != file[0]: + return False, f"Invalid checksum on {file[1]}" + + # get list of files in zip minus `checksums` file itself + # and turn in to set to compare against list of files + # from checksum file. If difference exists, something is + # wrong + files_in_zip = zip_file.namelist() + files_in_zip.remove("checksums") + files_in_zip = set(files_in_zip) + files_in_checksum = set([file[1] for file in checksums]) + diff = files_in_zip.difference(files_in_checksum) + if diff: + return False, f"Missing files {diff}" + + return True, "All ok" + + @staticmethod + def _validate_dir(path: Path) -> tuple: + checksums_file = Path(path / "checksums") + if not checksums_file.exists(): + # FIXME: This should be set to False sometimes in the future + return True, "Cannot read checksums for archive." + checksums_data = checksums_file.read_text() + checksums = [ + tuple(line.split(":")) + for line in checksums_data.split("\n") if line + ] + files_in_dir = [ + file.relative_to(path).as_posix() + for file in path.iterdir() if file.is_file() + ] + files_in_dir.remove("checksums") + files_in_dir = set(files_in_dir) + files_in_checksum = set([file[1] for file in checksums]) + + for file in checksums: + try: + current = sha256sum((path / file[1]).as_posix()) + except FileNotFoundError: + return False, f"Missing file [ {file[1]} ]" + + if file[0] != current: + return False, f"Invalid checksum on {file[1]}" + diff = files_in_dir.difference(files_in_checksum) + if diff: + return False, f"Missing files {diff}" + + return True, "All ok" + @staticmethod def add_paths_from_archive(archive: Path) -> None: """Add first-level directory and 'repos' as paths to :mod:`sys.path`. @@ -837,6 +967,7 @@ class BootstrapRepos: # test if destination directory already exist, if so lets delete it. if destination.exists() and force: + self._print("removing existing directory") try: shutil.rmtree(destination) except OSError as e: @@ -846,6 +977,7 @@ class BootstrapRepos: raise OpenPypeVersionIOError( f"cannot remove existing {destination}") from e elif destination.exists() and not force: + self._print("destination directory already exists") raise OpenPypeVersionExists(f"{destination} already exist.") else: # create destination parent directories even if they don't exist. @@ -855,6 +987,7 @@ class BootstrapRepos: if openpype_version.path.is_dir(): # create zip inside temporary directory. self._print("Creating zip from directory ...") + self._progress_callback(0) with tempfile.TemporaryDirectory() as temp_dir: temp_zip = \ Path(temp_dir) / f"openpype-v{openpype_version}.zip" @@ -880,13 +1013,16 @@ class BootstrapRepos: raise OpenPypeVersionInvalid("Invalid file format") if not self.is_inside_user_data(openpype_version.path): + self._progress_callback(35) openpype_version.path = self._copy_zip( openpype_version.path, destination) # extract zip there self._print("extracting zip to destination ...") with ZipFile(openpype_version.path, "r") as zip_ref: + self._progress_callback(75) zip_ref.extractall(destination) + self._progress_callback(100) return destination diff --git a/igniter/install_dialog.py b/igniter/install_dialog.py index 1ec8cc6768..1fe67e3397 100644 --- a/igniter/install_dialog.py +++ b/igniter/install_dialog.py @@ -14,21 +14,13 @@ from .tools import ( validate_mongo_connection, get_openpype_path_from_db ) + +from .nice_progress_bar import NiceProgressBar from .user_settings import OpenPypeSecureRegistry +from .tools import load_stylesheet from .version import __version__ -def load_stylesheet(): - stylesheet_path = os.path.join( - os.path.dirname(__file__), - "stylesheet.css" - ) - with open(stylesheet_path, "r") as file_stream: - stylesheet = file_stream.read() - - return stylesheet - - class ButtonWithOptions(QtWidgets.QFrame): option_clicked = QtCore.Signal(str) @@ -91,25 +83,6 @@ class ButtonWithOptions(QtWidgets.QFrame): self.option_clicked.emit(self._default_value) -class NiceProgressBar(QtWidgets.QProgressBar): - def __init__(self, parent=None): - super(NiceProgressBar, self).__init__(parent) - self._real_value = 0 - - def setValue(self, value): - self._real_value = value - if value != 0 and value < 11: - value = 11 - - super(NiceProgressBar, self).setValue(value) - - def value(self): - return self._real_value - - def text(self): - return "{} %".format(self._real_value) - - class ConsoleWidget(QtWidgets.QWidget): def __init__(self, parent=None): super(ConsoleWidget, self).__init__(parent) diff --git a/igniter/nice_progress_bar.py b/igniter/nice_progress_bar.py new file mode 100644 index 0000000000..47d695a101 --- /dev/null +++ b/igniter/nice_progress_bar.py @@ -0,0 +1,20 @@ +from Qt import QtCore, QtGui, QtWidgets # noqa + + +class NiceProgressBar(QtWidgets.QProgressBar): + def __init__(self, parent=None): + super(NiceProgressBar, self).__init__(parent) + self._real_value = 0 + + def setValue(self, value): + self._real_value = value + if value != 0 and value < 11: + value = 11 + + super(NiceProgressBar, self).setValue(value) + + def value(self): + return self._real_value + + def text(self): + return "{} %".format(self._real_value) diff --git a/igniter/tools.py b/igniter/tools.py index 529d535c25..c934289064 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -248,3 +248,15 @@ def get_openpype_path_from_db(url: str) -> Union[str, None]: if os.path.exists(path): return path return None + + +def load_stylesheet() -> str: + """Load css style sheet. + + Returns: + str: content of the stylesheet + + """ + stylesheet_path = Path(__file__).parent.resolve() / "stylesheet.css" + + return stylesheet_path.read_text() diff --git a/igniter/update_thread.py b/igniter/update_thread.py new file mode 100644 index 0000000000..f4fc729faf --- /dev/null +++ b/igniter/update_thread.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""Working thread for update.""" +from Qt.QtCore import QThread, Signal, QObject # noqa + +from .bootstrap_repos import ( + BootstrapRepos, + OpenPypeVersion +) + + +class UpdateThread(QThread): + """Install Worker thread. + + This class takes care of finding OpenPype version on user entered path + (or loading this path from database). If nothing is entered by user, + OpenPype will create its zip files from repositories that comes with it. + + If path contains plain repositories, they are zipped and installed to + user data dir. + + """ + progress = Signal(int) + message = Signal((str, bool)) + + def __init__(self, parent=None): + self._result = None + self._openpype_version = None + QThread.__init__(self, parent) + + def set_version(self, openpype_version: OpenPypeVersion): + self._openpype_version = openpype_version + + def result(self): + """Result of finished installation.""" + return self._result + + def _set_result(self, value): + if self._result is not None: + raise AssertionError("BUG: Result was set more than once!") + self._result = value + + def run(self): + """Thread entry point. + + Using :class:`BootstrapRepos` to either install OpenPype as zip files + or copy them from location specified by user or retrieved from + database. + """ + bs = BootstrapRepos( + progress_callback=self.set_progress, message=self.message) + version_path = bs.install_version(self._openpype_version) + self._set_result(version_path) + + def set_progress(self, progress: int) -> None: + """Helper to set progress bar. + + Args: + progress (int): Progress in percents. + + """ + self.progress.emit(progress) diff --git a/igniter/update_window.py b/igniter/update_window.py new file mode 100644 index 0000000000..d7908c240b --- /dev/null +++ b/igniter/update_window.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +"""Progress window to show when OpenPype is updating/installing locally.""" +import os +from .update_thread import UpdateThread +from Qt import QtCore, QtGui, QtWidgets # noqa +from .bootstrap_repos import OpenPypeVersion +from .nice_progress_bar import NiceProgressBar +from .tools import load_stylesheet + + +class UpdateWindow(QtWidgets.QDialog): + """OpenPype update window.""" + + _width = 500 + _height = 100 + + def __init__(self, version: OpenPypeVersion, parent=None): + super(UpdateWindow, self).__init__(parent) + self._openpype_version = version + self._result_version_path = None + + self.setWindowTitle( + f"OpenPype is updating ..." + ) + self.setModal(True) + self.setWindowFlags( + QtCore.Qt.WindowMinimizeButtonHint + ) + + current_dir = os.path.dirname(os.path.abspath(__file__)) + roboto_font_path = os.path.join(current_dir, "RobotoMono-Regular.ttf") + poppins_font_path = os.path.join(current_dir, "Poppins") + icon_path = os.path.join(current_dir, "openpype_icon.png") + + # Install roboto font + QtGui.QFontDatabase.addApplicationFont(roboto_font_path) + for filename in os.listdir(poppins_font_path): + if os.path.splitext(filename)[1] == ".ttf": + QtGui.QFontDatabase.addApplicationFont(filename) + + # Load logo + pixmap_openpype_logo = QtGui.QPixmap(icon_path) + # Set logo as icon of window + self.setWindowIcon(QtGui.QIcon(pixmap_openpype_logo)) + + self._pixmap_openpype_logo = pixmap_openpype_logo + + self._update_thread = None + + self.resize(QtCore.QSize(self._width, self._height)) + self._init_ui() + + # Set stylesheet + self.setStyleSheet(load_stylesheet()) + self._run_update() + + def _init_ui(self): + + # Main info + # -------------------------------------------------------------------- + main_label = QtWidgets.QLabel( + f"OpenPype is updating to {self._openpype_version}", self) + main_label.setWordWrap(True) + main_label.setObjectName("MainLabel") + + # Progress bar + # -------------------------------------------------------------------- + progress_bar = NiceProgressBar(self) + progress_bar.setAlignment(QtCore.Qt.AlignCenter) + progress_bar.setTextVisible(False) + + # add all to main + main = QtWidgets.QVBoxLayout(self) + main.addSpacing(15) + main.addWidget(main_label, 0) + main.addSpacing(15) + main.addWidget(progress_bar, 0) + main.addSpacing(15) + + self._progress_bar = progress_bar + + def _run_update(self): + """Start install process. + + This will once again validate entered path and mongo if ok, start + working thread that will do actual job. + """ + # Check if install thread is not already running + if self._update_thread and self._update_thread.isRunning(): + return + self._progress_bar.setRange(0, 0) + update_thread = UpdateThread(self) + update_thread.set_version(self._openpype_version) + update_thread.message.connect(self.update_console) + update_thread.progress.connect(self._update_progress) + update_thread.finished.connect(self._installation_finished) + + self._update_thread = update_thread + + update_thread.start() + + def get_version_path(self): + return self._result_version_path + + def _installation_finished(self): + status = self._update_thread.result() + self._result_version_path = status + self._progress_bar.setRange(0, 1) + self._update_progress(100) + QtWidgets.QApplication.processEvents() + self.done(0) + + def _update_progress(self, progress: int): + # not updating progress as we are not able to determine it + # correctly now. Progress bar is set to un-deterministic mode + # until we are able to get progress in better way. + """ + self._progress_bar.setRange(0, 0) + self._progress_bar.setValue(progress) + text_visible = self._progress_bar.isTextVisible() + if progress == 0: + if text_visible: + self._progress_bar.setTextVisible(False) + elif not text_visible: + self._progress_bar.setTextVisible(True) + """ + return + + def update_console(self, msg: str, error: bool = False) -> None: + """Display message in console. + + Args: + msg (str): message. + error (bool): if True, print it red. + """ + print(msg) diff --git a/openpype/api.py b/openpype/api.py index ce18097eca..e4bbb104a3 100644 --- a/openpype/api.py +++ b/openpype/api.py @@ -24,7 +24,9 @@ from .lib import ( get_latest_version, get_global_environments, get_local_site_id, - change_openpype_mongo_url + change_openpype_mongo_url, + create_project_folders, + get_project_basic_paths ) from .lib.mongo import ( @@ -72,6 +74,7 @@ __all__ = [ "get_current_project_settings", "get_anatomy_settings", "get_environments", + "get_project_basic_paths", "SystemSettings", @@ -120,5 +123,9 @@ __all__ = [ "get_global_environments", "get_local_site_id", - "change_openpype_mongo_url" + "change_openpype_mongo_url", + + "get_project_basic_paths", + "create_project_folders" + ] diff --git a/openpype/cli.py b/openpype/cli.py index c446d5e443..18cc1c63cd 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -18,6 +18,8 @@ from .pype_commands import PypeCommands @click.option("--list-versions", is_flag=True, expose_value=False, help=("list all detected versions. Use With `--use-staging " "to list staging versions.")) +@click.option("--validate-version", expose_value=False, + help="validate given version integrity") def main(ctx): """Pype is main command serving as entry point to pipeline system. diff --git a/openpype/hosts/nuke/plugins/publish/extract_thumbnail.py b/openpype/hosts/nuke/plugins/publish/extract_thumbnail.py index da30dcc632..55f7b746fc 100644 --- a/openpype/hosts/nuke/plugins/publish/extract_thumbnail.py +++ b/openpype/hosts/nuke/plugins/publish/extract_thumbnail.py @@ -112,24 +112,26 @@ class ExtractThumbnail(openpype.api.Extractor): # create write node write_node = nuke.createNode("Write") - file = fhead + "jpeg" + file = fhead + "jpg" name = "thumbnail" path = os.path.join(staging_dir, file).replace("\\", "/") instance.data["thumbnail"] = path write_node["file"].setValue(path) - write_node["file_type"].setValue("jpeg") + write_node["file_type"].setValue("jpg") write_node["raw"].setValue(1) write_node.setInput(0, previous_node) temporary_nodes.append(write_node) tags = ["thumbnail", "publish_on_farm"] # retime for + mid_frame = int((int(last_frame) - int(first_frame)) / 2) \ + + int(first_frame) first_frame = int(last_frame) / 2 last_frame = int(last_frame) / 2 repre = { 'name': name, - 'ext': "jpeg", + 'ext': "jpg", "outputName": "thumb", 'files': file, "stagingDir": staging_dir, @@ -140,7 +142,7 @@ class ExtractThumbnail(openpype.api.Extractor): instance.data["representations"].append(repre) # Render frames - nuke.execute(write_node.name(), int(first_frame), int(last_frame)) + nuke.execute(write_node.name(), int(mid_frame), int(mid_frame)) self.log.debug( "representations: {}".format(instance.data["representations"])) diff --git a/openpype/hosts/standalonepublisher/plugins/publish/extract_harmony_zip.py b/openpype/hosts/standalonepublisher/plugins/publish/extract_harmony_zip.py index f7f96c7d03..adbac6ef09 100644 --- a/openpype/hosts/standalonepublisher/plugins/publish/extract_harmony_zip.py +++ b/openpype/hosts/standalonepublisher/plugins/publish/extract_harmony_zip.py @@ -11,6 +11,7 @@ import zipfile import pyblish.api from avalon import api, io import openpype.api +from openpype.lib import get_workfile_template_key_from_context class ExtractHarmonyZip(openpype.api.Extractor): @@ -65,10 +66,10 @@ class ExtractHarmonyZip(openpype.api.Extractor): # Get Task types and Statuses for creation if needed self.task_types = self._get_all_task_types(project_entity) - self.task_statuses = self.get_all_task_statuses(project_entity) + self.task_statuses = self._get_all_task_statuses(project_entity) # Get Statuses of AssetVersions - self.assetversion_statuses = self.get_all_assetversion_statuses( + self.assetversion_statuses = self._get_all_assetversion_statuses( project_entity ) @@ -233,18 +234,28 @@ class ExtractHarmonyZip(openpype.api.Extractor): "version": 1, "ext": "zip", } + host_name = "harmony" + template_name = get_workfile_template_key_from_context( + instance.data["asset"], + instance.data.get("task"), + host_name, + project_name=project_entity["name"], + dbcon=io + ) # Get a valid work filename first with version 1 - file_template = anatomy.templates["work"]["file"] + file_template = anatomy.templates[template_name]["file"] anatomy_filled = anatomy.format(data) - work_path = anatomy_filled["work"]["path"] + work_path = anatomy_filled[template_name]["path"] # Get the final work filename with the proper version data["version"] = api.last_workfile_with_version( - os.path.dirname(work_path), file_template, data, [".zip"] + os.path.dirname(work_path), + file_template, + data, + api.HOST_WORKFILE_EXTENSIONS[host_name] )[1] - work_path = anatomy_filled["work"]["path"] base_name = os.path.splitext(os.path.basename(work_path))[0] staging_work_path = os.path.join(os.path.dirname(staging_scene), diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index 3d392dc745..9bc68c9558 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -143,7 +143,9 @@ from .plugin_tools import ( from .path_tools import ( version_up, get_version_from_path, - get_last_version_from_path + get_last_version_from_path, + create_project_folders, + get_project_basic_paths ) from .editorial import ( @@ -276,5 +278,7 @@ __all__ = [ "range_from_frames", "frames_to_secons", "frames_to_timecode", - "make_sequence_collection" + "make_sequence_collection", + "create_project_folders", + "get_project_basic_paths" ] diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index fbf991a32e..45b8e6468d 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -29,7 +29,7 @@ from .local_settings import get_openpype_username from .avalon_context import ( get_workdir_data, get_workdir_with_workdir_data, - get_workfile_template_key_from_context + get_workfile_template_key ) from .python_module_tools import ( @@ -1226,8 +1226,12 @@ def prepare_context_environments(data): # Load project specific environments project_name = project_doc["name"] + project_settings = get_project_settings(project_name) + data["project_settings"] = project_settings # Apply project specific environments on current env value - apply_project_environments_value(project_name, data["env"]) + apply_project_environments_value( + project_name, data["env"], project_settings + ) app = data["app"] workdir_data = get_workdir_data( @@ -1237,17 +1241,19 @@ def prepare_context_environments(data): anatomy = data["anatomy"] - template_key = get_workfile_template_key_from_context( - asset_doc["name"], - task_name, + asset_tasks = asset_doc.get("data", {}).get("tasks") or {} + task_info = asset_tasks.get(task_name) or {} + task_type = task_info.get("type") + workfile_template_key = get_workfile_template_key( + task_type, app.host_name, project_name=project_name, - dbcon=data["dbcon"] + project_settings=project_settings ) try: workdir = get_workdir_with_workdir_data( - workdir_data, anatomy, template_key=template_key + workdir_data, anatomy, template_key=workfile_template_key ) except Exception as exc: @@ -1281,10 +1287,10 @@ def prepare_context_environments(data): ) data["env"].update(context_env) - _prepare_last_workfile(data, workdir) + _prepare_last_workfile(data, workdir, workfile_template_key) -def _prepare_last_workfile(data, workdir): +def _prepare_last_workfile(data, workdir, workfile_template_key): """last workfile workflow preparation. Function check if should care about last workfile workflow and tries @@ -1345,7 +1351,7 @@ def _prepare_last_workfile(data, workdir): if extensions: anatomy = data["anatomy"] # Find last workfile - file_template = anatomy.templates["work"]["file"] + file_template = anatomy.templates[workfile_template_key]["file"] workdir_data.update({ "version": 1, "user": get_openpype_username(), diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index e1dd1e7f10..9dc14497a4 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -1,6 +1,11 @@ +import json +import logging import os import re -import logging + + +from .anatomy import Anatomy +from openpype.settings import get_project_settings log = logging.getLogger(__name__) @@ -77,7 +82,7 @@ def get_version_from_path(file): """ pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) try: - return pattern.findall(file)[0] + return pattern.findall(file)[-1] except IndexError: log.error( "templates:get_version_from_workfile:" @@ -119,3 +124,73 @@ def get_last_version_from_path(path_dir, filter): return filtred_files[-1] return None + + +def compute_paths(basic_paths_items, project_root): + pattern_array = re.compile(r"\[.*\]") + project_root_key = "__project_root__" + output = [] + for path_items in basic_paths_items: + clean_items = [] + for path_item in path_items: + matches = re.findall(pattern_array, path_item) + if len(matches) > 0: + path_item = path_item.replace(matches[0], "") + if path_item == project_root_key: + path_item = project_root + clean_items.append(path_item) + output.append(os.path.normpath(os.path.sep.join(clean_items))) + return output + + +def create_project_folders(basic_paths, project_name): + anatomy = Anatomy(project_name) + roots_paths = [] + if isinstance(anatomy.roots, dict): + for root in anatomy.roots.values(): + roots_paths.append(root.value) + else: + roots_paths.append(anatomy.roots.value) + + for root_path in roots_paths: + project_root = os.path.join(root_path, project_name) + full_paths = compute_paths(basic_paths, project_root) + # Create folders + for path in full_paths: + full_path = path.format(project_root=project_root) + if os.path.exists(full_path): + log.debug( + "Folder already exists: {}".format(full_path) + ) + else: + log.debug("Creating folder: {}".format(full_path)) + os.makedirs(full_path) + + +def _list_path_items(folder_structure): + output = [] + for key, value in folder_structure.items(): + if not value: + output.append(key) + else: + paths = _list_path_items(value) + for path in paths: + if not isinstance(path, (list, tuple)): + path = [path] + + output.append([key, *path]) + + return output + + +def get_project_basic_paths(project_name): + project_settings = get_project_settings(project_name) + folder_structure = ( + project_settings["global"]["project_folder_structure"] + ) + if not folder_structure: + return [] + + if isinstance(folder_structure, str): + folder_structure = json.loads(folder_structure) + return _list_path_items(folder_structure) diff --git a/openpype/lib/profiles_filtering.py b/openpype/lib/profiles_filtering.py index c4410204dd..992d757059 100644 --- a/openpype/lib/profiles_filtering.py +++ b/openpype/lib/profiles_filtering.py @@ -165,7 +165,8 @@ def filter_profiles(profiles_data, key_values, keys_order=None, logger=None): if match == -1: profile_value = profile.get(key) or [] logger.debug( - "\"{}\" not found in {}".format(key, profile_value) + "\"{}\" not found in \"{}\": {}".format(value, key, + profile_value) ) profile_points = -1 break @@ -192,13 +193,13 @@ def filter_profiles(profiles_data, key_values, keys_order=None, logger=None): ]) if not matching_profiles: - logger.warning( + logger.info( "None of profiles match your setup. {}".format(log_parts) ) return None if len(matching_profiles) > 1: - logger.warning( + logger.info( "More than one profile match your setup. {}".format(log_parts) ) diff --git a/openpype/modules/__init__.py b/openpype/modules/__init__.py index 583480b049..68b5f6c247 100644 --- a/openpype/modules/__init__.py +++ b/openpype/modules/__init__.py @@ -1,21 +1,35 @@ # -*- coding: utf-8 -*- from .base import ( OpenPypeModule, + OpenPypeAddOn, OpenPypeInterface, load_modules, ModulesManager, - TrayModulesManager + TrayModulesManager, + + BaseModuleSettingsDef, + ModuleSettingsDef, + JsonFilesSettingsDef, + + get_module_settings_defs ) __all__ = ( "OpenPypeModule", + "OpenPypeAddOn", "OpenPypeInterface", "load_modules", "ModulesManager", - "TrayModulesManager" + "TrayModulesManager", + + "BaseModuleSettingsDef", + "ModuleSettingsDef", + "JsonFilesSettingsDef", + + "get_module_settings_defs" ) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index d43d5635d1..01c3cebe60 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -2,9 +2,11 @@ """Base class for Pype Modules.""" import os import sys +import json import time import inspect import logging +import platform import threading import collections from uuid import uuid4 @@ -12,7 +14,18 @@ from abc import ABCMeta, abstractmethod import six import openpype -from openpype.settings import get_system_settings +from openpype.settings import ( + get_system_settings, + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS +) + +from openpype.settings.lib import ( + get_studio_system_settings_overrides, + load_json_file +) from openpype.lib import PypeLogger @@ -115,11 +128,51 @@ def get_default_modules_dir(): return os.path.join(current_dir, "default_modules") +def get_dynamic_modules_dirs(): + """Possible paths to OpenPype Addons of Modules. + + Paths are loaded from studio settings under: + `modules -> addon_paths -> {platform name}` + + Path may contain environment variable as a formatting string. + + They are not validated or checked their existence. + + Returns: + list: Paths loaded from studio overrides. + """ + output = [] + value = get_studio_system_settings_overrides() + for key in ("modules", "addon_paths", platform.system().lower()): + if key not in value: + return output + value = value[key] + + for path in value: + if not path: + continue + + try: + path = path.format(**os.environ) + except Exception: + pass + output.append(path) + return output + + def get_module_dirs(): """List of paths where OpenPype modules can be found.""" - dirpaths = [ - get_default_modules_dir() - ] + _dirpaths = [] + _dirpaths.append(get_default_modules_dir()) + _dirpaths.extend(get_dynamic_modules_dirs()) + + dirpaths = [] + for path in _dirpaths: + if not path: + continue + normalized = os.path.normpath(path) + if normalized not in dirpaths: + dirpaths.append(normalized) return dirpaths @@ -165,6 +218,9 @@ def _load_interfaces(): os.path.join(get_default_modules_dir(), "interfaces.py") ) for dirpath in dirpaths: + if not os.path.exists(dirpath): + continue + for filename in os.listdir(dirpath): if filename in ("__pycache__", ): continue @@ -272,12 +328,19 @@ def _load_modules(): # TODO add more logic how to define if folder is module or not # - check manifest and content of manifest - if os.path.isdir(fullpath): - import_module_from_dirpath(dirpath, filename, modules_key) + try: + if os.path.isdir(fullpath): + import_module_from_dirpath(dirpath, filename, modules_key) - elif ext in (".py", ): - module = import_filepath(fullpath) - setattr(openpype_modules, basename, module) + elif ext in (".py", ): + module = import_filepath(fullpath) + setattr(openpype_modules, basename, module) + + except Exception: + log.error( + "Failed to import '{}'.".format(fullpath), + exc_info=True + ) class _OpenPypeInterfaceMeta(ABCMeta): @@ -368,7 +431,16 @@ class OpenPypeModule: class OpenPypeAddOn(OpenPypeModule): - pass + # Enable Addon by default + enabled = True + + def initialize(self, module_settings): + """Initialization is not be required for most of addons.""" + pass + + def connect_with_modules(self, enabled_modules): + """Do not require to implement connection with modules for addon.""" + pass class ModulesManager: @@ -920,3 +992,424 @@ class TrayModulesManager(ModulesManager): ), exc_info=True ) + + +def get_module_settings_defs(): + """Check loaded addons/modules for existence of thei settings definition. + + Check if OpenPype addon/module as python module has class that inherit + from `ModuleSettingsDef` in python module variables (imported + in `__init__py`). + + Returns: + list: All valid and not abstract settings definitions from imported + openpype addons and modules. + """ + # Make sure modules are loaded + load_modules() + + import openpype_modules + + settings_defs = [] + + log = PypeLogger.get_logger("ModuleSettingsLoad") + + for raw_module in openpype_modules: + for attr_name in dir(raw_module): + attr = getattr(raw_module, attr_name) + if ( + not inspect.isclass(attr) + or attr is ModuleSettingsDef + or not issubclass(attr, ModuleSettingsDef) + ): + continue + + if inspect.isabstract(attr): + # Find missing implementations by convetion on `abc` module + not_implemented = [] + for attr_name in dir(attr): + attr = getattr(attr, attr_name, None) + abs_method = getattr( + attr, "__isabstractmethod__", None + ) + if attr and abs_method: + not_implemented.append(attr_name) + + # Log missing implementations + log.warning(( + "Skipping abstract Class: {} in module {}." + " Missing implementations: {}" + ).format( + attr_name, raw_module.__name__, ", ".join(not_implemented) + )) + continue + + settings_defs.append(attr) + + return settings_defs + + +@six.add_metaclass(ABCMeta) +class BaseModuleSettingsDef: + """Definition of settings for OpenPype module or AddOn.""" + _id = None + + @property + def id(self): + """ID created on initialization. + + ID should be per created object. Helps to store objects. + """ + if self._id is None: + self._id = uuid4() + return self._id + + @abstractmethod + def get_settings_schemas(self, schema_type): + """Setting schemas for passed schema type. + + These are main schemas by dynamic schema keys. If they're using + sub schemas or templates they should be loaded with + `get_dynamic_schemas`. + + Returns: + dict: Schema by `dynamic_schema` keys. + """ + pass + + @abstractmethod + def get_dynamic_schemas(self, schema_type): + """Settings schemas and templates that can be used anywhere. + + It is recommended to add prefix specific for addon/module to keys + (e.g. "my_addon/real_schema_name"). + + Returns: + dict: Schemas and templates by their keys. + """ + pass + + @abstractmethod + def get_defaults(self, top_key): + """Default values for passed top key. + + Top keys are (currently) "system_settings" or "project_settings". + + Should return exactly what was passed with `save_defaults`. + + Returns: + dict: Default values by path to first key in OpenPype defaults. + """ + pass + + @abstractmethod + def save_defaults(self, top_key, data): + """Save default values for passed top key. + + Top keys are (currently) "system_settings" or "project_settings". + + Passed data are by path to first key defined in main schemas. + """ + pass + + +class ModuleSettingsDef(BaseModuleSettingsDef): + """Settings definiton with separated system and procect settings parts. + + Reduce conditions that must be checked and adds predefined methods for + each case. + """ + def get_defaults(self, top_key): + """Split method into 2 methods by top key.""" + if top_key == SYSTEM_SETTINGS_KEY: + return self.get_default_system_settings() or {} + elif top_key == PROJECT_SETTINGS_KEY: + return self.get_default_project_settings() or {} + return {} + + def save_defaults(self, top_key, data): + """Split method into 2 methods by top key.""" + if top_key == SYSTEM_SETTINGS_KEY: + self.save_system_defaults(data) + elif top_key == PROJECT_SETTINGS_KEY: + self.save_project_defaults(data) + + def get_settings_schemas(self, schema_type): + """Split method into 2 methods by schema type.""" + if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS: + return self.get_system_settings_schemas() or {} + elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS: + return self.get_project_settings_schemas() or {} + return {} + + def get_dynamic_schemas(self, schema_type): + """Split method into 2 methods by schema type.""" + if schema_type == SCHEMA_KEY_SYSTEM_SETTINGS: + return self.get_system_dynamic_schemas() or {} + elif schema_type == SCHEMA_KEY_PROJECT_SETTINGS: + return self.get_project_dynamic_schemas() or {} + return {} + + @abstractmethod + def get_system_settings_schemas(self): + """Schemas and templates usable in system settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + pass + + @abstractmethod + def get_project_settings_schemas(self): + """Schemas and templates usable in project settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + pass + + @abstractmethod + def get_system_dynamic_schemas(self): + """System schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + pass + + @abstractmethod + def get_project_dynamic_schemas(self): + """Project schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + pass + + @abstractmethod + def get_default_system_settings(self): + """Default system settings values. + + Returns: + dict: Default values by path to first key. + """ + pass + + @abstractmethod + def get_default_project_settings(self): + """Default project settings values. + + Returns: + dict: Default values by path to first key. + """ + pass + + @abstractmethod + def save_system_defaults(self, data): + """Save default system settings values. + + Passed data are by path to first key defined in main schemas. + """ + pass + + @abstractmethod + def save_project_defaults(self, data): + """Save default project settings values. + + Passed data are by path to first key defined in main schemas. + """ + pass + + +class JsonFilesSettingsDef(ModuleSettingsDef): + """Preimplemented settings definition using json files and file structure. + + Expected file structure: + ┕ root + │ + │ # Default values + ┝ defaults + │ ┝ system_settings.json + │ ┕ project_settings.json + │ + │ # Schemas for `dynamic_template` type + ┝ dynamic_schemas + │ ┝ system_dynamic_schemas.json + │ ┕ project_dynamic_schemas.json + │ + │ # Schemas that can be used anywhere (enhancement for `dynamic_schemas`) + ┕ schemas + ┝ system_schemas + │ ┝ # Any schema or template files + │ ┕ ... + ┕ project_schemas + ┝ # Any schema or template files + ┕ ... + + Schemas can be loaded with prefix to avoid duplicated schema/template names + across all OpenPype addons/modules. Prefix can be defined with class + attribute `schema_prefix`. + + Only think which must be implemented in `get_settings_root_path` which + should return directory path to `root` (in structure graph above). + """ + # Possible way how to define `schemas` prefix + schema_prefix = "" + + @abstractmethod + def get_settings_root_path(self): + """Directory path where settings and it's schemas are located.""" + pass + + def __init__(self): + settings_root_dir = self.get_settings_root_path() + defaults_dir = os.path.join( + settings_root_dir, "defaults" + ) + dynamic_schemas_dir = os.path.join( + settings_root_dir, "dynamic_schemas" + ) + schemas_dir = os.path.join( + settings_root_dir, "schemas" + ) + + self.system_defaults_filepath = os.path.join( + defaults_dir, "system_settings.json" + ) + self.project_defaults_filepath = os.path.join( + defaults_dir, "project_settings.json" + ) + + self.system_dynamic_schemas_filepath = os.path.join( + dynamic_schemas_dir, "system_dynamic_schemas.json" + ) + self.project_dynamic_schemas_filepath = os.path.join( + dynamic_schemas_dir, "project_dynamic_schemas.json" + ) + + self.system_schemas_dir = os.path.join( + schemas_dir, "system_schemas" + ) + self.project_schemas_dir = os.path.join( + schemas_dir, "project_schemas" + ) + + def _load_json_file_data(self, path): + if os.path.exists(path): + return load_json_file(path) + return {} + + def get_default_system_settings(self): + """Default system settings values. + + Returns: + dict: Default values by path to first key. + """ + return self._load_json_file_data(self.system_defaults_filepath) + + def get_default_project_settings(self): + """Default project settings values. + + Returns: + dict: Default values by path to first key. + """ + return self._load_json_file_data(self.project_defaults_filepath) + + def _save_data_to_filepath(self, path, data): + dirpath = os.path.dirname(path) + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + with open(path, "w") as file_stream: + json.dump(data, file_stream, indent=4) + + def save_system_defaults(self, data): + """Save default system settings values. + + Passed data are by path to first key defined in main schemas. + """ + self._save_data_to_filepath(self.system_defaults_filepath, data) + + def save_project_defaults(self, data): + """Save default project settings values. + + Passed data are by path to first key defined in main schemas. + """ + self._save_data_to_filepath(self.project_defaults_filepath, data) + + def get_system_dynamic_schemas(self): + """System schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + return self._load_json_file_data(self.system_dynamic_schemas_filepath) + + def get_project_dynamic_schemas(self): + """Project schemas by dynamic schema name. + + If dynamic schema name is not available in then schema will not used. + + Returns: + dict: Schemas or list of schemas by dynamic schema name. + """ + return self._load_json_file_data(self.project_dynamic_schemas_filepath) + + def _load_files_from_path(self, path): + output = {} + if not path or not os.path.exists(path): + return output + + if os.path.isfile(path): + filename = os.path.basename(path) + basename, ext = os.path.splitext(filename) + if ext == ".json": + if self.schema_prefix: + key = "{}/{}".format(self.schema_prefix, basename) + else: + key = basename + output[key] = self._load_json_file_data(path) + return output + + path = os.path.normpath(path) + for root, _, files in os.walk(path, topdown=False): + for filename in files: + basename, ext = os.path.splitext(filename) + if ext != ".json": + continue + + json_path = os.path.join(root, filename) + store_key = os.path.join( + root.replace(path, ""), basename + ).replace("\\", "/") + if self.schema_prefix: + store_key = "{}/{}".format(self.schema_prefix, store_key) + output[store_key] = self._load_json_file_data(json_path) + + return output + + def get_system_settings_schemas(self): + """Schemas and templates usable in system settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + return self._load_files_from_path(self.system_schemas_dir) + + def get_project_settings_schemas(self): + """Schemas and templates usable in project settings schemas. + + Returns: + dict: Schemas and templates by it's names. Names must be unique + across whole OpenPype. + """ + return self._load_files_from_path(self.project_schemas_dir) diff --git a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py index 121c9f652b..94f359c317 100644 --- a/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py +++ b/openpype/modules/default_modules/ftrack/event_handlers_user/action_create_project_structure.py @@ -3,7 +3,7 @@ import re import json from openpype_modules.ftrack.lib import BaseAction, statics_icon -from openpype.api import Anatomy, get_project_settings +from openpype.api import get_project_basic_paths, create_project_folders class CreateProjectFolders(BaseAction): @@ -72,25 +72,18 @@ class CreateProjectFolders(BaseAction): def launch(self, session, entities, event): # Get project entity project_entity = self.get_project_from_entity(entities[0]) - # Load settings for project project_name = project_entity["full_name"] - project_settings = get_project_settings(project_name) - project_folder_structure = ( - project_settings["global"]["project_folder_structure"] - ) - if not project_folder_structure: - return { - "success": False, - "message": "Project structure is not set." - } - try: - if isinstance(project_folder_structure, str): - project_folder_structure = json.loads(project_folder_structure) - # Get paths based on presets - basic_paths = self.get_path_items(project_folder_structure) - self.create_folders(basic_paths, project_entity) + basic_paths = get_project_basic_paths(project_name) + if not basic_paths: + return { + "success": False, + "message": "Project structure is not set." + } + + # Invoking OpenPype API to create the project folders + create_project_folders(basic_paths, project_name) self.create_ftrack_entities(basic_paths, project_entity) except Exception as exc: @@ -195,58 +188,6 @@ class CreateProjectFolders(BaseAction): self.session.commit() return new_ent - def get_path_items(self, in_dict): - output = [] - for key, value in in_dict.items(): - if not value: - output.append(key) - else: - paths = self.get_path_items(value) - for path in paths: - if not isinstance(path, (list, tuple)): - path = [path] - - output.append([key, *path]) - - return output - - def compute_paths(self, basic_paths_items, project_root): - output = [] - for path_items in basic_paths_items: - clean_items = [] - for path_item in path_items: - matches = re.findall(self.pattern_array, path_item) - if len(matches) > 0: - path_item = path_item.replace(matches[0], "") - if path_item == self.project_root_key: - path_item = project_root - clean_items.append(path_item) - output.append(os.path.normpath(os.path.sep.join(clean_items))) - return output - - def create_folders(self, basic_paths, project): - anatomy = Anatomy(project["full_name"]) - roots_paths = [] - if isinstance(anatomy.roots, dict): - for root in anatomy.roots.values(): - roots_paths.append(root.value) - else: - roots_paths.append(anatomy.roots.value) - - for root_path in roots_paths: - project_root = os.path.join(root_path, project["full_name"]) - full_paths = self.compute_paths(basic_paths, project_root) - # Create folders - for path in full_paths: - full_path = path.format(project_root=project_root) - if os.path.exists(full_path): - self.log.debug( - "Folder already exists: {}".format(full_path) - ) - else: - self.log.debug("Creating folder: {}".format(full_path)) - os.makedirs(full_path) - def register(session): CreateProjectFolders(session).register() diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py index cc2a5b7d37..70030acad9 100644 --- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py +++ b/openpype/modules/default_modules/ftrack/plugins/publish/collect_ftrack_family.py @@ -68,9 +68,6 @@ class CollectFtrackFamily(pyblish.api.InstancePlugin): instance.data["families"].append("ftrack") else: instance.data["families"] = ["ftrack"] - else: - self.log.debug("Instance '{}' doesn't match any profile".format( - instance.data.get("family"))) def _get_add_ftrack_f_from_addit_filters(self, additional_filters, diff --git a/openpype/modules/default_modules/sync_server/providers/abstract_provider.py b/openpype/modules/default_modules/sync_server/providers/abstract_provider.py index 2e9632134c..7fd25b9852 100644 --- a/openpype/modules/default_modules/sync_server/providers/abstract_provider.py +++ b/openpype/modules/default_modules/sync_server/providers/abstract_provider.py @@ -29,13 +29,35 @@ class AbstractProvider: @classmethod @abc.abstractmethod - def get_configurable_items(cls): + def get_system_settings_schema(cls): """ - Returns filtered dict of editable properties + Returns dict for editable properties on system settings level Returns: - (dict) + (list) of dict + """ + + @classmethod + @abc.abstractmethod + def get_project_settings_schema(cls): + """ + Returns dict for editable properties on project settings level + + + Returns: + (list) of dict + """ + + @classmethod + @abc.abstractmethod + def get_local_settings_schema(cls): + """ + Returns dict for editable properties on local settings level + + + Returns: + (list) of dict """ @abc.abstractmethod diff --git a/openpype/modules/default_modules/sync_server/providers/gdrive.py b/openpype/modules/default_modules/sync_server/providers/gdrive.py index 18d679b833..f1ec0b6a0d 100644 --- a/openpype/modules/default_modules/sync_server/providers/gdrive.py +++ b/openpype/modules/default_modules/sync_server/providers/gdrive.py @@ -8,7 +8,7 @@ import platform from openpype.api import Logger from openpype.api import get_system_settings from .abstract_provider import AbstractProvider -from ..utils import time_function, ResumableError, EditableScopes +from ..utils import time_function, ResumableError log = Logger().get_logger("SyncServer") @@ -96,30 +96,61 @@ class GDriveHandler(AbstractProvider): return self.service is not None @classmethod - def get_configurable_items(cls): + def get_system_settings_schema(cls): """ - Returns filtered dict of editable properties. + Returns dict for editable properties on system settings level + + + Returns: + (list) of dict + """ + return [] + + @classmethod + def get_project_settings_schema(cls): + """ + Returns dict for editable properties on project settings level + + + Returns: + (list) of dict + """ + # {platform} tells that value is multiplatform and only specific OS + # should be returned + editable = [ + # credentials could be overriden on Project or User level + { + 'key': "credentials_url", + 'label': "Credentials url", + 'type': 'text' + }, + # roots could be overriden only on Project leve, User cannot + { + 'key': "roots", + 'label': "Roots", + 'type': 'dict' + } + ] + return editable + + @classmethod + def get_local_settings_schema(cls): + """ + Returns dict for editable properties on local settings level Returns: (dict) """ - # {platform} tells that value is multiplatform and only specific OS - # should be returned - editable = { + editable = [ # credentials could be override on Project or User level - 'credentials_url': { - 'scope': [EditableScopes.PROJECT, - EditableScopes.LOCAL], + { + 'key': "credentials_url", 'label': "Credentials url", 'type': 'text', 'namespace': '{project_settings}/global/sync_server/sites/{site}/credentials_url/{platform}' # noqa: E501 - }, - # roots could be override only on Project leve, User cannot - 'root': {'scope': [EditableScopes.PROJECT], - 'label': "Roots", - 'type': 'dict'} - } + } + ] return editable def get_roots_config(self, anatomy=None): diff --git a/openpype/modules/default_modules/sync_server/providers/lib.py b/openpype/modules/default_modules/sync_server/providers/lib.py index 816ccca981..463e49dd4d 100644 --- a/openpype/modules/default_modules/sync_server/providers/lib.py +++ b/openpype/modules/default_modules/sync_server/providers/lib.py @@ -76,6 +76,14 @@ class ProviderFactory: return provider_info[0].get_configurable_items() + def get_provider_cls(self, provider_code): + """ + Returns class object for 'provider_code' to run class methods on. + """ + provider_info = self._get_creator_info(provider_code) + + return provider_info[0] + def _get_creator_info(self, provider): """ Collect all necessary info for provider. Currently only creator diff --git a/openpype/modules/default_modules/sync_server/providers/local_drive.py b/openpype/modules/default_modules/sync_server/providers/local_drive.py index 4b80ed44f2..8e5f170bc9 100644 --- a/openpype/modules/default_modules/sync_server/providers/local_drive.py +++ b/openpype/modules/default_modules/sync_server/providers/local_drive.py @@ -7,8 +7,6 @@ import time from openpype.api import Logger, Anatomy from .abstract_provider import AbstractProvider -from ..utils import EditableScopes - log = Logger().get_logger("SyncServer") @@ -30,18 +28,51 @@ class LocalDriveHandler(AbstractProvider): return True @classmethod - def get_configurable_items(cls): + def get_system_settings_schema(cls): """ - Returns filtered dict of editable properties + Returns dict for editable properties on system settings level + + + Returns: + (list) of dict + """ + return [] + + @classmethod + def get_project_settings_schema(cls): + """ + Returns dict for editable properties on project settings level + + + Returns: + (list) of dict + """ + # for non 'studio' sites, 'studio' is configured in Anatomy + editable = [ + { + 'key': "roots", + 'label': "Roots", + 'type': 'dict' + } + ] + return editable + + @classmethod + def get_local_settings_schema(cls): + """ + Returns dict for editable properties on local settings level + Returns: (dict) """ - editable = { - 'root': {'scope': [EditableScopes.LOCAL], - 'label': "Roots", - 'type': 'dict'} - } + editable = [ + { + 'key': "roots", + 'label': "Roots", + 'type': 'dict' + } + ] return editable def upload_file(self, source_path, target_path, diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index e65a410551..976a349bfa 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -16,14 +16,13 @@ from openpype.api import ( get_local_site_id) from openpype.lib import PypeLogger from openpype.settings.lib import ( - get_default_project_settings, get_default_anatomy_settings, get_anatomy_settings) from .providers.local_drive import LocalDriveHandler from .providers import lib -from .utils import time_function, SyncStatus, EditableScopes +from .utils import time_function, SyncStatus log = PypeLogger().get_logger("SyncServer") @@ -399,204 +398,239 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return remote_site - def get_local_settings_schema(self): - """Wrapper for Local settings - all projects incl. Default""" - return self.get_configurable_items(EditableScopes.LOCAL) + # Methods for Settings UI to draw appropriate forms + @classmethod + def get_system_settings_schema(cls): + """ Gets system level schema of configurable items - def get_configurable_items(self, scope=None): + Used for Setting UI to provide forms. """ - Returns list of sites that could be configurable for all projects. + ret_dict = {} + for provider_code in lib.factory.providers: + ret_dict[provider_code] = \ + lib.factory.get_provider_cls(provider_code). \ + get_system_settings_schema() - Could be filtered by 'scope' argument (list) + return ret_dict - Args: - scope (list of utils.EditableScope) + @classmethod + def get_project_settings_schema(cls): + """ Gets project level schema of configurable items. - Returns: - (dict of list of dict) - { - siteA : [ - { - key:"root", label:"root", - "value":"{'work': 'c:/projects'}", - "type": "dict", - "children":[ - { "key": "work", - "type": "text", - "value": "c:/projects"} - ] - }, - { - key:"credentials_url", label:"Credentials url", - "value":"'c:/projects/cred.json'", "type": "text", - "namespace": "{project_setting}/global/sync_server/ - sites" - } - ] - } + It is not using Setting! Used for Setting UI to provide forms. """ - editable = {} - applicable_projects = list(self.connection.projects()) - applicable_projects.append(None) - for project in applicable_projects: - project_name = None - if project: - project_name = project["name"] + ret_dict = {} + for provider_code in lib.factory.providers: + ret_dict[provider_code] = \ + lib.factory.get_provider_cls(provider_code). \ + get_project_settings_schema() - items = self.get_configurable_items_for_project(project_name, - scope) - editable.update(items) + return ret_dict - return editable + @classmethod + def get_local_settings_schema(cls): + """ Gets local level schema of configurable items. - def get_local_settings_schema_for_project(self, project_name): - """Wrapper for Local settings - for specific 'project_name'""" - return self.get_configurable_items_for_project(project_name, - EditableScopes.LOCAL) - - def get_configurable_items_for_project(self, project_name=None, - scope=None): + It is not using Setting! Used for Setting UI to provide forms. """ - Returns list of items that could be configurable for specific - 'project_name' + ret_dict = {} + for provider_code in lib.factory.providers: + ret_dict[provider_code] = \ + lib.factory.get_provider_cls(provider_code). \ + get_local_settings_schema() - Args: - project_name (str) - None > default project, - scope (list of utils.EditableScope) - (optional, None is all scopes, default is LOCAL) + return ret_dict - Returns: - (dict of list of dict) - { - siteA : [ - { - key:"root", label:"root", - "type": "dict", - "children":[ - { "key": "work", - "type": "text", - "value": "c:/projects"} - ] - }, - { - key:"credentials_url", label:"Credentials url", - "value":"'c:/projects/cred.json'", "type": "text", - "namespace": "{project_setting}/global/sync_server/ - sites" - } - ] - } - """ - allowed_sites = set() - sites = self.get_all_site_configs(project_name) - if project_name: - # Local Settings can select only from allowed sites for project - allowed_sites.update(set(self.get_active_sites(project_name))) - allowed_sites.update(set(self.get_remote_sites(project_name))) - - editable = {} - for site_name in sites.keys(): - if allowed_sites and site_name not in allowed_sites: - continue - - items = self.get_configurable_items_for_site(project_name, - site_name, - scope) - # Local Settings need 'local' instead of real value - site_name = site_name.replace(get_local_site_id(), 'local') - editable[site_name] = items - - return editable - - def get_local_settings_schema_for_site(self, project_name, site_name): - """Wrapper for Local settings - for particular 'site_name and proj.""" - return self.get_configurable_items_for_site(project_name, - site_name, - EditableScopes.LOCAL) - - def get_configurable_items_for_site(self, project_name=None, - site_name=None, - scope=None): - """ - Returns list of items that could be configurable. - - Args: - project_name (str) - None > default project - site_name (str) - scope (list of utils.EditableScope) - (optional, None is all scopes) - - Returns: - (list) - [ - { - key:"root", label:"root", type:"dict", - "children":[ - { "key": "work", - "type": "text", - "value": "c:/projects"} - ] - }, ... - ] - """ - provider_name = self.get_provider_for_site(site=site_name) - items = lib.factory.get_provider_configurable_items(provider_name) - - if project_name: - sync_s = self.get_sync_project_setting(project_name, - exclude_locals=True, - cached=False) - else: - sync_s = get_default_project_settings(exclude_locals=True) - sync_s = sync_s["global"]["sync_server"] - sync_s["sites"].update( - self._get_default_site_configs(self.enabled)) - - editable = [] - if type(scope) is not list: - scope = [scope] - scope = set(scope) - for key, properties in items.items(): - if scope is None or scope.intersection(set(properties["scope"])): - val = sync_s.get("sites", {}).get(site_name, {}).get(key) - - item = { - "key": key, - "label": properties["label"], - "type": properties["type"] - } - - if properties.get("namespace"): - item["namespace"] = properties.get("namespace") - if "platform" in item["namespace"]: - try: - if val: - val = val[platform.system().lower()] - except KeyError: - st = "{}'s field value {} should be".format(key, val) # noqa: E501 - log.error(st + " multiplatform dict") - - item["namespace"] = item["namespace"].replace('{site}', - site_name) - children = [] - if properties["type"] == "dict": - if val: - for val_key, val_val in val.items(): - child = { - "type": "text", - "key": val_key, - "value": val_val - } - children.append(child) - - if properties["type"] == "dict": - item["children"] = children - else: - item["value"] = val - - editable.append(item) - - return editable + # Needs to be refactored after Settings are updated + # # Methods for Settings to get appriate values to fill forms + # def get_configurable_items(self, scope=None): + # """ + # Returns list of sites that could be configurable for all projects + # + # Could be filtered by 'scope' argument (list) + # + # Args: + # scope (list of utils.EditableScope) + # + # Returns: + # (dict of list of dict) + # { + # siteA : [ + # { + # key:"root", label:"root", + # "value":"{'work': 'c:/projects'}", + # "type": "dict", + # "children":[ + # { "key": "work", + # "type": "text", + # "value": "c:/projects"} + # ] + # }, + # { + # key:"credentials_url", label:"Credentials url", + # "value":"'c:/projects/cred.json'", "type": "text", # noqa: E501 + # "namespace": "{project_setting}/global/sync_server/ # noqa: E501 + # sites" + # } + # ] + # } + # """ + # editable = {} + # applicable_projects = list(self.connection.projects()) + # applicable_projects.append(None) + # for project in applicable_projects: + # project_name = None + # if project: + # project_name = project["name"] + # + # items = self.get_configurable_items_for_project(project_name, + # scope) + # editable.update(items) + # + # return editable + # + # def get_local_settings_schema_for_project(self, project_name): + # """Wrapper for Local settings - for specific 'project_name'""" + # return self.get_configurable_items_for_project(project_name, + # EditableScopes.LOCAL) + # + # def get_configurable_items_for_project(self, project_name=None, + # scope=None): + # """ + # Returns list of items that could be configurable for specific + # 'project_name' + # + # Args: + # project_name (str) - None > default project, + # scope (list of utils.EditableScope) + # (optional, None is all scopes, default is LOCAL) + # + # Returns: + # (dict of list of dict) + # { + # siteA : [ + # { + # key:"root", label:"root", + # "type": "dict", + # "children":[ + # { "key": "work", + # "type": "text", + # "value": "c:/projects"} + # ] + # }, + # { + # key:"credentials_url", label:"Credentials url", + # "value":"'c:/projects/cred.json'", "type": "text", + # "namespace": "{project_setting}/global/sync_server/ + # sites" + # } + # ] + # } + # """ + # allowed_sites = set() + # sites = self.get_all_site_configs(project_name) + # if project_name: + # # Local Settings can select only from allowed sites for project + # allowed_sites.update(set(self.get_active_sites(project_name))) + # allowed_sites.update(set(self.get_remote_sites(project_name))) + # + # editable = {} + # for site_name in sites.keys(): + # if allowed_sites and site_name not in allowed_sites: + # continue + # + # items = self.get_configurable_items_for_site(project_name, + # site_name, + # scope) + # # Local Settings need 'local' instead of real value + # site_name = site_name.replace(get_local_site_id(), 'local') + # editable[site_name] = items + # + # return editable + # + # def get_configurable_items_for_site(self, project_name=None, + # site_name=None, + # scope=None): + # """ + # Returns list of items that could be configurable. + # + # Args: + # project_name (str) - None > default project + # site_name (str) + # scope (list of utils.EditableScope) + # (optional, None is all scopes) + # + # Returns: + # (list) + # [ + # { + # key:"root", label:"root", type:"dict", + # "children":[ + # { "key": "work", + # "type": "text", + # "value": "c:/projects"} + # ] + # }, ... + # ] + # """ + # provider_name = self.get_provider_for_site(site=site_name) + # items = lib.factory.get_provider_configurable_items(provider_name) + # + # if project_name: + # sync_s = self.get_sync_project_setting(project_name, + # exclude_locals=True, + # cached=False) + # else: + # sync_s = get_default_project_settings(exclude_locals=True) + # sync_s = sync_s["global"]["sync_server"] + # sync_s["sites"].update( + # self._get_default_site_configs(self.enabled)) + # + # editable = [] + # if type(scope) is not list: + # scope = [scope] + # scope = set(scope) + # for key, properties in items.items(): + # if scope is None or scope.intersection(set(properties["scope"])): + # val = sync_s.get("sites", {}).get(site_name, {}).get(key) + # + # item = { + # "key": key, + # "label": properties["label"], + # "type": properties["type"] + # } + # + # if properties.get("namespace"): + # item["namespace"] = properties.get("namespace") + # if "platform" in item["namespace"]: + # try: + # if val: + # val = val[platform.system().lower()] + # except KeyError: + # st = "{}'s field value {} should be".format(key, val) # noqa: E501 + # log.error(st + " multiplatform dict") + # + # item["namespace"] = item["namespace"].replace('{site}', + # site_name) + # children = [] + # if properties["type"] == "dict": + # if val: + # for val_key, val_val in val.items(): + # child = { + # "type": "text", + # "key": val_key, + # "value": val_val + # } + # children.append(child) + # + # if properties["type"] == "dict": + # item["children"] = children + # else: + # item["value"] = val + # + # editable.append(item) + # + # return editable def reset_timer(self): """ @@ -611,7 +645,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): enabled_projects = [] if self.enabled: - for project in self.connection.projects(): + for project in self.connection.projects(projection={"name": 1}): project_name = project["name"] project_settings = self.get_sync_project_setting(project_name) if project_settings and project_settings.get("enabled"): diff --git a/openpype/plugins/publish/stop_timer.py b/openpype/plugins/publish/stop_timer.py index 81afd16378..5c939b5f1b 100644 --- a/openpype/plugins/publish/stop_timer.py +++ b/openpype/plugins/publish/stop_timer.py @@ -8,7 +8,7 @@ from openpype.api import get_system_settings class StopTimer(pyblish.api.ContextPlugin): label = "Stop Timer" - order = pyblish.api.ExtractorOrder - 0.5 + order = pyblish.api.ExtractorOrder - 0.49 hosts = ["*"] def process(self, context): diff --git a/openpype/settings/__init__.py b/openpype/settings/__init__.py index b5810deef4..74f2684b2a 100644 --- a/openpype/settings/__init__.py +++ b/openpype/settings/__init__.py @@ -1,7 +1,21 @@ +from .constants import ( + GLOBAL_SETTINGS_KEY, + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + PROJECT_ANATOMY_KEY, + LOCAL_SETTING_KEY, + + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS, + + KEY_ALLOWED_SYMBOLS, + KEY_REGEX +) from .exceptions import ( SaveWarningExc ) from .lib import ( + get_general_environments, get_system_settings, get_project_settings, get_current_project_settings, @@ -16,15 +30,27 @@ from .entities import ( __all__ = ( + "GLOBAL_SETTINGS_KEY", + "SYSTEM_SETTINGS_KEY", + "PROJECT_SETTINGS_KEY", + "PROJECT_ANATOMY_KEY", + "LOCAL_SETTING_KEY", + + "SCHEMA_KEY_SYSTEM_SETTINGS", + "SCHEMA_KEY_PROJECT_SETTINGS", + + "KEY_ALLOWED_SYMBOLS", + "KEY_REGEX", + "SaveWarningExc", + "get_general_environments", "get_system_settings", "get_project_settings", "get_current_project_settings", "get_anatomy_settings", "get_environments", "get_local_settings", - "SystemSettings", "ProjectSettings" ) diff --git a/openpype/settings/constants.py b/openpype/settings/constants.py index a53e88a91e..2ea19ead4b 100644 --- a/openpype/settings/constants.py +++ b/openpype/settings/constants.py @@ -14,13 +14,17 @@ METADATA_KEYS = ( M_DYNAMIC_KEY_LABEL ) -# File where studio's system overrides are stored +# Keys where studio's system overrides are stored GLOBAL_SETTINGS_KEY = "global_settings" SYSTEM_SETTINGS_KEY = "system_settings" PROJECT_SETTINGS_KEY = "project_settings" PROJECT_ANATOMY_KEY = "project_anatomy" LOCAL_SETTING_KEY = "local_settings" +# Schema hub names +SCHEMA_KEY_SYSTEM_SETTINGS = "system_schema" +SCHEMA_KEY_PROJECT_SETTINGS = "projects_schema" + DEFAULT_PROJECT_KEY = "__default_project__" KEY_ALLOWED_SYMBOLS = "a-zA-Z0-9-_ " @@ -39,6 +43,9 @@ __all__ = ( "PROJECT_ANATOMY_KEY", "LOCAL_SETTING_KEY", + "SCHEMA_KEY_SYSTEM_SETTINGS", + "SCHEMA_KEY_PROJECT_SETTINGS", + "DEFAULT_PROJECT_KEY", "KEY_ALLOWED_SYMBOLS", diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index a0ba607edc..229b867327 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -1,4 +1,9 @@ { + "addon_paths": { + "windows": [], + "darwin": [], + "linux": [] + }, "avalon": { "AVALON_TIMEOUT": 1000, "AVALON_THUMBNAIL_ROOT": { diff --git a/openpype/settings/entities/__init__.py b/openpype/settings/entities/__init__.py index 8c30d5044c..aae2d1fa89 100644 --- a/openpype/settings/entities/__init__.py +++ b/openpype/settings/entities/__init__.py @@ -105,7 +105,6 @@ from .enum_entity import ( AppsEnumEntity, ToolsEnumEntity, TaskTypeEnumEntity, - ProvidersEnum, DeadlineUrlEnumEntity, AnatomyTemplatesEnumEntity ) @@ -113,7 +112,10 @@ from .enum_entity import ( from .list_entity import ListEntity from .dict_immutable_keys_entity import DictImmutableKeysEntity from .dict_mutable_keys_entity import DictMutableKeysEntity -from .dict_conditional import DictConditionalEntity +from .dict_conditional import ( + DictConditionalEntity, + SyncServerProviders +) from .anatomy_entities import AnatomyEntity @@ -161,7 +163,6 @@ __all__ = ( "AppsEnumEntity", "ToolsEnumEntity", "TaskTypeEnumEntity", - "ProvidersEnum", "DeadlineUrlEnumEntity", "AnatomyTemplatesEnumEntity", @@ -172,6 +173,7 @@ __all__ = ( "DictMutableKeysEntity", "DictConditionalEntity", + "SyncServerProviders", "AnatomyEntity" ) diff --git a/openpype/settings/entities/base_entity.py b/openpype/settings/entities/base_entity.py index 851684520b..0e8274d374 100644 --- a/openpype/settings/entities/base_entity.py +++ b/openpype/settings/entities/base_entity.py @@ -104,6 +104,12 @@ class BaseItemEntity(BaseEntity): self.is_group = False # Entity's value will be stored into file with name of it's key self.is_file = False + # Default values are not stored to an openpype file + # - these must not be set through schemas directly + self.dynamic_schema_id = None + self.is_dynamic_schema_node = False + self.is_in_dynamic_schema_node = False + # Reference to parent entity which has `is_group` == True # - stays as None if none of parents is group self.group_item = None @@ -255,13 +261,22 @@ class BaseItemEntity(BaseEntity): ) # Group item can be only once in on hierarchy branch. - if self.is_group and self.group_item: + if self.is_group and self.group_item is not None: raise SchemeGroupHierarchyBug(self) + # Group item can be only once in on hierarchy branch. + if self.group_item is not None and self.is_dynamic_schema_node: + reason = ( + "Dynamic schema is inside grouped item {}." + " Change group hierarchy or remove dynamic" + " schema to be able work properly." + ).format(self.group_item.path) + raise EntitySchemaError(self, reason) + # Validate that env group entities will be stored into file. # - env group entities must store metadata which is not possible if # metadata would be outside of file - if not self.file_item and self.is_env_group: + if self.file_item is None and self.is_env_group: reason = ( "Environment item is not inside file" " item so can't store metadata for defaults." @@ -478,7 +493,15 @@ class BaseItemEntity(BaseEntity): @abstractmethod def settings_value(self): - """Value of an item without key.""" + """Value of an item without key without dynamic items.""" + pass + + @abstractmethod + def collect_dynamic_schema_entities(self): + """Collect entities that are on top of dynamically added schemas. + + This method make sence only when defaults are saved. + """ pass @abstractmethod @@ -808,6 +831,12 @@ class ItemEntity(BaseItemEntity): self.is_dynamic_item = is_dynamic_item self.is_file = self.schema_data.get("is_file", False) + # These keys have underscore as they must not be set in schemas + self.dynamic_schema_id = self.schema_data.get( + "_dynamic_schema_id", None + ) + self.is_dynamic_schema_node = self.dynamic_schema_id is not None + self.is_group = self.schema_data.get("is_group", False) self.is_in_dynamic_item = bool( not self.is_dynamic_item @@ -837,10 +866,20 @@ class ItemEntity(BaseItemEntity): self._require_restart_on_change = require_restart_on_change # File item reference - if self.parent.is_file: - self.file_item = self.parent - elif self.parent.file_item: - self.file_item = self.parent.file_item + if not self.is_dynamic_schema_node: + self.is_in_dynamic_schema_node = ( + self.parent.is_dynamic_schema_node + or self.parent.is_in_dynamic_schema_node + ) + + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + ): + if self.parent.is_file: + self.file_item = self.parent + elif self.parent.file_item: + self.file_item = self.parent.file_item # Group item reference if self.parent.is_group: @@ -891,6 +930,18 @@ class ItemEntity(BaseItemEntity): def root_key(self): return self.root_item.root_key + @abstractmethod + def collect_dynamic_schema_entities(self, collector): + """Collect entities that are on top of dynamically added schemas. + + This method make sence only when defaults are saved. + + Args: + collector(DynamicSchemaValueCollector): Object where dynamic + entities are stored. + """ + pass + def schema_validations(self): if not self.label and self.use_label_wrap: reason = ( @@ -899,7 +950,12 @@ class ItemEntity(BaseItemEntity): ) raise EntitySchemaError(self, reason) - if self.is_file and self.file_item is not None: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and self.is_file + and self.file_item is not None + ): reason = ( "Entity has set `is_file` to true but" " it's parent is already marked as file item." diff --git a/openpype/settings/entities/dict_conditional.py b/openpype/settings/entities/dict_conditional.py index 988464d059..6f27760570 100644 --- a/openpype/settings/entities/dict_conditional.py +++ b/openpype/settings/entities/dict_conditional.py @@ -469,6 +469,10 @@ class DictConditionalEntity(ItemEntity): return True return False + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET @@ -482,13 +486,7 @@ class DictConditionalEntity(ItemEntity): output = {} for key, child_obj in children_items: - child_value = child_obj.settings_value() - if not child_obj.is_file and not child_obj.file_item: - for _key, _value in child_value.items(): - new_key = "/".join([key, _key]) - output[new_key] = _value - else: - output[key] = child_value + output[key] = child_obj.settings_value() return output if self.is_group: @@ -726,3 +724,49 @@ class DictConditionalEntity(ItemEntity): for children in self.children.values(): for child_entity in children: child_entity.reset_callbacks() + + +class SyncServerProviders(DictConditionalEntity): + schema_types = ["sync-server-providers"] + + def _add_children(self): + self.enum_key = "provider" + self.enum_label = "Provider" + + enum_children = self._get_enum_children() + if not enum_children: + enum_children.append({ + "key": None, + "label": "< Nothing >" + }) + self.enum_children = enum_children + + super(SyncServerProviders, self)._add_children() + + def _get_enum_children(self): + from openpype_modules import sync_server + + from openpype_modules.sync_server.providers import lib as lib_providers + + provider_code_to_label = {} + providers = lib_providers.factory.providers + for provider_code, provider_info in providers.items(): + provider, _ = provider_info + provider_code_to_label[provider_code] = provider.LABEL + + system_settings_schema = ( + sync_server + .SyncServerModule + .get_system_settings_schema() + ) + + enum_children = [] + for provider_code, configurables in system_settings_schema.items(): + label = provider_code_to_label.get(provider_code) or provider_code + + enum_children.append({ + "key": provider_code, + "label": label, + "children": configurables + }) + return enum_children diff --git a/openpype/settings/entities/dict_immutable_keys_entity.py b/openpype/settings/entities/dict_immutable_keys_entity.py index 73b08f101a..57e21ff5f3 100644 --- a/openpype/settings/entities/dict_immutable_keys_entity.py +++ b/openpype/settings/entities/dict_immutable_keys_entity.py @@ -330,15 +330,32 @@ class DictImmutableKeysEntity(ItemEntity): return True return False + def collect_dynamic_schema_entities(self, collector): + for child_obj in self.non_gui_children.values(): + child_obj.collect_dynamic_schema_entities(collector) + + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET if self._override_state is OverrideState.DEFAULTS: + is_dynamic_schema_node = ( + self.is_dynamic_schema_node or self.is_in_dynamic_schema_node + ) output = {} for key, child_obj in self.non_gui_children.items(): + if child_obj.is_dynamic_schema_node: + continue + child_value = child_obj.settings_value() - if not child_obj.is_file and not child_obj.file_item: + if ( + not is_dynamic_schema_node + and not child_obj.is_file + and not child_obj.file_item + ): for _key, _value in child_value.items(): new_key = "/".join([key, _key]) output[new_key] = _value diff --git a/openpype/settings/entities/dict_mutable_keys_entity.py b/openpype/settings/entities/dict_mutable_keys_entity.py index c3df935269..f75fb23d82 100644 --- a/openpype/settings/entities/dict_mutable_keys_entity.py +++ b/openpype/settings/entities/dict_mutable_keys_entity.py @@ -261,7 +261,7 @@ class DictMutableKeysEntity(EndpointEntity): raise EntitySchemaError(self, reason) # TODO Ability to store labels should be defined with different key - if self.collapsible_key and not self.file_item: + if self.collapsible_key and self.file_item is None: reason = ( "Modifiable dictionary with collapsible keys is not under" " file item so can't store metadata." diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index ee54bc6e02..a5e734f039 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -448,44 +448,6 @@ class TaskTypeEnumEntity(BaseEnumEntity): self.set(value_on_not_set) -class ProvidersEnum(BaseEnumEntity): - schema_types = ["providers-enum"] - - def _item_initalization(self): - self.multiselection = False - self.value_on_not_set = "" - self.enum_items = [] - self.valid_keys = set() - self.valid_value_types = (STRING_TYPE, ) - self.placeholder = None - - def _get_enum_values(self): - from openpype_modules.sync_server.providers import lib as lib_providers - - providers = lib_providers.factory.providers - - valid_keys = set() - valid_keys.add('') - enum_items = [{'': 'Choose Provider'}] - for provider_code, provider_info in providers.items(): - provider, _ = provider_info - enum_items.append({provider_code: provider.LABEL}) - valid_keys.add(provider_code) - - return enum_items, valid_keys - - def set_override_state(self, *args, **kwargs): - super(ProvidersEnum, self).set_override_state(*args, **kwargs) - - self.enum_items, self.valid_keys = self._get_enum_values() - - value_on_not_set = list(self.valid_keys)[0] - if self._current_value is NOT_SET: - self._current_value = value_on_not_set - - self.value_on_not_set = value_on_not_set - - class DeadlineUrlEnumEntity(BaseEnumEntity): schema_types = ["deadline_url-enum"] diff --git a/openpype/settings/entities/input_entities.py b/openpype/settings/entities/input_entities.py index 336d1f5c1e..0ded3ab7e5 100644 --- a/openpype/settings/entities/input_entities.py +++ b/openpype/settings/entities/input_entities.py @@ -49,6 +49,10 @@ class EndpointEntity(ItemEntity): super(EndpointEntity, self).schema_validations() + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + @abstractmethod def _settings_value(self): pass @@ -121,7 +125,11 @@ class InputEntity(EndpointEntity): def schema_validations(self): # Input entity must have file parent. - if not self.file_item: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and self.file_item is None + ): raise EntitySchemaError(self, "Missing parent file entity.") super(InputEntity, self).schema_validations() @@ -369,6 +377,14 @@ class NumberEntity(InputEntity): self.valid_value_types = valid_value_types self.value_on_not_set = value_on_not_set + # UI specific attributes + self.show_slider = self.schema_data.get("show_slider", False) + steps = self.schema_data.get("steps", None) + # Make sure that steps are not set to `0` + if steps == 0: + steps = None + self.steps = steps + def _convert_to_valid_type(self, value): if isinstance(value, str): new_value = None diff --git a/openpype/settings/entities/item_entities.py b/openpype/settings/entities/item_entities.py index ac6b3e76dd..c7c9c3097e 100644 --- a/openpype/settings/entities/item_entities.py +++ b/openpype/settings/entities/item_entities.py @@ -115,6 +115,9 @@ class PathEntity(ItemEntity): def set(self, value): self.child_obj.set(value) + def collect_dynamic_schema_entities(self, *args, **kwargs): + self.child_obj.collect_dynamic_schema_entities(*args, **kwargs) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET @@ -236,7 +239,12 @@ class ListStrictEntity(ItemEntity): def schema_validations(self): # List entity must have file parent. - if not self.file_item and not self.is_file: + if ( + not self.is_dynamic_schema_node + and not self.is_in_dynamic_schema_node + and not self.is_file + and self.file_item is None + ): raise EntitySchemaError( self, "Missing file entity in hierarchy." ) @@ -279,6 +287,10 @@ class ListStrictEntity(ItemEntity): for idx, item in enumerate(new_value): self.children[idx].set(item) + def collect_dynamic_schema_entities(self, collector): + if self.is_dynamic_schema_node: + collector.add_entity(self) + def settings_value(self): if self._override_state is OverrideState.NOT_DEFINED: return NOT_SET diff --git a/openpype/settings/entities/lib.py b/openpype/settings/entities/lib.py index 01f61d8bdf..f207322dee 100644 --- a/openpype/settings/entities/lib.py +++ b/openpype/settings/entities/lib.py @@ -3,6 +3,7 @@ import re import json import copy import inspect +import collections import contextlib from .exceptions import ( @@ -10,6 +11,12 @@ from .exceptions import ( SchemaDuplicatedEnvGroupKeys ) +from openpype.settings.constants import ( + SYSTEM_SETTINGS_KEY, + PROJECT_SETTINGS_KEY, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS +) try: STRING_TYPE = basestring except Exception: @@ -24,6 +31,10 @@ TEMPLATE_METADATA_KEYS = ( DEFAULT_VALUES_KEY, ) +SCHEMA_EXTEND_TYPES = ( + "schema", "template", "schema_template", "dynamic_schema" +) + template_key_pattern = re.compile(r"(\{.*?[^{0]*\})") @@ -102,8 +113,8 @@ class OverrideState: class SchemasHub: - def __init__(self, schema_subfolder, reset=True): - self._schema_subfolder = schema_subfolder + def __init__(self, schema_type, reset=True): + self._schema_type = schema_type self._loaded_types = {} self._gui_types = tuple() @@ -112,25 +123,56 @@ class SchemasHub: self._loaded_templates = {} self._loaded_schemas = {} + # Attributes for modules settings + self._dynamic_schemas_defs_by_id = {} + self._dynamic_schemas_by_id = {} + # Store validating and validated dynamic template or schemas self._validating_dynamic = set() self._validated_dynamic = set() - # It doesn't make sence to reload types on each reset as they can't be - # changed - self._load_types() - # Trigger reset if reset: self.reset() + @property + def schema_type(self): + return self._schema_type + def reset(self): + self._load_modules_settings_defs() + self._load_types() self._load_schemas() + def _load_modules_settings_defs(self): + from openpype.modules import get_module_settings_defs + + module_settings_defs = get_module_settings_defs() + for module_settings_def_cls in module_settings_defs: + module_settings_def = module_settings_def_cls() + def_id = module_settings_def.id + self._dynamic_schemas_defs_by_id[def_id] = module_settings_def + @property def gui_types(self): return self._gui_types + def resolve_dynamic_schema(self, dynamic_key): + output = [] + for def_id, def_keys in self._dynamic_schemas_by_id.items(): + if dynamic_key in def_keys: + def_schema = def_keys[dynamic_key] + if not def_schema: + continue + + if isinstance(def_schema, dict): + def_schema = [def_schema] + + for item in def_schema: + item["_dynamic_schema_id"] = def_id + output.extend(def_schema) + return output + def get_template_name(self, item_def, default=None): """Get template name from passed item definition. @@ -260,7 +302,7 @@ class SchemasHub: list: Resolved schema data. """ schema_type = schema_data["type"] - if schema_type not in ("schema", "template", "schema_template"): + if schema_type not in SCHEMA_EXTEND_TYPES: return [schema_data] if schema_type == "schema": @@ -268,6 +310,9 @@ class SchemasHub: self.get_schema(schema_data["name"]) ) + if schema_type == "dynamic_schema": + return self.resolve_dynamic_schema(schema_data["name"]) + template_name = schema_data["name"] template_def = self.get_template(template_name) @@ -368,14 +413,16 @@ class SchemasHub: self._crashed_on_load = {} self._loaded_templates = {} self._loaded_schemas = {} + self._dynamic_schemas_by_id = {} dirpath = os.path.join( os.path.dirname(os.path.abspath(__file__)), "schemas", - self._schema_subfolder + self.schema_type ) loaded_schemas = {} loaded_templates = {} + dynamic_schemas_by_id = {} for root, _, filenames in os.walk(dirpath): for filename in filenames: basename, ext = os.path.splitext(filename) @@ -425,8 +472,34 @@ class SchemasHub: ) loaded_schemas[basename] = schema_data + defs_iter = self._dynamic_schemas_defs_by_id.items() + for def_id, module_settings_def in defs_iter: + dynamic_schemas_by_id[def_id] = ( + module_settings_def.get_dynamic_schemas(self.schema_type) + ) + module_schemas = module_settings_def.get_settings_schemas( + self.schema_type + ) + for key, schema_data in module_schemas.items(): + if isinstance(schema_data, list): + if key in loaded_templates: + raise KeyError( + "Duplicated template key \"{}\"".format(key) + ) + loaded_templates[key] = schema_data + else: + if key in loaded_schemas: + raise KeyError( + "Duplicated schema key \"{}\"".format(key) + ) + loaded_schemas[key] = schema_data + self._loaded_templates = loaded_templates self._loaded_schemas = loaded_schemas + self._dynamic_schemas_by_id = dynamic_schemas_by_id + + def get_dynamic_modules_settings_defs(self, schema_def_id): + return self._dynamic_schemas_defs_by_id.get(schema_def_id) def _fill_template(self, child_data, template_def): """Fill template based on schema definition and template definition. @@ -660,3 +733,38 @@ class SchemasHub: if found_idx is not None: metadata_item = template_def.pop(found_idx) return metadata_item + + +class DynamicSchemaValueCollector: + # Map schema hub type to store keys + schema_hub_type_map = { + SCHEMA_KEY_SYSTEM_SETTINGS: SYSTEM_SETTINGS_KEY, + SCHEMA_KEY_PROJECT_SETTINGS: PROJECT_SETTINGS_KEY + } + + def __init__(self, schema_hub): + self._schema_hub = schema_hub + self._dynamic_entities = [] + + def add_entity(self, entity): + self._dynamic_entities.append(entity) + + def create_hierarchy(self): + output = collections.defaultdict(dict) + for entity in self._dynamic_entities: + output[entity.dynamic_schema_id][entity.path] = ( + entity.settings_value() + ) + return output + + def save_values(self): + hierarchy = self.create_hierarchy() + + for schema_def_id, schema_def_value in hierarchy.items(): + schema_def = self._schema_hub.get_dynamic_modules_settings_defs( + schema_def_id + ) + top_key = self.schema_hub_type_map.get( + self._schema_hub.schema_type + ) + schema_def.save_defaults(top_key, schema_def_value) diff --git a/openpype/settings/entities/root_entities.py b/openpype/settings/entities/root_entities.py index 4a06d2d591..05d20ee60b 100644 --- a/openpype/settings/entities/root_entities.py +++ b/openpype/settings/entities/root_entities.py @@ -9,8 +9,11 @@ from .base_entity import BaseItemEntity from .lib import ( NOT_SET, WRAPPER_TYPES, + SCHEMA_KEY_SYSTEM_SETTINGS, + SCHEMA_KEY_PROJECT_SETTINGS, OverrideState, - SchemasHub + SchemasHub, + DynamicSchemaValueCollector ) from .exceptions import ( SchemaError, @@ -28,6 +31,7 @@ from openpype.settings.lib import ( DEFAULTS_DIR, get_default_settings, + reset_default_settings, get_studio_system_settings_overrides, save_studio_settings, @@ -265,6 +269,16 @@ class RootEntity(BaseItemEntity): output[key] = child_obj.value return output + def collect_dynamic_schema_entities(self): + output = DynamicSchemaValueCollector(self.schema_hub) + if self._override_state is not OverrideState.DEFAULTS: + return output + + for child_obj in self.non_gui_children.values(): + child_obj.collect_dynamic_schema_entities(output) + + return output + def settings_value(self): """Value for current override state with metadata. @@ -276,6 +290,8 @@ class RootEntity(BaseItemEntity): if self._override_state is not OverrideState.DEFAULTS: output = {} for key, child_obj in self.non_gui_children.items(): + if child_obj.is_dynamic_schema_node: + continue value = child_obj.settings_value() if value is not NOT_SET: output[key] = value @@ -374,6 +390,7 @@ class RootEntity(BaseItemEntity): if self._override_state is OverrideState.DEFAULTS: self._save_default_values() + reset_default_settings() elif self._override_state is OverrideState.STUDIO: self._save_studio_values() @@ -421,6 +438,9 @@ class RootEntity(BaseItemEntity): with open(output_path, "w") as file_stream: json.dump(value, file_stream, indent=4) + dynamic_values_item = self.collect_dynamic_schema_entities() + dynamic_values_item.save_values() + @abstractmethod def _save_studio_values(self): """Save studio override values.""" @@ -476,7 +496,7 @@ class SystemSettings(RootEntity): ): if schema_hub is None: # Load system schemas - schema_hub = SchemasHub("system_schema") + schema_hub = SchemasHub(SCHEMA_KEY_SYSTEM_SETTINGS) super(SystemSettings, self).__init__(schema_hub, reset) @@ -607,7 +627,7 @@ class ProjectSettings(RootEntity): if schema_hub is None: # Load system schemas - schema_hub = SchemasHub("projects_schema") + schema_hub = SchemasHub(SCHEMA_KEY_PROJECT_SETTINGS) super(ProjectSettings, self).__init__(schema_hub, reset) diff --git a/openpype/settings/entities/schemas/README.md b/openpype/settings/entities/schemas/README.md index 05605f8ce1..c8432f0f2e 100644 --- a/openpype/settings/entities/schemas/README.md +++ b/openpype/settings/entities/schemas/README.md @@ -112,6 +112,22 @@ ``` - It is possible to define default values for unfilled fields to do so one of items in list must be dictionary with key `"__default_values__"` and value as dictionary with default key: values (as in example above). +### dynamic_schema +- dynamic templates that can be defined by class of `ModuleSettingsDef` +- example: +``` +{ + "type": "dynamic_schema", + "name": "project_settings/global" +} +``` +- all valid `ModuleSettingsDef` classes where calling of `get_settings_schemas` + will return dictionary where is key "project_settings/global" with schemas + will extend and replace this item +- works almost the same way as templates + - one item can be replaced by multiple items (or by 0 items) +- goal is to dynamically loaded settings of OpenPype addons without having + their schemas or default values in main repository ## Basic Dictionary inputs - these inputs wraps another inputs into {key: value} relation @@ -300,6 +316,8 @@ How output of the schema could look like on save: - key `"decimal"` defines how many decimal places will be used, 0 is for integer input (Default: `0`) - key `"minimum"` as minimum allowed number to enter (Default: `-99999`) - key `"maxium"` as maximum allowed number to enter (Default: `99999`) +- key `"steps"` will change single step value of UI inputs (using arrows and wheel scroll) +- for UI it is possible to show slider to enable this option set `show_slider` to `true` ``` { "type": "number", @@ -311,6 +329,18 @@ How output of the schema could look like on save: } ``` +``` +{ + "type": "number", + "key": "ratio", + "label": "Ratio" + "decimal": 3, + "minimum": 0, + "maximum": 1, + "show_slider": true +} +``` + ### text - simple text input - key `"multiline"` allows to enter multiple lines of text (Default: `False`) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_main.json b/openpype/settings/entities/schemas/projects_schema/schema_main.json index 575cfc9e72..c9eca5dedd 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_main.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_main.json @@ -125,6 +125,10 @@ { "type": "schema", "name": "schema_project_unreal" + }, + { + "type": "dynamic_schema", + "name": "project_settings/global" } ] } diff --git a/openpype/settings/entities/schemas/system_schema/example_schema.json b/openpype/settings/entities/schemas/system_schema/example_schema.json index f633d5cb1a..af6a2d49f4 100644 --- a/openpype/settings/entities/schemas/system_schema/example_schema.json +++ b/openpype/settings/entities/schemas/system_schema/example_schema.json @@ -183,6 +183,15 @@ "minimum": -10, "maximum": -5 }, + { + "type": "number", + "key": "number_with_slider", + "label": "Number with slider", + "decimal": 2, + "minimum": 0.0, + "maximum": 1.0, + "show_slider": true + }, { "type": "text", "key": "singleline_text", diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index dd85f9351a..91e3091f42 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -5,6 +5,18 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "path", + "key": "addon_paths", + "label": "OpenPype AddOn Paths", + "use_label_wrap": true, + "multiplatform": true, + "multipath": true, + "require_restart": true + }, + { + "type": "separator" + }, { "type": "dict", "key": "avalon", @@ -16,7 +28,8 @@ "type": "number", "key": "AVALON_TIMEOUT", "minimum": 0, - "label": "Avalon Mongo Timeout (ms)" + "label": "Avalon Mongo Timeout (ms)", + "steps": 100 }, { "type": "path", @@ -109,14 +122,7 @@ "collapsible_key": false, "object_type": { - "type": "dict", - "children": [ - { - "type": "providers-enum", - "key": "provider", - "label": "Provider" - } - ] + "type": "sync-server-providers" } } ] diff --git a/openpype/settings/lib.py b/openpype/settings/lib.py index 4a363910b8..60ed54bd4a 100644 --- a/openpype/settings/lib.py +++ b/openpype/settings/lib.py @@ -329,6 +329,45 @@ def reset_default_settings(): _DEFAULT_SETTINGS = None +def _get_default_settings(): + from openpype.modules import get_module_settings_defs + + defaults = load_openpype_default_settings() + + module_settings_defs = get_module_settings_defs() + for module_settings_def_cls in module_settings_defs: + module_settings_def = module_settings_def_cls() + system_defaults = module_settings_def.get_defaults( + SYSTEM_SETTINGS_KEY + ) or {} + for path, value in system_defaults.items(): + if not path: + continue + + subdict = defaults["system_settings"] + path_items = list(path.split("/")) + last_key = path_items.pop(-1) + for key in path_items: + subdict = subdict[key] + subdict[last_key] = value + + project_defaults = module_settings_def.get_defaults( + PROJECT_SETTINGS_KEY + ) or {} + for path, value in project_defaults.items(): + if not path: + continue + + subdict = defaults + path_items = list(path.split("/")) + last_key = path_items.pop(-1) + for key in path_items: + subdict = subdict[key] + subdict[last_key] = value + + return defaults + + def get_default_settings(): """Get default settings. @@ -338,12 +377,10 @@ def get_default_settings(): Returns: dict: Loaded default settings. """ - # TODO add cacher - return load_openpype_default_settings() - # global _DEFAULT_SETTINGS - # if _DEFAULT_SETTINGS is None: - # _DEFAULT_SETTINGS = load_jsons_from_dir(DEFAULTS_DIR) - # return copy.deepcopy(_DEFAULT_SETTINGS) + global _DEFAULT_SETTINGS + if _DEFAULT_SETTINGS is None: + _DEFAULT_SETTINGS = _get_default_settings() + return copy.deepcopy(_DEFAULT_SETTINGS) def load_json_file(fpath): @@ -380,8 +417,8 @@ def load_jsons_from_dir(path, *args, **kwargs): "data1": "CONTENT OF FILE" }, "folder2": { - "data1": { - "subfolder1": "CONTENT OF FILE" + "subfolder1": { + "data2": "CONTENT OF FILE" } } } diff --git a/openpype/tools/settings/settings/item_widgets.py b/openpype/tools/settings/settings/item_widgets.py index b2b129da86..da74c2adc5 100644 --- a/openpype/tools/settings/settings/item_widgets.py +++ b/openpype/tools/settings/settings/item_widgets.py @@ -21,6 +21,7 @@ from .base import ( BaseWidget, InputWidget ) +from openpype.widgets.sliders import NiceSlider from openpype.tools.settings import CHILD_OFFSET @@ -404,21 +405,53 @@ class TextWidget(InputWidget): class NumberWidget(InputWidget): + _slider_widget = None + def _add_inputs_to_layout(self): kwargs = { "minimum": self.entity.minimum, "maximum": self.entity.maximum, - "decimal": self.entity.decimal + "decimal": self.entity.decimal, + "steps": self.entity.steps } self.input_field = NumberSpinBox(self.content_widget, **kwargs) + input_field_stretch = 1 + + slider_multiplier = 1 + if self.entity.show_slider: + # Slider can't handle float numbers so all decimals are converted + # to integer range. + slider_multiplier = 10 ** self.entity.decimal + slider_widget = NiceSlider(QtCore.Qt.Horizontal, self) + slider_widget.setRange( + int(self.entity.minimum * slider_multiplier), + int(self.entity.maximum * slider_multiplier) + ) + if self.entity.steps is not None: + slider_widget.setSingleStep( + self.entity.steps * slider_multiplier + ) + + self.content_layout.addWidget(slider_widget, 1) + + slider_widget.valueChanged.connect(self._on_slider_change) + + self._slider_widget = slider_widget + + input_field_stretch = 0 + + self._slider_multiplier = slider_multiplier self.setFocusProxy(self.input_field) - self.content_layout.addWidget(self.input_field, 1) + self.content_layout.addWidget(self.input_field, input_field_stretch) self.input_field.valueChanged.connect(self._on_value_change) self.input_field.focused_in.connect(self._on_input_focus) + self._ignore_slider_change = False + self._ignore_input_change = False + def _on_input_focus(self): self.focused_in() @@ -429,10 +462,25 @@ class NumberWidget(InputWidget): def set_entity_value(self): self.input_field.setValue(self.entity.value) + def _on_slider_change(self, new_value): + if self._ignore_slider_change: + return + + self._ignore_input_change = True + self.input_field.setValue(new_value / self._slider_multiplier) + self._ignore_input_change = False + def _on_value_change(self): if self.ignore_input_changes: return - self.entity.set(self.input_field.value()) + + value = self.input_field.value() + if self._slider_widget is not None and not self._ignore_input_change: + self._ignore_slider_change = True + self._slider_widget.setValue(value * self._slider_multiplier) + self._ignore_slider_change = False + + self.entity.set(value) class RawJsonInput(SettingsPlainTextEdit): diff --git a/openpype/tools/settings/settings/style/style.css b/openpype/tools/settings/settings/style/style.css index 250c15063f..d9d85a481e 100644 --- a/openpype/tools/settings/settings/style/style.css +++ b/openpype/tools/settings/settings/style/style.css @@ -114,6 +114,30 @@ QPushButton[btn-type="expand-toggle"] { background: #21252B; } +/* SLider */ +QSlider::groove { + border: 1px solid #464b54; + border-radius: 0.3em; +} +QSlider::groove:horizontal { + height: 8px; +} +QSlider::groove:vertical { + width: 8px; +} +QSlider::handle { + width: 10px; + height: 10px; + + border-radius: 5px; +} +QSlider::handle:horizontal { + margin: -2px 0; +} +QSlider::handle:vertical { + margin: 0 -2px; +} + #GroupWidget { border-bottom: 1px solid #21252B; } diff --git a/openpype/tools/settings/settings/widgets.py b/openpype/tools/settings/settings/widgets.py index b821c3bb2c..2caf8c33ba 100644 --- a/openpype/tools/settings/settings/widgets.py +++ b/openpype/tools/settings/settings/widgets.py @@ -92,11 +92,15 @@ class NumberSpinBox(QtWidgets.QDoubleSpinBox): min_value = kwargs.pop("minimum", -99999) max_value = kwargs.pop("maximum", 99999) decimals = kwargs.pop("decimal", 0) + steps = kwargs.pop("steps", None) + super(NumberSpinBox, self).__init__(*args, **kwargs) self.setFocusPolicy(QtCore.Qt.StrongFocus) self.setDecimals(decimals) self.setMinimum(min_value) self.setMaximum(max_value) + if steps is not None: + self.setSingleStep(steps) def focusInEvent(self, event): super(NumberSpinBox, self).focusInEvent(event) diff --git a/openpype/tools/workfiles/app.py b/openpype/tools/workfiles/app.py index b542e6e718..3d2633f8dc 100644 --- a/openpype/tools/workfiles/app.py +++ b/openpype/tools/workfiles/app.py @@ -430,7 +430,6 @@ class FilesWidget(QtWidgets.QWidget): # Pype's anatomy object for current project self.anatomy = Anatomy(io.Session["AVALON_PROJECT"]) # Template key used to get work template from anatomy templates - # TODO change template key based on task self.template_key = "work" # This is not root but workfile directory diff --git a/openpype/widgets/sliders.py b/openpype/widgets/sliders.py new file mode 100644 index 0000000000..32ade58af5 --- /dev/null +++ b/openpype/widgets/sliders.py @@ -0,0 +1,139 @@ +from Qt import QtWidgets, QtCore, QtGui + + +class NiceSlider(QtWidgets.QSlider): + def __init__(self, *args, **kwargs): + super(NiceSlider, self).__init__(*args, **kwargs) + self._mouse_clicked = False + self._handle_size = 0 + + self._bg_brush = QtGui.QBrush(QtGui.QColor("#21252B")) + self._fill_brush = QtGui.QBrush(QtGui.QColor("#5cadd6")) + + def mousePressEvent(self, event): + self._mouse_clicked = True + if event.button() == QtCore.Qt.LeftButton: + self._set_value_to_pos(event.pos()) + return event.accept() + return super(NiceSlider, self).mousePressEvent(event) + + def mouseMoveEvent(self, event): + if self._mouse_clicked: + self._set_value_to_pos(event.pos()) + + super(NiceSlider, self).mouseMoveEvent(event) + + def mouseReleaseEvent(self, event): + self._mouse_clicked = True + super(NiceSlider, self).mouseReleaseEvent(event) + + def _set_value_to_pos(self, pos): + if self.orientation() == QtCore.Qt.Horizontal: + self._set_value_to_pos_x(pos.x()) + else: + self._set_value_to_pos_y(pos.y()) + + def _set_value_to_pos_x(self, pos_x): + _range = self.maximum() - self.minimum() + handle_size = self._handle_size + half_handle = handle_size / 2 + pos_x -= half_handle + width = self.width() - handle_size + value = ((_range * pos_x) / width) + self.minimum() + self.setValue(value) + + def _set_value_to_pos_y(self, pos_y): + _range = self.maximum() - self.minimum() + handle_size = self._handle_size + half_handle = handle_size / 2 + pos_y = self.height() - pos_y - half_handle + height = self.height() - handle_size + value = (_range * pos_y / height) + self.minimum() + self.setValue(value) + + def paintEvent(self, event): + painter = QtGui.QPainter(self) + opt = QtWidgets.QStyleOptionSlider() + self.initStyleOption(opt) + + painter.fillRect(event.rect(), QtCore.Qt.transparent) + + painter.setRenderHint(QtGui.QPainter.HighQualityAntialiasing) + + horizontal = self.orientation() == QtCore.Qt.Horizontal + + rect = self.style().subControlRect( + QtWidgets.QStyle.CC_Slider, + opt, + QtWidgets.QStyle.SC_SliderGroove, + self + ) + + _range = self.maximum() - self.minimum() + _offset = self.value() - self.minimum() + if horizontal: + _handle_half = rect.height() / 2 + _handle_size = _handle_half * 2 + width = rect.width() - _handle_size + pos_x = ((width / _range) * _offset) + pos_y = rect.center().y() - _handle_half + 1 + else: + _handle_half = rect.width() / 2 + _handle_size = _handle_half * 2 + height = rect.height() - _handle_size + pos_x = rect.center().x() - _handle_half + 1 + pos_y = height - ((height / _range) * _offset) + + handle_rect = QtCore.QRect( + pos_x, pos_y, _handle_size, _handle_size + ) + + self._handle_size = _handle_size + _offset = 2 + _size = _handle_size - _offset + if horizontal: + if rect.height() > _size: + new_rect = QtCore.QRect(0, 0, rect.width(), _size) + center_point = QtCore.QPoint( + rect.center().x(), handle_rect.center().y() + ) + new_rect.moveCenter(center_point) + rect = new_rect + + ratio = rect.height() / 2 + fill_rect = QtCore.QRect( + rect.x(), + rect.y(), + handle_rect.right() - rect.x(), + rect.height() + ) + + else: + if rect.width() > _size: + new_rect = QtCore.QRect(0, 0, _size, rect.height()) + center_point = QtCore.QPoint( + handle_rect.center().x(), rect.center().y() + ) + new_rect.moveCenter(center_point) + rect = new_rect + + ratio = rect.width() / 2 + fill_rect = QtCore.QRect( + rect.x(), + handle_rect.y(), + rect.width(), + rect.height() - handle_rect.y(), + ) + + painter.save() + painter.setPen(QtCore.Qt.NoPen) + painter.setBrush(self._bg_brush) + painter.drawRoundedRect(rect, ratio, ratio) + + painter.setBrush(self._fill_brush) + painter.drawRoundedRect(fill_rect, ratio, ratio) + + painter.setPen(QtCore.Qt.NoPen) + painter.setBrush(self._fill_brush) + painter.drawEllipse(handle_rect) + painter.restore() diff --git a/repos/avalon-core b/repos/avalon-core index f48fce09c0..b3e4959778 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit f48fce09c0986c1fd7f6731de33907be46b436c5 +Subproject commit b3e49597786c931c13bca207769727d5fc56d5f6 diff --git a/start.py b/start.py index 6473a926d0..00f9a50cbb 100644 --- a/start.py +++ b/start.py @@ -179,6 +179,12 @@ else: ssl_cert_file = certifi.where() os.environ["SSL_CERT_FILE"] = ssl_cert_file +if "--headless" in sys.argv: + os.environ["OPENPYPE_HEADLESS_MODE"] = "1" + sys.argv.remove("--headless") +else: + if os.getenv("OPENPYPE_HEADLESS_MODE") != "1": + os.environ.pop("OPENPYPE_HEADLESS_MODE", None) import igniter # noqa: E402 from igniter import BootstrapRepos # noqa: E402 @@ -343,7 +349,7 @@ def _process_arguments() -> tuple: # check for `--use-version=3.0.0` argument and `--use-staging` use_version = None use_staging = False - print_versions = False + commands = [] for arg in sys.argv: if arg == "--use-version": _print("!!! Please use option --use-version like:") @@ -366,17 +372,38 @@ def _process_arguments() -> tuple: " proper version string.")) sys.exit(1) + if arg == "--validate-version": + _print("!!! Please use option --validate-version like:") + _print(" --validate-version=3.0.0") + sys.exit(1) + + if arg.startswith("--validate-version="): + m = re.search( + r"--validate-version=(?P\d+\.\d+\.\d+(?:\S*)?)", arg) + if m and m.group('version'): + use_version = m.group('version') + sys.argv.remove(arg) + commands.append("validate") + else: + _print("!!! Requested version isn't in correct format.") + _print((" Use --list-versions to find out" + " proper version string.")) + sys.exit(1) + if "--use-staging" in sys.argv: use_staging = True sys.argv.remove("--use-staging") if "--list-versions" in sys.argv: - print_versions = True + commands.append("print_versions") sys.argv.remove("--list-versions") # handle igniter # this is helper to run igniter before anything else if "igniter" in sys.argv: + if os.getenv("OPENPYPE_HEADLESS_MODE") == "1": + _print("!!! Cannot open Igniter dialog in headless mode.") + sys.exit(1) import igniter return_code = igniter.open_dialog() @@ -389,7 +416,7 @@ def _process_arguments() -> tuple: sys.argv.pop(idx) sys.argv.insert(idx, "tray") - return use_version, use_staging, print_versions + return use_version, use_staging, commands def _determine_mongodb() -> str: @@ -424,6 +451,11 @@ def _determine_mongodb() -> str: if not openpype_mongo: _print("*** No DB connection string specified.") + if os.getenv("OPENPYPE_HEADLESS_MODE") == "1": + _print("!!! Cannot open Igniter dialog in headless mode.") + _print( + "!!! Please use `OPENPYPE_MONGO` to specify server address.") + sys.exit(1) _print("--- launching setup UI ...") result = igniter.open_dialog() @@ -527,6 +559,9 @@ def _find_frozen_openpype(use_version: str = None, except IndexError: # no OpenPype version found, run Igniter and ask for them. _print('*** No OpenPype versions found.') + if os.getenv("OPENPYPE_HEADLESS_MODE") == "1": + _print("!!! Cannot open Igniter dialog in headless mode.") + sys.exit(1) _print("--- launching setup UI ...") import igniter return_code = igniter.open_dialog() @@ -590,8 +625,16 @@ def _find_frozen_openpype(use_version: str = None, if not is_inside: # install latest version to user data dir - version_path = bootstrap.install_version( - openpype_version, force=True) + if os.getenv("OPENPYPE_HEADLESS_MODE", "0") != "1": + import igniter + version_path = igniter.open_update_window(openpype_version) + else: + version_path = bootstrap.install_version( + openpype_version, force=True) + + openpype_version.path = version_path + _initialize_environment(openpype_version) + return openpype_version.path if openpype_version.path.is_file(): _print(">>> Extracting zip file ...") @@ -738,7 +781,7 @@ def boot(): # Process arguments # ------------------------------------------------------------------------ - use_version, use_staging, print_versions = _process_arguments() + use_version, use_staging, commands = _process_arguments() if os.getenv("OPENPYPE_VERSION"): if use_version: @@ -766,13 +809,47 @@ def boot(): # Get openpype path from database and set it to environment so openpype can # find its versions there and bootstrap them. openpype_path = get_openpype_path_from_db(openpype_mongo) + + if getattr(sys, 'frozen', False): + local_version = bootstrap.get_version(Path(OPENPYPE_ROOT)) + else: + local_version = bootstrap.get_local_live_version() + + if "validate" in commands: + _print(f">>> Validating version [ {use_version} ]") + openpype_versions = bootstrap.find_openpype(include_zips=True, + staging=True) + openpype_versions += bootstrap.find_openpype(include_zips=True, + staging=False) + v: OpenPypeVersion + found = [v for v in openpype_versions if str(v) == use_version] + if not found: + _print(f"!!! Version [ {use_version} ] not found.") + list_versions(openpype_versions, local_version) + sys.exit(1) + + # print result + result = bootstrap.validate_openpype_version( + bootstrap.get_version_path_from_list( + use_version, openpype_versions)) + + _print("{}{}".format( + ">>> " if result[0] else "!!! ", + bootstrap.validate_openpype_version( + bootstrap.get_version_path_from_list( + use_version, openpype_versions) + )[1]) + ) + sys.exit(1) + + if not openpype_path: _print("*** Cannot get OpenPype path from database.") if not os.getenv("OPENPYPE_PATH") and openpype_path: os.environ["OPENPYPE_PATH"] = openpype_path - if print_versions: + if "print_versions" in commands: if not use_staging: _print("--- This will list only non-staging versions detected.") _print(" To see staging versions, use --use-staging argument.") @@ -803,6 +880,13 @@ def boot(): # no version to run _print(f"!!! {e}") sys.exit(1) + # validate version + _print(f">>> Validating version [ {str(version_path)} ]") + result = bootstrap.validate_openpype_version(version_path) + if not result[0]: + _print(f"!!! Invalid version: {result[1]}") + sys.exit(1) + _print(f"--- version is valid") else: version_path = _bootstrap_from_code(use_version, use_staging) diff --git a/website/docs/admin_openpype_commands.md b/website/docs/admin_openpype_commands.md index 1a91e2e7fe..d6ccc883b0 100644 --- a/website/docs/admin_openpype_commands.md +++ b/website/docs/admin_openpype_commands.md @@ -18,11 +18,14 @@ Running OpenPype without any commands will default to `tray`. ```shell openpype_console --use-version=3.0.0-foo+bar ``` +`--headless` - to run OpenPype in headless mode (without using graphical UI) `--use-staging` - to use staging versions of OpenPype. `--list-versions [--use-staging]` - to list available versions. +`--validate-version` to validate integrity of given version + For more information [see here](admin_use#run-openpype). ## Commands diff --git a/website/docs/admin_use.md b/website/docs/admin_use.md index 4ad08a0174..178241ad19 100644 --- a/website/docs/admin_use.md +++ b/website/docs/admin_use.md @@ -56,6 +56,19 @@ openpype_console --list-versions You can add `--use-staging` to list staging versions. ::: +If you want to validate integrity of some available version, you can use: + +```shell +openpype_console --validate-version=3.3.0 +``` + +This will go through the version and validate file content against sha 256 hashes +stored in `checksums` file. + +:::tip Headless mode +Add `--headless` to run OpenPype without graphical UI (useful on server or on automated tasks, etc.) +::: + ### Details When you run OpenPype from executable, few check are made: