diff --git a/client/ayon_core/pipeline/delivery.py b/client/ayon_core/pipeline/delivery.py index 029775e1db..2a2adf984a 100644 --- a/client/ayon_core/pipeline/delivery.py +++ b/client/ayon_core/pipeline/delivery.py @@ -3,11 +3,20 @@ import os import copy import shutil import glob -import clique import collections +from typing import Dict, Any, Iterable + +import clique +import ayon_api from ayon_core.lib import create_hard_link +from .template_data import ( + get_general_template_data, + get_folder_template_data, + get_task_template_data, +) + def _copy_file(src_path, dst_path): """Hardlink file if possible(to save space), copy if not. @@ -327,3 +336,71 @@ def deliver_sequence( uploaded += 1 return report_items, uploaded + + +def _merge_data(data, new_data): + queue = collections.deque() + queue.append((data, new_data)) + while queue: + q_data, q_new_data = queue.popleft() + for key, value in q_new_data.items(): + if key in q_data and isinstance(value, dict): + queue.append((q_data[key], value)) + continue + q_data[key] = value + + +def get_representations_delivery_template_data( + project_name: str, + representation_ids: Iterable[str], +) -> Dict[str, Dict[str, Any]]: + representation_ids = set(representation_ids) + + output = { + repre_id: {} + for repre_id in representation_ids + } + if not representation_ids: + return output + + project_entity = ayon_api.get_project(project_name) + + general_template_data = get_general_template_data() + + repres_hierarchy = ayon_api.get_representations_hierarchy( + project_name, + representation_ids, + project_fields=set(), + folder_fields={"path", "folderType"}, + task_fields={"name", "taskType"}, + product_fields={"name", "productType"}, + version_fields={"version", "productId"}, + representation_fields=None, + ) + for repre_id, repre_hierarchy in repres_hierarchy.items(): + repre_entity = repre_hierarchy.representation + if repre_entity is None: + continue + + template_data = repre_entity["context"] + template_data.update(copy.deepcopy(general_template_data)) + template_data.update(get_folder_template_data( + repre_hierarchy.folder, project_name + )) + if repre_hierarchy.task: + template_data.update(get_task_template_data( + project_entity, repre_hierarchy.task + )) + + product_entity = repre_hierarchy.product + version_entity = repre_hierarchy.version + template_data.update({ + "product": { + "name": product_entity["name"], + "type": product_entity["productType"], + }, + "version": version_entity["version"], + }) + _merge_data(template_data, repre_entity["context"]) + output[repre_id] = template_data + return output diff --git a/client/ayon_core/pipeline/farm/pyblish_functions.py b/client/ayon_core/pipeline/farm/pyblish_functions.py index af90903bd8..98951b2766 100644 --- a/client/ayon_core/pipeline/farm/pyblish_functions.py +++ b/client/ayon_core/pipeline/farm/pyblish_functions.py @@ -788,15 +788,15 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data, colorspace = product.colorspace break - if isinstance(files, (list, tuple)): - files = [os.path.basename(f) for f in files] + if isinstance(collected_files, (list, tuple)): + collected_files = [os.path.basename(f) for f in collected_files] else: - files = os.path.basename(files) + collected_files = os.path.basename(collected_files) rep = { "name": ext, "ext": ext, - "files": files, + "files": collected_files, "frameStart": int(skeleton["frameStartHandle"]), "frameEnd": int(skeleton["frameEndHandle"]), # If expectedFile are absolute, we need only filenames diff --git a/client/ayon_core/plugins/load/delivery.py b/client/ayon_core/plugins/load/delivery.py index 5c53d170eb..406040d936 100644 --- a/client/ayon_core/plugins/load/delivery.py +++ b/client/ayon_core/plugins/load/delivery.py @@ -1,23 +1,22 @@ -import copy import platform from collections import defaultdict import ayon_api from qtpy import QtWidgets, QtCore, QtGui -from ayon_core.pipeline import load, Anatomy from ayon_core import resources, style - from ayon_core.lib import ( format_file_size, collect_frames, get_datetime_data, ) +from ayon_core.pipeline import load, Anatomy from ayon_core.pipeline.load import get_representation_path_with_anatomy from ayon_core.pipeline.delivery import ( get_format_dict, check_destination_path, - deliver_single_file + deliver_single_file, + get_representations_delivery_template_data, ) @@ -200,20 +199,31 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): format_dict = get_format_dict(self.anatomy, self.root_line_edit.text()) renumber_frame = self.renumber_frame.isChecked() frame_offset = self.first_frame_start.value() + filtered_repres = [] + repre_ids = set() for repre in self._representations: - if repre["name"] not in selected_repres: - continue + if repre["name"] in selected_repres: + filtered_repres.append(repre) + repre_ids.add(repre["id"]) + template_data_by_repre_id = ( + get_representations_delivery_template_data( + self.anatomy.project_name, repre_ids + ) + ) + for repre in filtered_repres: repre_path = get_representation_path_with_anatomy( repre, self.anatomy ) - anatomy_data = copy.deepcopy(repre["context"]) - new_report_items = check_destination_path(repre["id"], - self.anatomy, - anatomy_data, - datetime_data, - template_name) + template_data = template_data_by_repre_id[repre["id"]] + new_report_items = check_destination_path( + repre["id"], + self.anatomy, + template_data, + datetime_data, + template_name + ) report_items.update(new_report_items) if new_report_items: @@ -224,7 +234,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): repre, self.anatomy, template_name, - anatomy_data, + template_data, format_dict, report_items, self.log @@ -267,9 +277,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): if frame is not None: if repre["context"].get("frame"): - anatomy_data["frame"] = frame + template_data["frame"] = frame elif repre["context"].get("udim"): - anatomy_data["udim"] = frame + template_data["udim"] = frame else: # Fallback self.log.warning( @@ -277,7 +287,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): " data. Supplying sequence frame to '{frame}'" " formatting data." ) - anatomy_data["frame"] = frame + template_data["frame"] = frame new_report_items, uploaded = deliver_single_file(*args) report_items.update(new_report_items) self._update_progress(uploaded) @@ -342,8 +352,8 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): def _get_selected_repres(self): """Returns list of representation names filtered from checkboxes.""" selected_repres = [] - for repre_name, chckbox in self._representation_checkboxes.items(): - if chckbox.isChecked(): + for repre_name, checkbox in self._representation_checkboxes.items(): + if checkbox.isChecked(): selected_repres.append(repre_name) return selected_repres diff --git a/client/ayon_core/plugins/publish/extract_thumbnail.py b/client/ayon_core/plugins/publish/extract_thumbnail.py index 4ffabf6028..37bbac8898 100644 --- a/client/ayon_core/plugins/publish/extract_thumbnail.py +++ b/client/ayon_core/plugins/publish/extract_thumbnail.py @@ -36,7 +36,8 @@ class ExtractThumbnail(pyblish.api.InstancePlugin): "traypublisher", "substancepainter", "nuke", - "aftereffects" + "aftereffects", + "unreal" ] enabled = False diff --git a/client/ayon_core/tools/experimental_tools/pyblish_debug_stepper.py b/client/ayon_core/tools/experimental_tools/pyblish_debug_stepper.py new file mode 100644 index 0000000000..33de4bf036 --- /dev/null +++ b/client/ayon_core/tools/experimental_tools/pyblish_debug_stepper.py @@ -0,0 +1,273 @@ +""" +Brought from https://gist.github.com/BigRoy/1972822065e38f8fae7521078e44eca2 +Code Credits: [BigRoy](https://github.com/BigRoy) + +Requirement: + It requires pyblish version >= 1.8.12 + +How it works: + This tool makes use of pyblish event `pluginProcessed` to: + 1. Pause the publishing. + 2. Collect some info about the plugin. + 3. Show that info to the tool's window. + 4. Continue publishing on clicking `step` button. + +How to use it: + 1. Launch the tool from AYON experimental tools window. + 2. Launch the publisher tool and click validate. + 3. Click Step to run plugins one by one. + +Note : + Pyblish debugger also works when triggering the validation or + publishing from code. + Here's an example about validating from code: + https://github.com/MustafaJafar/ayon-recipes/blob/main/validate_from_code.py + +""" + +import copy +import json +from qtpy import QtWidgets, QtCore, QtGui + +import pyblish.api +from ayon_core import style + +TAB = 4* " " +HEADER_SIZE = "15px" + +KEY_COLOR = QtGui.QColor("#ffffff") +NEW_KEY_COLOR = QtGui.QColor("#00ff00") +VALUE_TYPE_COLOR = QtGui.QColor("#ffbbbb") +NEW_VALUE_TYPE_COLOR = QtGui.QColor("#ff4444") +VALUE_COLOR = QtGui.QColor("#777799") +NEW_VALUE_COLOR = QtGui.QColor("#DDDDCC") +CHANGED_VALUE_COLOR = QtGui.QColor("#CCFFCC") + +MAX_VALUE_STR_LEN = 100 + + +def failsafe_deepcopy(data): + """Allow skipping the deepcopy for unsupported types""" + try: + return copy.deepcopy(data) + except TypeError: + if isinstance(data, dict): + return { + key: failsafe_deepcopy(value) + for key, value in data.items() + } + elif isinstance(data, list): + return data.copy() + return data + + +class DictChangesModel(QtGui.QStandardItemModel): + # TODO: Replace this with a QAbstractItemModel + def __init__(self, *args, **kwargs): + super(DictChangesModel, self).__init__(*args, **kwargs) + self._data = {} + + columns = ["Key", "Type", "Value"] + self.setColumnCount(len(columns)) + for i, label in enumerate(columns): + self.setHeaderData(i, QtCore.Qt.Horizontal, label) + + def _update_recursive(self, data, parent, previous_data): + for key, value in data.items(): + + # Find existing item or add new row + parent_index = parent.index() + for row in range(self.rowCount(parent_index)): + # Update existing item if it exists + index = self.index(row, 0, parent_index) + if index.data() == key: + item = self.itemFromIndex(index) + type_item = self.itemFromIndex(self.index(row, 1, parent_index)) # noqa + value_item = self.itemFromIndex(self.index(row, 2, parent_index)) # noqa + break + else: + item = QtGui.QStandardItem(key) + type_item = QtGui.QStandardItem() + value_item = QtGui.QStandardItem() + parent.appendRow([item, type_item, value_item]) + + # Key + key_color = NEW_KEY_COLOR if key not in previous_data else KEY_COLOR # noqa + item.setData(key_color, QtCore.Qt.ForegroundRole) + + # Type + type_str = type(value).__name__ + type_color = VALUE_TYPE_COLOR + if ( + key in previous_data + and type(previous_data[key]).__name__ != type_str + ): + type_color = NEW_VALUE_TYPE_COLOR + + type_item.setText(type_str) + type_item.setData(type_color, QtCore.Qt.ForegroundRole) + + # Value + value_changed = False + if key not in previous_data or previous_data[key] != value: + value_changed = True + value_color = NEW_VALUE_COLOR if value_changed else VALUE_COLOR + + value_item.setData(value_color, QtCore.Qt.ForegroundRole) + if value_changed: + value_str = str(value) + if len(value_str) > MAX_VALUE_STR_LEN: + value_str = value_str[:MAX_VALUE_STR_LEN] + "..." + value_item.setText(value_str) + + # Preferably this is deferred to only when the data gets + # requested since this formatting can be slow for very large + # data sets like project settings and system settings + # This will also be MUCH faster if we don't clear the + # items on each update but only updated/add/remove changed + # items so that this also runs much less often + value_item.setData( + json.dumps(value, default=str, indent=4), + QtCore.Qt.ToolTipRole + ) + + if isinstance(value, dict): + previous_value = previous_data.get(key, {}) + if previous_data.get(key) != value: + # Update children if the value is not the same as before + self._update_recursive(value, + parent=item, + previous_data=previous_value) + else: + # TODO: Ensure all children are updated to be not marked + # as 'changed' in the most optimal way possible + self._update_recursive(value, + parent=item, + previous_data=previous_value) + + self._data = data + + def update(self, data): + parent = self.invisibleRootItem() + + data = failsafe_deepcopy(data) + previous_data = self._data + self._update_recursive(data, parent, previous_data) + self._data = data # store previous data for next update + + +class DebugUI(QtWidgets.QDialog): + + def __init__(self, parent=None): + super(DebugUI, self).__init__(parent=parent) + self.setStyleSheet(style.load_stylesheet()) + + self._set_window_title() + self.setWindowFlags( + QtCore.Qt.Window + | QtCore.Qt.CustomizeWindowHint + | QtCore.Qt.WindowTitleHint + | QtCore.Qt.WindowMinimizeButtonHint + | QtCore.Qt.WindowCloseButtonHint + | QtCore.Qt.WindowStaysOnTopHint + ) + + layout = QtWidgets.QVBoxLayout(self) + text_edit = QtWidgets.QTextEdit() + text_edit.setFixedHeight(65) + font = QtGui.QFont("NONEXISTENTFONT") + font.setStyleHint(QtGui.QFont.TypeWriter) + text_edit.setFont(font) + text_edit.setLineWrapMode(QtWidgets.QTextEdit.NoWrap) + + step = QtWidgets.QPushButton("Step") + step.setEnabled(False) + + model = DictChangesModel() + proxy = QtCore.QSortFilterProxyModel() + proxy.setRecursiveFilteringEnabled(True) + proxy.setSourceModel(model) + view = QtWidgets.QTreeView() + view.setModel(proxy) + view.setSortingEnabled(True) + + filter_field = QtWidgets.QLineEdit() + filter_field.setPlaceholderText("Filter keys...") + filter_field.textChanged.connect(proxy.setFilterFixedString) + + layout.addWidget(text_edit) + layout.addWidget(filter_field) + layout.addWidget(view) + layout.addWidget(step) + + step.clicked.connect(self.on_step) + + self._pause = False + self.model = model + self.filter = filter_field + self.proxy = proxy + self.view = view + self.text = text_edit + self.step = step + self.resize(700, 500) + + self._previous_data = {} + + def _set_window_title(self, plugin=None): + title = "Pyblish Debug Stepper" + if plugin is not None: + plugin_label = plugin.label or plugin.__name__ + title += f" | {plugin_label}" + self.setWindowTitle(title) + + def pause(self, state): + self._pause = state + self.step.setEnabled(state) + + def on_step(self): + self.pause(False) + + def showEvent(self, event): + print("Registering callback..") + pyblish.api.register_callback("pluginProcessed", + self.on_plugin_processed) + + def hideEvent(self, event): + self.pause(False) + print("Deregistering callback..") + pyblish.api.deregister_callback("pluginProcessed", + self.on_plugin_processed) + + def on_plugin_processed(self, result): + self.pause(True) + + self._set_window_title(plugin=result["plugin"]) + + print(10*"<", result["plugin"].__name__, 10*">") + + plugin_order = result["plugin"].order + plugin_name = result["plugin"].__name__ + duration = result['duration'] + plugin_instance = result["instance"] + context = result["context"] + + msg = "" + msg += f"Order: {plugin_order}
" + msg += f"Plugin: {plugin_name}" + if plugin_instance is not None: + msg += f" -> instance: {plugin_instance}" + msg += "
" + msg += f"Duration: {duration} ms
" + self.text.setHtml(msg) + + data = { + "context": context.data + } + for instance in context: + data[instance.name] = instance.data + self.model.update(data) + + app = QtWidgets.QApplication.instance() + while self._pause: + # Allow user interaction with the UI + app.processEvents() diff --git a/client/ayon_core/tools/experimental_tools/tools_def.py b/client/ayon_core/tools/experimental_tools/tools_def.py index 7def3551de..30e5211b41 100644 --- a/client/ayon_core/tools/experimental_tools/tools_def.py +++ b/client/ayon_core/tools/experimental_tools/tools_def.py @@ -1,4 +1,5 @@ import os +from .pyblish_debug_stepper import DebugUI # Constant key under which local settings are stored LOCAL_EXPERIMENTAL_KEY = "experimental_tools" @@ -95,6 +96,12 @@ class ExperimentalTools: "hiero", "resolve", ] + ), + ExperimentalHostTool( + "pyblish_debug_stepper", + "Pyblish Debug Stepper", + "Debug Pyblish plugins step by step.", + self._show_pyblish_debugger, ) ] @@ -162,9 +169,16 @@ class ExperimentalTools: local_settings.get(LOCAL_EXPERIMENTAL_KEY) ) or {} - for identifier, eperimental_tool in self.tools_by_identifier.items(): + # Enable the following tools by default. + # Because they will always be disabled due + # to the fact their settings don't exist. + experimental_settings.update({ + "pyblish_debug_stepper": True, + }) + + for identifier, experimental_tool in self.tools_by_identifier.items(): enabled = experimental_settings.get(identifier, False) - eperimental_tool.set_enabled(enabled) + experimental_tool.set_enabled(enabled) def _show_publisher(self): if self._publisher_tool is None: @@ -175,3 +189,7 @@ class ExperimentalTools: ) self._publisher_tool.show() + + def _show_pyblish_debugger(self): + window = DebugUI(parent=self._parent_widget) + window.show() diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 458129f367..9a951d7fd4 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON addon 'core' version.""" -__version__ = "1.0.1+dev" +__version__ = "1.0.3+dev" diff --git a/package.py b/package.py index c059eed423..5d5218748c 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.1+dev" +version = "1.0.3+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index 0abec2d5b1..641faf2536 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.1+dev" +version = "1.0.3+dev" description = "" authors = ["Ynput Team "] readme = "README.md"