Merge remote-tracking branch 'origin/develop' into feature/909-define-basic-trait-type-using-dataclasses

client/ayon_core/pipeline/delivery.py

@@ -3,11 +3,20 @@ import os
 import copy
 import shutil
 import glob
-import clique
 import collections
+from typing import Dict, Any, Iterable
+
+import clique
+import ayon_api
 
 from ayon_core.lib import create_hard_link
 
+from .template_data import (
+    get_general_template_data,
+    get_folder_template_data,
+    get_task_template_data,
+)
+
 
 def _copy_file(src_path, dst_path):
     """Hardlink file if possible(to save space), copy if not.
@@ -327,3 +336,71 @@ def deliver_sequence(
         uploaded += 1
 
     return report_items, uploaded
+
+
+def _merge_data(data, new_data):
+    queue = collections.deque()
+    queue.append((data, new_data))
+    while queue:
+        q_data, q_new_data = queue.popleft()
+        for key, value in q_new_data.items():
+            if key in q_data and isinstance(value, dict):
+                queue.append((q_data[key], value))
+                continue
+            q_data[key] = value
+
+
+def get_representations_delivery_template_data(
+    project_name: str,
+    representation_ids: Iterable[str],
+) -> Dict[str, Dict[str, Any]]:
+    representation_ids = set(representation_ids)
+
+    output = {
+        repre_id: {}
+        for repre_id in representation_ids
+    }
+    if not representation_ids:
+        return output
+
+    project_entity = ayon_api.get_project(project_name)
+
+    general_template_data = get_general_template_data()
+
+    repres_hierarchy = ayon_api.get_representations_hierarchy(
+        project_name,
+        representation_ids,
+        project_fields=set(),
+        folder_fields={"path", "folderType"},
+        task_fields={"name", "taskType"},
+        product_fields={"name", "productType"},
+        version_fields={"version", "productId"},
+        representation_fields=None,
+    )
+    for repre_id, repre_hierarchy in repres_hierarchy.items():
+        repre_entity = repre_hierarchy.representation
+        if repre_entity is None:
+            continue
+
+        template_data = repre_entity["context"]
+        template_data.update(copy.deepcopy(general_template_data))
+        template_data.update(get_folder_template_data(
+            repre_hierarchy.folder, project_name
+        ))
+        if repre_hierarchy.task:
+            template_data.update(get_task_template_data(
+                project_entity, repre_hierarchy.task
+            ))
+
+        product_entity = repre_hierarchy.product
+        version_entity = repre_hierarchy.version
+        template_data.update({
+            "product": {
+                "name": product_entity["name"],
+                "type": product_entity["productType"],
+            },
+            "version": version_entity["version"],
+        })
+        _merge_data(template_data, repre_entity["context"])
+        output[repre_id] = template_data
+    return output

client/ayon_core/pipeline/farm/pyblish_functions.py

@@ -788,15 +788,15 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
                 colorspace = product.colorspace
                 break
 
-        if isinstance(files, (list, tuple)):
-            files = [os.path.basename(f) for f in files]
+        if isinstance(collected_files, (list, tuple)):
+            collected_files = [os.path.basename(f) for f in collected_files]
         else:
-            files = os.path.basename(files)
+            collected_files = os.path.basename(collected_files)
 
         rep = {
             "name": ext,
             "ext": ext,
-            "files": files,
+            "files": collected_files,
             "frameStart": int(skeleton["frameStartHandle"]),
             "frameEnd": int(skeleton["frameEndHandle"]),
             # If expectedFile are absolute, we need only filenames

client/ayon_core/plugins/load/delivery.py

@@ -1,23 +1,22 @@
-import copy
 import platform
 from collections import defaultdict
 
 import ayon_api
 from qtpy import QtWidgets, QtCore, QtGui
 
-from ayon_core.pipeline import load, Anatomy
 from ayon_core import resources, style
-
 from ayon_core.lib import (
     format_file_size,
     collect_frames,
     get_datetime_data,
 )
+from ayon_core.pipeline import load, Anatomy
 from ayon_core.pipeline.load import get_representation_path_with_anatomy
 from ayon_core.pipeline.delivery import (
     get_format_dict,
     check_destination_path,
-    deliver_single_file
+    deliver_single_file,
+    get_representations_delivery_template_data,
 )
 
 
@@ -200,20 +199,31 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
         format_dict = get_format_dict(self.anatomy, self.root_line_edit.text())
         renumber_frame = self.renumber_frame.isChecked()
         frame_offset = self.first_frame_start.value()
+        filtered_repres = []
+        repre_ids = set()
         for repre in self._representations:
-            if repre["name"] not in selected_repres:
-                continue
+            if repre["name"] in selected_repres:
+                filtered_repres.append(repre)
+                repre_ids.add(repre["id"])
 
+        template_data_by_repre_id = (
+            get_representations_delivery_template_data(
+                self.anatomy.project_name, repre_ids
+            )
+        )
+        for repre in filtered_repres:
             repre_path = get_representation_path_with_anatomy(
                 repre, self.anatomy
             )
 
-            anatomy_data = copy.deepcopy(repre["context"])
-            new_report_items = check_destination_path(repre["id"],
-                                                      self.anatomy,
-                                                      anatomy_data,
-                                                      datetime_data,
-                                                      template_name)
+            template_data = template_data_by_repre_id[repre["id"]]
+            new_report_items = check_destination_path(
+                repre["id"],
+                self.anatomy,
+                template_data,
+                datetime_data,
+                template_name
+            )
 
             report_items.update(new_report_items)
             if new_report_items:
@@ -224,7 +234,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
                 repre,
                 self.anatomy,
                 template_name,
-                anatomy_data,
+                template_data,
                 format_dict,
                 report_items,
                 self.log
@@ -267,9 +277,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
 
                 if frame is not None:
                     if repre["context"].get("frame"):
-                        anatomy_data["frame"] = frame
+                        template_data["frame"] = frame
                     elif repre["context"].get("udim"):
-                        anatomy_data["udim"] = frame
+                        template_data["udim"] = frame
                     else:
                         # Fallback
                         self.log.warning(
@@ -277,7 +287,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
                             " data. Supplying sequence frame to '{frame}'"
                             " formatting data."
                         )
-                        anatomy_data["frame"] = frame
+                        template_data["frame"] = frame
                 new_report_items, uploaded = deliver_single_file(*args)
                 report_items.update(new_report_items)
                 self._update_progress(uploaded)
@@ -342,8 +352,8 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
     def _get_selected_repres(self):
         """Returns list of representation names filtered from checkboxes."""
         selected_repres = []
-        for repre_name, chckbox in self._representation_checkboxes.items():
-            if chckbox.isChecked():
+        for repre_name, checkbox in self._representation_checkboxes.items():
+            if checkbox.isChecked():
                 selected_repres.append(repre_name)
 
         return selected_repres

client/ayon_core/plugins/publish/extract_thumbnail.py

@@ -36,7 +36,8 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
         "traypublisher",
         "substancepainter",
         "nuke",
-        "aftereffects"
+        "aftereffects",
+        "unreal"
     ]
     enabled = False
 

client/ayon_core/tools/experimental_tools/pyblish_debug_stepper.py

@@ -0,0 +1,273 @@
+"""
+Brought from https://gist.github.com/BigRoy/1972822065e38f8fae7521078e44eca2
+Code Credits: [BigRoy](https://github.com/BigRoy)
+
+Requirement:
+    It requires pyblish version >= 1.8.12
+
+How it works:
+    This tool makes use of pyblish event `pluginProcessed` to:
+        1. Pause the publishing.
+        2. Collect some info about the plugin.
+        3. Show that info to the tool's window.
+        4. Continue publishing on clicking `step` button.
+
+How to use it:
+    1. Launch the tool from AYON experimental tools window.
+    2. Launch the publisher tool and click validate.
+    3. Click Step to run plugins one by one.
+
+Note :
+    Pyblish debugger also works when triggering the validation or
+      publishing from code.
+    Here's an example about validating from code:
+    https://github.com/MustafaJafar/ayon-recipes/blob/main/validate_from_code.py
+
+"""
+
+import copy
+import json
+from qtpy import QtWidgets, QtCore, QtGui
+
+import pyblish.api
+from ayon_core import style
+
+TAB = 4* " "
+HEADER_SIZE = "15px"
+
+KEY_COLOR = QtGui.QColor("#ffffff")
+NEW_KEY_COLOR = QtGui.QColor("#00ff00")
+VALUE_TYPE_COLOR = QtGui.QColor("#ffbbbb")
+NEW_VALUE_TYPE_COLOR = QtGui.QColor("#ff4444")
+VALUE_COLOR = QtGui.QColor("#777799")
+NEW_VALUE_COLOR = QtGui.QColor("#DDDDCC")
+CHANGED_VALUE_COLOR = QtGui.QColor("#CCFFCC")
+
+MAX_VALUE_STR_LEN = 100
+
+
+def failsafe_deepcopy(data):
+    """Allow skipping the deepcopy for unsupported types"""
+    try:
+        return copy.deepcopy(data)
+    except TypeError:
+        if isinstance(data, dict):
+            return {
+                key: failsafe_deepcopy(value)
+                for key, value in data.items()
+            }
+        elif isinstance(data, list):
+            return data.copy()
+    return data
+
+
+class DictChangesModel(QtGui.QStandardItemModel):
+    # TODO: Replace this with a QAbstractItemModel
+    def __init__(self, *args, **kwargs):
+        super(DictChangesModel, self).__init__(*args, **kwargs)
+        self._data = {}
+
+        columns = ["Key", "Type", "Value"]
+        self.setColumnCount(len(columns))
+        for i, label in enumerate(columns):
+            self.setHeaderData(i, QtCore.Qt.Horizontal, label)
+
+    def _update_recursive(self, data, parent, previous_data):
+        for key, value in data.items():
+
+            # Find existing item or add new row
+            parent_index = parent.index()
+            for row in range(self.rowCount(parent_index)):
+                # Update existing item if it exists
+                index = self.index(row, 0, parent_index)
+                if index.data() == key:
+                    item = self.itemFromIndex(index)
+                    type_item = self.itemFromIndex(self.index(row, 1, parent_index))  # noqa
+                    value_item = self.itemFromIndex(self.index(row, 2, parent_index))  # noqa
+                    break
+            else:
+                item = QtGui.QStandardItem(key)
+                type_item = QtGui.QStandardItem()
+                value_item = QtGui.QStandardItem()
+                parent.appendRow([item, type_item, value_item])
+
+            # Key
+            key_color = NEW_KEY_COLOR if key not in previous_data else KEY_COLOR  # noqa
+            item.setData(key_color, QtCore.Qt.ForegroundRole)
+
+            # Type
+            type_str = type(value).__name__
+            type_color = VALUE_TYPE_COLOR
+            if (
+                key in previous_data
+                and type(previous_data[key]).__name__ != type_str
+            ):
+                type_color = NEW_VALUE_TYPE_COLOR
+
+            type_item.setText(type_str)
+            type_item.setData(type_color, QtCore.Qt.ForegroundRole)
+
+            # Value
+            value_changed = False
+            if key not in previous_data or previous_data[key] != value:
+                value_changed = True
+            value_color = NEW_VALUE_COLOR if value_changed else VALUE_COLOR
+
+            value_item.setData(value_color, QtCore.Qt.ForegroundRole)
+            if value_changed:
+                value_str = str(value)
+                if len(value_str) > MAX_VALUE_STR_LEN:
+                    value_str = value_str[:MAX_VALUE_STR_LEN] + "..."
+                value_item.setText(value_str)
+
+                # Preferably this is deferred to only when the data gets
+                # requested since this formatting can be slow for very large
+                # data sets like project settings and system settings
+                # This will also be MUCH faster if we don't clear the
+                # items on each update but only updated/add/remove changed
+                # items so that this also runs much less often
+                value_item.setData(
+                    json.dumps(value, default=str, indent=4),
+                    QtCore.Qt.ToolTipRole
+                )
+
+            if isinstance(value, dict):
+                previous_value = previous_data.get(key, {})
+                if previous_data.get(key) != value:
+                    # Update children if the value is not the same as before
+                    self._update_recursive(value,
+                                           parent=item,
+                                           previous_data=previous_value)
+                else:
+                    # TODO: Ensure all children are updated to be not marked
+                    #  as 'changed' in the most optimal way possible
+                    self._update_recursive(value,
+                                           parent=item,
+                                           previous_data=previous_value)
+
+        self._data = data
+
+    def update(self, data):
+        parent = self.invisibleRootItem()
+
+        data = failsafe_deepcopy(data)
+        previous_data = self._data
+        self._update_recursive(data, parent, previous_data)
+        self._data = data  # store previous data for next update
+
+
+class DebugUI(QtWidgets.QDialog):
+
+    def __init__(self, parent=None):
+        super(DebugUI, self).__init__(parent=parent)
+        self.setStyleSheet(style.load_stylesheet())
+
+        self._set_window_title()
+        self.setWindowFlags(
+            QtCore.Qt.Window
+            | QtCore.Qt.CustomizeWindowHint
+            | QtCore.Qt.WindowTitleHint
+            | QtCore.Qt.WindowMinimizeButtonHint
+            | QtCore.Qt.WindowCloseButtonHint
+            | QtCore.Qt.WindowStaysOnTopHint
+        )
+
+        layout = QtWidgets.QVBoxLayout(self)
+        text_edit = QtWidgets.QTextEdit()
+        text_edit.setFixedHeight(65)
+        font = QtGui.QFont("NONEXISTENTFONT")
+        font.setStyleHint(QtGui.QFont.TypeWriter)
+        text_edit.setFont(font)
+        text_edit.setLineWrapMode(QtWidgets.QTextEdit.NoWrap)
+
+        step = QtWidgets.QPushButton("Step")
+        step.setEnabled(False)
+
+        model = DictChangesModel()
+        proxy = QtCore.QSortFilterProxyModel()
+        proxy.setRecursiveFilteringEnabled(True)
+        proxy.setSourceModel(model)
+        view = QtWidgets.QTreeView()
+        view.setModel(proxy)
+        view.setSortingEnabled(True)
+
+        filter_field = QtWidgets.QLineEdit()
+        filter_field.setPlaceholderText("Filter keys...")
+        filter_field.textChanged.connect(proxy.setFilterFixedString)
+
+        layout.addWidget(text_edit)
+        layout.addWidget(filter_field)
+        layout.addWidget(view)
+        layout.addWidget(step)
+
+        step.clicked.connect(self.on_step)
+
+        self._pause = False
+        self.model = model
+        self.filter = filter_field
+        self.proxy = proxy
+        self.view = view
+        self.text = text_edit
+        self.step = step
+        self.resize(700, 500)
+
+        self._previous_data = {}
+
+    def _set_window_title(self, plugin=None):
+        title = "Pyblish Debug Stepper"
+        if plugin is not None:
+            plugin_label = plugin.label or plugin.__name__
+            title += f" | {plugin_label}"
+        self.setWindowTitle(title)
+
+    def pause(self, state):
+        self._pause = state
+        self.step.setEnabled(state)
+
+    def on_step(self):
+        self.pause(False)
+
+    def showEvent(self, event):
+        print("Registering callback..")
+        pyblish.api.register_callback("pluginProcessed",
+                                      self.on_plugin_processed)
+
+    def hideEvent(self, event):
+        self.pause(False)
+        print("Deregistering callback..")
+        pyblish.api.deregister_callback("pluginProcessed",
+                                        self.on_plugin_processed)
+
+    def on_plugin_processed(self, result):
+        self.pause(True)
+
+        self._set_window_title(plugin=result["plugin"])
+
+        print(10*"<", result["plugin"].__name__, 10*">")
+
+        plugin_order = result["plugin"].order
+        plugin_name = result["plugin"].__name__
+        duration = result['duration']
+        plugin_instance = result["instance"]
+        context = result["context"]
+
+        msg = ""
+        msg += f"Order: {plugin_order}<br>"
+        msg += f"Plugin: {plugin_name}"
+        if plugin_instance is not None:
+            msg += f" -> instance: {plugin_instance}"
+        msg += "<br>"
+        msg += f"Duration: {duration} ms<br>"
+        self.text.setHtml(msg)
+
+        data = {
+            "context": context.data
+        }
+        for instance in context:
+            data[instance.name] = instance.data
+        self.model.update(data)
+
+        app = QtWidgets.QApplication.instance()
+        while self._pause:
+            # Allow user interaction with the UI
+            app.processEvents()

client/ayon_core/tools/experimental_tools/tools_def.py

@@ -1,4 +1,5 @@
 import os
+from .pyblish_debug_stepper import DebugUI
 
 # Constant key under which local settings are stored
 LOCAL_EXPERIMENTAL_KEY = "experimental_tools"
@@ -95,6 +96,12 @@ class ExperimentalTools:
                     "hiero",
                     "resolve",
                 ]
+            ),
+            ExperimentalHostTool(
+                "pyblish_debug_stepper",
+                "Pyblish Debug Stepper",
+                "Debug Pyblish plugins step by step.",
+                self._show_pyblish_debugger,
             )
         ]
 
@@ -162,9 +169,16 @@ class ExperimentalTools:
             local_settings.get(LOCAL_EXPERIMENTAL_KEY)
         ) or {}
 
-        for identifier, eperimental_tool in self.tools_by_identifier.items():
+        # Enable the following tools by default.
+        # Because they will always be disabled due
+        #   to the fact their settings don't exist.
+        experimental_settings.update({
+            "pyblish_debug_stepper": True,
+        })
+
+        for identifier, experimental_tool in self.tools_by_identifier.items():
             enabled = experimental_settings.get(identifier, False)
-            eperimental_tool.set_enabled(enabled)
+            experimental_tool.set_enabled(enabled)
 
     def _show_publisher(self):
         if self._publisher_tool is None:
@@ -175,3 +189,7 @@ class ExperimentalTools:
             )
 
         self._publisher_tool.show()
+
+    def _show_pyblish_debugger(self):
+        window = DebugUI(parent=self._parent_widget)
+        window.show()

client/ayon_core/version.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 """Package declaring AYON addon 'core' version."""
-__version__ = "1.0.1+dev"
+__version__ = "1.0.3+dev"

package.py

@@ -1,6 +1,6 @@
 name = "core"
 title = "Core"
-version = "1.0.1+dev"
+version = "1.0.3+dev"
 
 client_dir = "ayon_core"
 

pyproject.toml

@@ -5,7 +5,7 @@
 
 [tool.poetry]
 name = "ayon-core"
-version = "1.0.1+dev"
+version = "1.0.3+dev"
 description = ""
 authors = ["Ynput Team <team@ynput.io>"]
 readme = "README.md"