From 09ad8cb4f1d9f6e8f7c72f99633e97f0bfa4fd4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Tue, 21 Nov 2023 17:12:19 +0100 Subject: [PATCH 1/8] :art: launcher action this needs to be moved to subprocess to show UI properly --- .../actions/generate_asset_usage_report.py | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 openpype/plugins/actions/generate_asset_usage_report.py diff --git a/openpype/plugins/actions/generate_asset_usage_report.py b/openpype/plugins/actions/generate_asset_usage_report.py new file mode 100644 index 0000000000..60630c5a43 --- /dev/null +++ b/openpype/plugins/actions/generate_asset_usage_report.py @@ -0,0 +1,157 @@ +""" +TODO: we need to move it to subprocess to show UI +""" +import csv +import os +import tempfile +import time + +from qtpy import QtWidgets +from qtpy.QtCore import Qt +from qtpy.QtGui import QClipboard + +from pymongo.collection import Collection + +from openpype.client import OpenPypeMongoConnection +from openpype.pipeline import LauncherAction + + +class ReportWindow(QtWidgets.QWidget): + def __init__(self): + super().__init__() + + self.text_area = QtWidgets.QTextEdit(self) + self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) + self.save_button = QtWidgets.QPushButton('Save to CSV File', self) + + self.copy_button.clicked.connect(self.copy_to_clipboard) + self.save_button.clicked.connect(self.save_to_file) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self.text_area) + layout.addWidget(self.copy_button) + layout.addWidget(self.save_button) + + def copy_to_clipboard(self): + clipboard = QtWidgets.QApplication.clipboard() + clipboard.setText(self.text_area.toPlainText(), QClipboard.Clipboard) + + def save_to_file(self): + file_name, _ = QtWidgets.QFileDialog.getSaveFileName(self, 'Save File') + if file_name: + with open(file_name, 'w') as file: + file.write(self.text_area.toPlainText()) + + def set_content(self, content): + self.text_area.setText(content) + + +class OpenTaskPath(LauncherAction): + name = "get_asset_usage_report" + label = "Asset Usage Report" + icon = "list" + order = 500 + + def is_compatible(self, session): + """Return whether the action is compatible with the session""" + return bool(session.get("AVALON_ASSET")) + + def _get_subset(self, version_id, project: Collection): + pipeline = [ + { + "$match": { + "_id": version_id + }, + }, { + "$lookup": { + "from": project.name, + "localField": "parent", + "foreignField": "_id", + "as": "parents" + } + } + ] + + result = project.aggregate(pipeline) + doc = next(result) + # print(doc) + return { + "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', + "family": doc["data"].get("family") or doc["data"].get("families")[0] + } + + def process(self, session, **kwargs): + start = time.perf_counter() + project = session["AVALON_PROJECT"] + + pipeline = [ + { + "$match": { + "data.inputLinks": { + "$exists": True, + "$ne": [] + }, + "data.families": {"$in": ["workfile"]} + } + }, { + "$lookup": { + "from": "OP01_CG_demo", + "localField": "data.inputLinks.id", + "foreignField": "_id", + "as": "linked_docs" + } + } + ] + + client = OpenPypeMongoConnection.get_mongo_client() + db = client["avalon"] + + result = db[project].aggregate(pipeline) + + asset_map = [] + for doc in result: + source = { + "source": self._get_subset(doc["parent"], db[project]), + } + source["source"].update({"version": doc["name"]}) + refs = [ + { + "subset": self._get_subset(linked["parent"], db[project]), + "version": linked.get("name") + } + for linked in doc["linked_docs"] + ] + source["refs"] = refs + asset_map.append(source) + + # for ref in asset_map: + # print(ref) + + grouped = {} + + for asset in asset_map: + for ref in asset["refs"]: + key = f'{ref["subset"]["name"]} (v{ref["version"]})' + if key in grouped: + grouped[key].append(asset["source"]) + else: + grouped[key] = [asset["source"]] + + temp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".csv") + try: + with open(temp.name, "w", newline="") as csvfile: + writer = csv.writer(csvfile, delimiter=";") + writer.writerow(["Subset", "Used in", "Version"]) + for key, value in grouped.items(): + writer.writerow([key, "", ""]) + for source in value: + writer.writerow(["", source["name"], source["version"]]) + finally: + temp.close() + + end = time.perf_counter() + app = QtWidgets.QApplication.instance() + window = ReportWindow() + # window.set_content(open(temp.name).read()) + window.show() + print(f"Finished in {end - start:0.4f} seconds", 2) From 61fe4825e61d707e914387c0c54e1d5a9e667ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 22 Nov 2023 16:25:33 +0100 Subject: [PATCH 2/8] :art: initial work --- openpype/modules/asset_reporter/__init__.py | 8 + openpype/modules/asset_reporter/module.py | 30 ++ openpype/modules/asset_reporter/window.py | 378 ++++++++++++++++++ .../defaults/system_settings/modules.json | 3 + .../schemas/system_schema/schema_modules.json | 14 + 5 files changed, 433 insertions(+) create mode 100644 openpype/modules/asset_reporter/__init__.py create mode 100644 openpype/modules/asset_reporter/module.py create mode 100644 openpype/modules/asset_reporter/window.py diff --git a/openpype/modules/asset_reporter/__init__.py b/openpype/modules/asset_reporter/__init__.py new file mode 100644 index 0000000000..6267b4824b --- /dev/null +++ b/openpype/modules/asset_reporter/__init__.py @@ -0,0 +1,8 @@ +from .module import ( + AssetReporterAction +) + + +__all__ = ( + "AssetReporterAction", +) diff --git a/openpype/modules/asset_reporter/module.py b/openpype/modules/asset_reporter/module.py new file mode 100644 index 0000000000..34b84795d5 --- /dev/null +++ b/openpype/modules/asset_reporter/module.py @@ -0,0 +1,30 @@ +import os.path + +from openpype import AYON_SERVER_ENABLED +from openpype.modules import OpenPypeModule, ITrayAction +from openpype.lib import run_detached_process, get_openpype_execute_args + + +class AssetReporterAction(OpenPypeModule, ITrayAction): + label = "Asset Usage Report" + name = "asset_reporter" + admin_action = True + + def initialize(self, modules_settings): + self.enabled = not AYON_SERVER_ENABLED + + def tray_init(self): + ... + + def tray_exit(self): + ... + + def on_action_trigger(self): + args = get_openpype_execute_args() + args += ["run", + os.path.join( + os.path.dirname(__file__), + "window.py")] + + print(" ".join(args)) + run_detached_process(args) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py new file mode 100644 index 0000000000..6cb2f0e0e0 --- /dev/null +++ b/openpype/modules/asset_reporter/window.py @@ -0,0 +1,378 @@ +import appdirs +import qtawesome +import csv +import tempfile +from qtpy import QtCore, QtWidgets +from qtpy.QtGui import QClipboard, QColor +import time + +from openpype import ( + resources, + style +) +from openpype.client import OpenPypeMongoConnection +from pymongo.collection import Collection +from openpype.lib import JSONSettingRegistry +from openpype.tools.utils import PlaceholderLineEdit, get_openpype_qt_app +from openpype.tools.utils.constants import PROJECT_NAME_ROLE +from openpype.tools.utils.models import ProjectModel, ProjectSortFilterProxy + + +class AssetReporterRegistry(JSONSettingRegistry): + """Class handling OpenPype general settings registry. + + Attributes: + vendor (str): Name used for path construction. + product (str): Additional name used for path construction. + + """ + + def __init__(self): + self.vendor = "ynput" + self.product = "openpype" + name = "asset_usage_reporter" + path = appdirs.user_data_dir(self.product, self.vendor) + super(AssetReporterRegistry, self).__init__(name, path) + + +class OverlayWidget(QtWidgets.QFrame): + project_selected = QtCore.Signal(str) + + def __init__(self, publisher_window): + super(OverlayWidget, self).__init__(publisher_window) + self.setObjectName("OverlayFrame") + + middle_frame = QtWidgets.QFrame(self) + middle_frame.setObjectName("ChooseProjectFrame") + + content_widget = QtWidgets.QWidget(middle_frame) + + header_label = QtWidgets.QLabel("Choose project", content_widget) + header_label.setObjectName("ChooseProjectLabel") + # Create project models and view + projects_model = ProjectModel() + projects_proxy = ProjectSortFilterProxy() + projects_proxy.setSourceModel(projects_model) + projects_proxy.setFilterKeyColumn(0) + + projects_view = QtWidgets.QListView(content_widget) + projects_view.setObjectName("ChooseProjectView") + projects_view.setModel(projects_proxy) + projects_view.setEditTriggers( + QtWidgets.QAbstractItemView.NoEditTriggers + ) + + confirm_btn = QtWidgets.QPushButton("Confirm", content_widget) + cancel_btn = QtWidgets.QPushButton("Cancel", content_widget) + cancel_btn.setVisible(False) + btns_layout = QtWidgets.QHBoxLayout() + btns_layout.addStretch(1) + btns_layout.addWidget(cancel_btn, 0) + btns_layout.addWidget(confirm_btn, 0) + + txt_filter = PlaceholderLineEdit(content_widget) + txt_filter.setPlaceholderText("Quick filter projects..") + txt_filter.setClearButtonEnabled(True) + txt_filter.addAction(qtawesome.icon("fa.filter", color="gray"), + QtWidgets.QLineEdit.LeadingPosition) + + content_layout = QtWidgets.QVBoxLayout(content_widget) + content_layout.setContentsMargins(0, 0, 0, 0) + content_layout.setSpacing(20) + content_layout.addWidget(header_label, 0) + content_layout.addWidget(txt_filter, 0) + content_layout.addWidget(projects_view, 1) + content_layout.addLayout(btns_layout, 0) + + middle_layout = QtWidgets.QHBoxLayout(middle_frame) + middle_layout.setContentsMargins(30, 30, 10, 10) + middle_layout.addWidget(content_widget) + + main_layout = QtWidgets.QHBoxLayout(self) + main_layout.setContentsMargins(10, 10, 10, 10) + main_layout.addStretch(1) + main_layout.addWidget(middle_frame, 2) + main_layout.addStretch(1) + + projects_view.doubleClicked.connect(self._on_double_click) + confirm_btn.clicked.connect(self._on_confirm_click) + cancel_btn.clicked.connect(self._on_cancel_click) + txt_filter.textChanged.connect(self._on_text_changed) + + self._projects_view = projects_view + self._projects_model = projects_model + self._projects_proxy = projects_proxy + self._cancel_btn = cancel_btn + self._confirm_btn = confirm_btn + self._txt_filter = txt_filter + + self._publisher_window = publisher_window + self._project_name = None + + def showEvent(self, event): + self._projects_model.refresh() + # Sort projects after refresh + self._projects_proxy.sort(0) + + setting_registry = AssetReporterRegistry() + try: + project_name = setting_registry.get_item("project_name") + except ValueError: + project_name = None + + if project_name: + index = None + src_index = self._projects_model.find_project(project_name) + if src_index is not None: + index = self._projects_proxy.mapFromSource(src_index) + + if index is not None: + selection_model = self._projects_view.selectionModel() + selection_model.select( + index, + QtCore.QItemSelectionModel.SelectCurrent + ) + self._projects_view.setCurrentIndex(index) + + self._cancel_btn.setVisible(self._project_name is not None) + super(OverlayWidget, self).showEvent(event) + + def _on_double_click(self): + self.set_selected_project() + + def _on_confirm_click(self): + self.set_selected_project() + + def _on_cancel_click(self): + self._set_project(self._project_name) + + def _on_text_changed(self): + self._projects_proxy.setFilterRegularExpression( + self._txt_filter.text()) + + def set_selected_project(self): + index = self._projects_view.currentIndex() + + if project_name := index.data(PROJECT_NAME_ROLE): + self._set_project(project_name) + + def _set_project(self, project_name): + self._project_name = project_name + self.setVisible(False) + self.project_selected.emit(project_name) + + setting_registry = AssetReporterRegistry() + setting_registry.set_item("project_name", project_name) + + +class AssetReporterWindow(QtWidgets.QDialog): + default_width = 1300 + default_height = 800 + footer_border = 8 + _content = None + + def __init__(self, parent=None, controller=None, reset_on_show=None): + super(AssetReporterWindow, self).__init__(parent) + + self._result = {} + self.setObjectName("AssetReporterWindow") + + self.setWindowTitle("Asset Usage Reporter") + + if parent is None: + on_top_flag = QtCore.Qt.WindowStaysOnTopHint + else: + on_top_flag = QtCore.Qt.Dialog + + self.setWindowFlags( + QtCore.Qt.WindowTitleHint + | QtCore.Qt.WindowMaximizeButtonHint + | QtCore.Qt.WindowMinimizeButtonHint + | QtCore.Qt.WindowCloseButtonHint + | on_top_flag + ) + self.table = QtWidgets.QTableWidget(self) + self.table.setColumnCount(3) + self.table.setColumnWidth(0, 400) + self.table.setColumnWidth(1, 300) + self.table.setHorizontalHeaderLabels(["Subset", "Used in", "Version"]) + + # self.text_area = QtWidgets.QTextEdit(self) + self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) + self.save_button = QtWidgets.QPushButton('Save to CSV File', self) + + self.copy_button.clicked.connect(self.copy_to_clipboard) + self.save_button.clicked.connect(self.save_to_file) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(self.table) + # layout.addWidget(self.text_area) + layout.addWidget(self.copy_button) + layout.addWidget(self.save_button) + + self.resize(self.default_width, self.default_height) + self.setStyleSheet(style.load_stylesheet()) + + overlay_widget = OverlayWidget(self) + overlay_widget.project_selected.connect(self._on_project_select) + self._overlay_widget = overlay_widget + + def _on_project_select(self, project_name): + self._project_name = project_name + self.process() + if not self._result: + self.set_content("no result generated") + return + + rows = sum(len(value) for key, value in self._result.items()) + self.table.setRowCount(rows) + + row = 0 + content = [] + for key, value in self._result.items(): + item = QtWidgets.QTableWidgetItem(key) + item.setBackground(QColor(32, 32, 32)) + self.table.setItem(row, 0, item) + for source in value: + self.table.setItem(row, 1, QtWidgets.QTableWidgetItem(source["name"])) + self.table.setItem(row, 2, QtWidgets.QTableWidgetItem(str(source["version"]))) + print(f' - {source["version"]}') + row += 1 + + content.append(f"{key}") + content.extend( + f"\t{source['name']} (v{source['version']})" for source in value + ) + self.set_content("\n".join(content)) + + def copy_to_clipboard(self): + clipboard = QtWidgets.QApplication.clipboard() + clipboard.setText(self._content, QClipboard.Clipboard) + + def save_to_file(self): + file_name, _ = QtWidgets.QFileDialog.getSaveFileName(self, 'Save File') + if file_name: + self._write_csv(file_name) + + def set_content(self, content): + self._content = content + + def get_content(self): + return self._content + + def _resize_overlay(self): + self._overlay_widget.resize( + self.width(), + self.height() + ) + + def resizeEvent(self, event): + super(AssetReporterWindow, self).resizeEvent(event) + self._resize_overlay() + + def _get_subset(self, version_id, project: Collection): + pipeline = [ + { + "$match": { + "_id": version_id + }, + }, { + "$lookup": { + "from": project.name, + "localField": "parent", + "foreignField": "_id", + "as": "parents" + } + } + ] + + result = project.aggregate(pipeline) + doc = next(result) + # print(doc) + return { + "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', + "family": doc["data"].get("family") or doc["data"].get("families")[0] + } + + def process(self): + start = time.perf_counter() + project = self._project_name + + pipeline = [ + { + "$match": { + "data.inputLinks": { + "$exists": True, + "$ne": [] + }, + "data.families": {"$in": ["workfile"]} + } + }, { + "$lookup": { + "from": project, + "localField": "data.inputLinks.id", + "foreignField": "_id", + "as": "linked_docs" + } + } + ] + + client = OpenPypeMongoConnection.get_mongo_client() + db = client["avalon"] + + result = db[project].aggregate(pipeline) + + asset_map = [] + for doc in result: + source = { + "source": self._get_subset(doc["parent"], db[project]), + } + source["source"].update({"version": doc["name"]}) + refs = [ + { + "subset": self._get_subset(linked["parent"], db[project]), + "version": linked.get("name") + } + for linked in doc["linked_docs"] + ] + source["refs"] = refs + asset_map.append(source) + + # for ref in asset_map: + # print(ref) + + grouped = {} + + for asset in asset_map: + for ref in asset["refs"]: + key = f'{ref["subset"]["name"]} (v{ref["version"]})' + if key in grouped: + grouped[key].append(asset["source"]) + else: + grouped[key] = [asset["source"]] + self._result = grouped + + end = time.perf_counter() + + print(f"Finished in {end - start:0.4f} seconds", 2) + + def _write_csv(self, file_name): + with open(file_name, "w", newline="") as csvfile: + writer = csv.writer(csvfile, delimiter=";") + writer.writerow(["Subset", "Used in", "Version"]) + for key, value in self._result.items(): + writer.writerow([key, "", ""]) + for source in value: + writer.writerow(["", source["name"], source["version"]]) + + + +def main(): + app_instance = get_openpype_qt_app() + window = AssetReporterWindow() + window.show() + app_instance.exec_() + + +if __name__ == "__main__": + main() diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index f524f01d45..bb943524f1 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -210,5 +210,8 @@ "darwin": "", "linux": "" } + }, + "asset_reporter": { + "enabled": false } } diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 952b38040c..2258dd42e3 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -355,6 +355,20 @@ { "type": "dynamic_schema", "name": "system_settings/modules" + }, + { + "type": "dict", + "key": "asset_reporter", + "label": "Asset Usage Report", + "collapsible": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + } + ] } ] } From 5125db72b4ad2ece64e20bef0406fb9596df29d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 22 Nov 2023 16:38:09 +0100 Subject: [PATCH 3/8] :memo: add docstrings and some polishing --- openpype/modules/asset_reporter/module.py | 11 ++- openpype/modules/asset_reporter/window.py | 71 +++++++++++++------ .../schemas/system_schema/schema_modules.json | 2 +- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/openpype/modules/asset_reporter/module.py b/openpype/modules/asset_reporter/module.py index 34b84795d5..8c754cc3c0 100644 --- a/openpype/modules/asset_reporter/module.py +++ b/openpype/modules/asset_reporter/module.py @@ -6,19 +6,16 @@ from openpype.lib import run_detached_process, get_openpype_execute_args class AssetReporterAction(OpenPypeModule, ITrayAction): + label = "Asset Usage Report" name = "asset_reporter" - admin_action = True + + def tray_init(self): + pass def initialize(self, modules_settings): self.enabled = not AYON_SERVER_ENABLED - def tray_init(self): - ... - - def tray_exit(self): - ... - def on_action_trigger(self): args = get_openpype_execute_args() args += ["run", diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index 6cb2f0e0e0..9dbcc3ea74 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -1,17 +1,22 @@ -import appdirs -import qtawesome +"""Tool for generating asset usage report. + +This tool is used to generate asset usage report for a project. +It is using links between published version to find out where +the asset is used. + +""" + import csv -import tempfile -from qtpy import QtCore, QtWidgets -from qtpy.QtGui import QClipboard, QColor import time -from openpype import ( - resources, - style -) -from openpype.client import OpenPypeMongoConnection +import appdirs +import qtawesome from pymongo.collection import Collection +from qtpy import QtCore, QtWidgets +from qtpy.QtGui import QClipboard, QColor + +from openpype import style +from openpype.client import OpenPypeMongoConnection from openpype.lib import JSONSettingRegistry from openpype.tools.utils import PlaceholderLineEdit, get_openpype_qt_app from openpype.tools.utils.constants import PROJECT_NAME_ROLE @@ -21,6 +26,8 @@ from openpype.tools.utils.models import ProjectModel, ProjectSortFilterProxy class AssetReporterRegistry(JSONSettingRegistry): """Class handling OpenPype general settings registry. + This is used to store last selected project. + Attributes: vendor (str): Name used for path construction. product (str): Additional name used for path construction. @@ -36,6 +43,10 @@ class AssetReporterRegistry(JSONSettingRegistry): class OverlayWidget(QtWidgets.QFrame): + """Overlay widget for choosing project. + + This code is taken from the Tray Publisher tool. + """ project_selected = QtCore.Signal(str) def __init__(self, publisher_window): @@ -116,7 +127,7 @@ class OverlayWidget(QtWidgets.QFrame): setting_registry = AssetReporterRegistry() try: - project_name = setting_registry.get_item("project_name") + project_name = str(setting_registry.get_item("project_name")) except ValueError: project_name = None @@ -166,9 +177,8 @@ class OverlayWidget(QtWidgets.QFrame): class AssetReporterWindow(QtWidgets.QDialog): - default_width = 1300 + default_width = 1000 default_height = 800 - footer_border = 8 _content = None def __init__(self, parent=None, controller=None, reset_on_show=None): @@ -217,7 +227,13 @@ class AssetReporterWindow(QtWidgets.QDialog): overlay_widget.project_selected.connect(self._on_project_select) self._overlay_widget = overlay_widget - def _on_project_select(self, project_name): + def _on_project_select(self, project_name: str): + """Generate table when project is selected. + + This will generate the table and fill it with data. + Source data are held in memory in `_result` attribute that + is used to transform them into clipboard or csv file. + """ self._project_name = project_name self.process() if not self._result: @@ -231,14 +247,15 @@ class AssetReporterWindow(QtWidgets.QDialog): content = [] for key, value in self._result.items(): item = QtWidgets.QTableWidgetItem(key) - item.setBackground(QColor(32, 32, 32)) + # this doesn't work as it is probably overriden by stylesheet? + # item.setBackground(QColor(32, 32, 32)) self.table.setItem(row, 0, item) for source in value: self.table.setItem(row, 1, QtWidgets.QTableWidgetItem(source["name"])) self.table.setItem(row, 2, QtWidgets.QTableWidgetItem(str(source["version"]))) - print(f' - {source["version"]}') row += 1 + # generate clipboard content content.append(f"{key}") content.extend( f"\t{source['name']} (v{source['version']})" for source in value @@ -295,9 +312,20 @@ class AssetReporterWindow(QtWidgets.QDialog): } def process(self): + """Generate asset usage report data. + + This is the main method of the tool. It is using MongoDB + aggregation pipeline to find all published versions that + are used as input for other published versions. Then it + generates a map of assets and their usage. + + """ start = time.perf_counter() project = self._project_name + # get all versions of published workfiles that has non-empty + # inputLinks and connect it with their respective documents + # using ID. pipeline = [ { "$match": { @@ -323,6 +351,9 @@ class AssetReporterWindow(QtWidgets.QDialog): result = db[project].aggregate(pipeline) asset_map = [] + # this is creating the map - for every workfile and its linked + # documents, create a dictionary with "source" and "refs" keys + # and resolve the subset name and version from the document for doc in result: source = { "source": self._get_subset(doc["parent"], db[project]), @@ -338,11 +369,9 @@ class AssetReporterWindow(QtWidgets.QDialog): source["refs"] = refs asset_map.append(source) - # for ref in asset_map: - # print(ref) - grouped = {} + # this will group the assets by subset name and version for asset in asset_map: for ref in asset["refs"]: key = f'{ref["subset"]["name"]} (v{ref["version"]})' @@ -356,7 +385,8 @@ class AssetReporterWindow(QtWidgets.QDialog): print(f"Finished in {end - start:0.4f} seconds", 2) - def _write_csv(self, file_name): + def _write_csv(self, file_name: str) -> None: + """Write CSV file with results.""" with open(file_name, "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=";") writer.writerow(["Subset", "Used in", "Version"]) @@ -366,7 +396,6 @@ class AssetReporterWindow(QtWidgets.QDialog): writer.writerow(["", source["name"], source["version"]]) - def main(): app_instance = get_openpype_qt_app() window = AssetReporterWindow() diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 2258dd42e3..5b189eae88 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -359,7 +359,7 @@ { "type": "dict", "key": "asset_reporter", - "label": "Asset Usage Report", + "label": "Asset Usage Reporter", "collapsible": true, "checkbox_key": "enabled", "children": [ From 455482041080894b1cc78731292f63a7139effdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 22 Nov 2023 16:42:11 +0100 Subject: [PATCH 4/8] :bug: invalid header order --- openpype/modules/asset_reporter/window.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index 9dbcc3ea74..bdcf0a9adf 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -205,7 +205,7 @@ class AssetReporterWindow(QtWidgets.QDialog): self.table.setColumnCount(3) self.table.setColumnWidth(0, 400) self.table.setColumnWidth(1, 300) - self.table.setHorizontalHeaderLabels(["Subset", "Used in", "Version"]) + self.table.setHorizontalHeaderLabels(["Used in", "Product", "Version"]) # self.text_area = QtWidgets.QTextEdit(self) self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) @@ -389,7 +389,7 @@ class AssetReporterWindow(QtWidgets.QDialog): """Write CSV file with results.""" with open(file_name, "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=";") - writer.writerow(["Subset", "Used in", "Version"]) + writer.writerow(["Used in", "Product", "Version"]) for key, value in self._result.items(): writer.writerow([key, "", ""]) for source in value: From 5ec3f92d7c79165ef324a5028a1125ffd39465cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 22 Nov 2023 17:02:30 +0100 Subject: [PATCH 5/8] :bug: remove unused plugin and hound fixes --- openpype/modules/asset_reporter/window.py | 15 +- .../actions/generate_asset_usage_report.py | 157 ------------------ 2 files changed, 9 insertions(+), 163 deletions(-) delete mode 100644 openpype/plugins/actions/generate_asset_usage_report.py diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index bdcf0a9adf..e34037965c 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -251,14 +251,17 @@ class AssetReporterWindow(QtWidgets.QDialog): # item.setBackground(QColor(32, 32, 32)) self.table.setItem(row, 0, item) for source in value: - self.table.setItem(row, 1, QtWidgets.QTableWidgetItem(source["name"])) - self.table.setItem(row, 2, QtWidgets.QTableWidgetItem(str(source["version"]))) + self.table.setItem( + row, 1, QtWidgets.QTableWidgetItem(source["name"])) + self.table.setItem( + row, 2, QtWidgets.QTableWidgetItem( + str(source["version"]))) row += 1 # generate clipboard content - content.append(f"{key}") + content.append(key) content.extend( - f"\t{source['name']} (v{source['version']})" for source in value + f"\t{source['name']} (v{source['version']})" for source in value # noqa: E501 ) self.set_content("\n".join(content)) @@ -307,8 +310,8 @@ class AssetReporterWindow(QtWidgets.QDialog): doc = next(result) # print(doc) return { - "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', - "family": doc["data"].get("family") or doc["data"].get("families")[0] + "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', # noqa: E501 + "family": doc["data"].get("family") or doc["data"].get("families")[0] # noqa: E501 } def process(self): diff --git a/openpype/plugins/actions/generate_asset_usage_report.py b/openpype/plugins/actions/generate_asset_usage_report.py deleted file mode 100644 index 60630c5a43..0000000000 --- a/openpype/plugins/actions/generate_asset_usage_report.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -TODO: we need to move it to subprocess to show UI -""" -import csv -import os -import tempfile -import time - -from qtpy import QtWidgets -from qtpy.QtCore import Qt -from qtpy.QtGui import QClipboard - -from pymongo.collection import Collection - -from openpype.client import OpenPypeMongoConnection -from openpype.pipeline import LauncherAction - - -class ReportWindow(QtWidgets.QWidget): - def __init__(self): - super().__init__() - - self.text_area = QtWidgets.QTextEdit(self) - self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) - self.save_button = QtWidgets.QPushButton('Save to CSV File', self) - - self.copy_button.clicked.connect(self.copy_to_clipboard) - self.save_button.clicked.connect(self.save_to_file) - - layout = QtWidgets.QVBoxLayout(self) - layout.addWidget(self.text_area) - layout.addWidget(self.copy_button) - layout.addWidget(self.save_button) - - def copy_to_clipboard(self): - clipboard = QtWidgets.QApplication.clipboard() - clipboard.setText(self.text_area.toPlainText(), QClipboard.Clipboard) - - def save_to_file(self): - file_name, _ = QtWidgets.QFileDialog.getSaveFileName(self, 'Save File') - if file_name: - with open(file_name, 'w') as file: - file.write(self.text_area.toPlainText()) - - def set_content(self, content): - self.text_area.setText(content) - - -class OpenTaskPath(LauncherAction): - name = "get_asset_usage_report" - label = "Asset Usage Report" - icon = "list" - order = 500 - - def is_compatible(self, session): - """Return whether the action is compatible with the session""" - return bool(session.get("AVALON_ASSET")) - - def _get_subset(self, version_id, project: Collection): - pipeline = [ - { - "$match": { - "_id": version_id - }, - }, { - "$lookup": { - "from": project.name, - "localField": "parent", - "foreignField": "_id", - "as": "parents" - } - } - ] - - result = project.aggregate(pipeline) - doc = next(result) - # print(doc) - return { - "name": f'{"/".join(doc["parents"][0]["data"]["parents"])}/{doc["parents"][0]["name"]}/{doc["name"]}', - "family": doc["data"].get("family") or doc["data"].get("families")[0] - } - - def process(self, session, **kwargs): - start = time.perf_counter() - project = session["AVALON_PROJECT"] - - pipeline = [ - { - "$match": { - "data.inputLinks": { - "$exists": True, - "$ne": [] - }, - "data.families": {"$in": ["workfile"]} - } - }, { - "$lookup": { - "from": "OP01_CG_demo", - "localField": "data.inputLinks.id", - "foreignField": "_id", - "as": "linked_docs" - } - } - ] - - client = OpenPypeMongoConnection.get_mongo_client() - db = client["avalon"] - - result = db[project].aggregate(pipeline) - - asset_map = [] - for doc in result: - source = { - "source": self._get_subset(doc["parent"], db[project]), - } - source["source"].update({"version": doc["name"]}) - refs = [ - { - "subset": self._get_subset(linked["parent"], db[project]), - "version": linked.get("name") - } - for linked in doc["linked_docs"] - ] - source["refs"] = refs - asset_map.append(source) - - # for ref in asset_map: - # print(ref) - - grouped = {} - - for asset in asset_map: - for ref in asset["refs"]: - key = f'{ref["subset"]["name"]} (v{ref["version"]})' - if key in grouped: - grouped[key].append(asset["source"]) - else: - grouped[key] = [asset["source"]] - - temp = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".csv") - try: - with open(temp.name, "w", newline="") as csvfile: - writer = csv.writer(csvfile, delimiter=";") - writer.writerow(["Subset", "Used in", "Version"]) - for key, value in grouped.items(): - writer.writerow([key, "", ""]) - for source in value: - writer.writerow(["", source["name"], source["version"]]) - finally: - temp.close() - - end = time.perf_counter() - app = QtWidgets.QApplication.instance() - window = ReportWindow() - # window.set_content(open(temp.name).read()) - window.show() - print(f"Finished in {end - start:0.4f} seconds", 2) From 8050d23529d8eded73f21b16c87df7480a359d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Wed, 22 Nov 2023 18:10:56 +0100 Subject: [PATCH 6/8] Revert ":bug: invalid header order" This reverts commit 455482041080894b1cc78731292f63a7139effdb. --- openpype/modules/asset_reporter/window.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index e34037965c..3dfc585528 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -205,7 +205,7 @@ class AssetReporterWindow(QtWidgets.QDialog): self.table.setColumnCount(3) self.table.setColumnWidth(0, 400) self.table.setColumnWidth(1, 300) - self.table.setHorizontalHeaderLabels(["Used in", "Product", "Version"]) + self.table.setHorizontalHeaderLabels(["Subset", "Used in", "Version"]) # self.text_area = QtWidgets.QTextEdit(self) self.copy_button = QtWidgets.QPushButton('Copy to Clipboard', self) @@ -392,7 +392,7 @@ class AssetReporterWindow(QtWidgets.QDialog): """Write CSV file with results.""" with open(file_name, "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=";") - writer.writerow(["Used in", "Product", "Version"]) + writer.writerow(["Subset", "Used in", "Version"]) for key, value in self._result.items(): writer.writerow([key, "", ""]) for source in value: From 00c1288c9d24d7a14e0b6b01da0867465ceae852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 23 Nov 2023 11:25:45 +0100 Subject: [PATCH 7/8] :bug: handle hero versions --- openpype/modules/asset_reporter/window.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index 3dfc585528..9b19d6295c 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -362,13 +362,20 @@ class AssetReporterWindow(QtWidgets.QDialog): "source": self._get_subset(doc["parent"], db[project]), } source["source"].update({"version": doc["name"]}) - refs = [ - { - "subset": self._get_subset(linked["parent"], db[project]), - "version": linked.get("name") - } - for linked in doc["linked_docs"] - ] + refs = [] + version = '' + for linked in doc["linked_docs"]: + try: + version = f'v{linked["name"]}' + except KeyError: + if linked["type"] == "hero_version": + version = "hero" + finally: + refs.append({ + "subset": self._get_subset(linked["parent"], db[project]), + "version": version + }) + source["refs"] = refs asset_map.append(source) @@ -377,7 +384,7 @@ class AssetReporterWindow(QtWidgets.QDialog): # this will group the assets by subset name and version for asset in asset_map: for ref in asset["refs"]: - key = f'{ref["subset"]["name"]} (v{ref["version"]})' + key = f'{ref["subset"]["name"]} ({ref["version"]})' if key in grouped: grouped[key].append(asset["source"]) else: From 5e5ed5b5177bada4484da014830c765f8be6587b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= Date: Thu, 23 Nov 2023 11:26:46 +0100 Subject: [PATCH 8/8] :dog: hound fix --- openpype/modules/asset_reporter/window.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/asset_reporter/window.py b/openpype/modules/asset_reporter/window.py index 9b19d6295c..ed3bc298e1 100644 --- a/openpype/modules/asset_reporter/window.py +++ b/openpype/modules/asset_reporter/window.py @@ -372,7 +372,8 @@ class AssetReporterWindow(QtWidgets.QDialog): version = "hero" finally: refs.append({ - "subset": self._get_subset(linked["parent"], db[project]), + "subset": self._get_subset( + linked["parent"], db[project]), "version": version })