From e2076c0f2fa264f4dc7ae6959c3292b60616ca91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 3 Feb 2022 19:03:16 +0100 Subject: [PATCH 01/82] Module: Kitsu module --- .../modules/default_modules/kitsu/__init__.py | 15 ++ .../default_modules/kitsu/kitsu_module.py | 147 ++++++++++++++++++ .../kitsu/plugins/publish/example_plugin.py | 9 ++ .../schemas/project_schemas/main.json | 30 ++++ .../schemas/project_schemas/the_template.json | 30 ++++ .../modules/default_modules/kitsu/widgets.py | 31 ++++ .../defaults/project_settings/kitsu.json | 3 + .../defaults/system_settings/modules.json | 4 + .../schemas/projects_schema/schema_main.json | 4 + .../projects_schema/schema_project_kitsu.json | 17 ++ .../module_settings/schema_kitsu.json | 23 +++ .../schemas/system_schema/schema_modules.json | 4 + 12 files changed, 317 insertions(+) create mode 100644 openpype/modules/default_modules/kitsu/__init__.py create mode 100644 openpype/modules/default_modules/kitsu/kitsu_module.py create mode 100644 openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py create mode 100644 openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json create mode 100644 openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json create mode 100644 openpype/modules/default_modules/kitsu/widgets.py create mode 100644 openpype/settings/defaults/project_settings/kitsu.json create mode 100644 openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json create mode 100644 openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json diff --git a/openpype/modules/default_modules/kitsu/__init__.py b/openpype/modules/default_modules/kitsu/__init__.py new file mode 100644 index 0000000000..cd0c2ea8af --- /dev/null +++ b/openpype/modules/default_modules/kitsu/__init__.py @@ -0,0 +1,15 @@ +""" Addon class definition and Settings definition must be imported here. + +If addon class or settings definition won't be here their definition won't +be found by OpenPype discovery. +""" + +from .kitsu_module import ( + AddonSettingsDef, + KitsuModule +) + +__all__ = ( + "AddonSettingsDef", + "KitsuModule" +) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py new file mode 100644 index 0000000000..81d7e56a81 --- /dev/null +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -0,0 +1,147 @@ +"""Addon definition is located here. + +Import of python packages that may not be available should not be imported +in global space here until are required or used. +- Qt related imports +- imports of Python 3 packages + - we still support Python 2 hosts where addon definition should available +""" + +import os +import click + +from openpype.modules import ( + JsonFilesSettingsDef, + OpenPypeModule, + ModulesManager +) +# Import interface defined by this addon to be able find other addons using it +from openpype_interfaces import ( + IPluginPaths, + ITrayAction +) + + +# Settings definition of this addon using `JsonFilesSettingsDef` +# - JsonFilesSettingsDef is prepared settings definition using json files +# to define settings and store default values +class AddonSettingsDef(JsonFilesSettingsDef): + # This will add prefixes to every schema and template from `schemas` + # subfolder. + # - it is not required to fill the prefix but it is highly + # recommended as schemas and templates may have name clashes across + # multiple addons + # - it is also recommended that prefix has addon name in it + schema_prefix = "kitsu" + + def get_settings_root_path(self): + """Implemented abstract class of JsonFilesSettingsDef. + + Return directory path where json files defying addon settings are + located. + """ + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "settings" + ) + + +class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): + """This Addon has defined it's settings and interface. + + This example has system settings with an enabled option. And use + few other interfaces: + - `IPluginPaths` to define custom plugin paths + - `ITrayAction` to be shown in tray tool + """ + label = "Kitsu" + name = "kitsu" + + def initialize(self, settings): + """Initialization of addon.""" + module_settings = settings[self.name] + # Enabled by settings + self.enabled = module_settings.get("enabled", False) + + # Prepare variables that can be used or set afterwards + self._connected_modules = None + # UI which must not be created at this time + self._dialog = None + + def tray_init(self): + """Implementation of abstract method for `ITrayAction`. + + We're definitely in tray tool so we can pre create dialog. + """ + + self._create_dialog() + + def _create_dialog(self): + # Don't recreate dialog if already exists + if self._dialog is not None: + return + + from .widgets import MyExampleDialog + + self._dialog = MyExampleDialog() + + def show_dialog(self): + """Show dialog with connected modules. + + This can be called from anywhere but can also crash in headless mode. + There is no way to prevent addon to do invalid operations if he's + not handling them. + """ + # Make sure dialog is created + self._create_dialog() + # Show dialog + self._dialog.open() + + def get_connected_modules(self): + """Custom implementation of addon.""" + names = set() + if self._connected_modules is not None: + for module in self._connected_modules: + names.add(module.name) + return names + + def on_action_trigger(self): + """Implementation of abstract method for `ITrayAction`.""" + self.show_dialog() + + def get_plugin_paths(self): + """Implementation of abstract method for `IPluginPaths`.""" + current_dir = os.path.dirname(os.path.abspath(__file__)) + + return { + "publish": [os.path.join(current_dir, "plugins", "publish")] + } + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group(KitsuModule.name, help="Kitsu dynamic cli commands.") +def cli_main(): + pass + + +@cli_main.command() +def nothing(): + """Does nothing but print a message.""" + print("You've triggered \"nothing\" command.") + + +@cli_main.command() +def show_dialog(): + """Show ExampleAddon dialog. + + We don't have access to addon directly through cli so we have to create + it again. + """ + from openpype.tools.utils.lib import qt_app_context + + manager = ModulesManager() + example_addon = manager.modules_by_name[KitsuModule.name] + with qt_app_context(): + example_addon.show_dialog() diff --git a/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py b/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py new file mode 100644 index 0000000000..61602f4e78 --- /dev/null +++ b/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py @@ -0,0 +1,9 @@ +import pyblish.api + + +class CollectExampleAddon(pyblish.api.ContextPlugin): + order = pyblish.api.CollectorOrder + 0.4 + label = "Collect Kitsu" + + def process(self, context): + self.log.info("I'm in Kitsu's plugin!") diff --git a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json new file mode 100644 index 0000000000..82e58ce9ab --- /dev/null +++ b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json @@ -0,0 +1,30 @@ +{ + "type": "dict", + "key": "kitsu", + "label": " Kitsu", + "collapsible": true, + "children": [ + { + "type": "number", + "key": "number", + "label": "This is your lucky number:", + "minimum": 7, + "maximum": 7, + "decimals": 0 + }, + { + "type": "template", + "name": "kitsu/the_template", + "template_data": [ + { + "name": "color_1", + "label": "Color 1" + }, + { + "name": "color_2", + "label": "Color 2" + } + ] + } + ] +} diff --git a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json new file mode 100644 index 0000000000..af8fd9dae4 --- /dev/null +++ b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json @@ -0,0 +1,30 @@ +[ + { + "type": "list-strict", + "key": "{name}", + "label": "{label}", + "object_types": [ + { + "label": "Red", + "type": "number", + "minimum": 0, + "maximum": 1, + "decimal": 3 + }, + { + "label": "Green", + "type": "number", + "minimum": 0, + "maximum": 1, + "decimal": 3 + }, + { + "label": "Blue", + "type": "number", + "minimum": 0, + "maximum": 1, + "decimal": 3 + } + ] + } +] diff --git a/openpype/modules/default_modules/kitsu/widgets.py b/openpype/modules/default_modules/kitsu/widgets.py new file mode 100644 index 0000000000..de232113fe --- /dev/null +++ b/openpype/modules/default_modules/kitsu/widgets.py @@ -0,0 +1,31 @@ +from Qt import QtWidgets + +from openpype.style import load_stylesheet + + +class MyExampleDialog(QtWidgets.QDialog): + def __init__(self, parent=None): + super(MyExampleDialog, self).__init__(parent) + + self.setWindowTitle("Connected modules") + + msg = "This is example dialog of Kitsu." + label_widget = QtWidgets.QLabel(msg, self) + + ok_btn = QtWidgets.QPushButton("OK", self) + btns_layout = QtWidgets.QHBoxLayout() + btns_layout.addStretch(1) + btns_layout.addWidget(ok_btn) + + layout = QtWidgets.QVBoxLayout(self) + layout.addWidget(label_widget) + layout.addLayout(btns_layout) + + ok_btn.clicked.connect(self._on_ok_clicked) + + self._label_widget = label_widget + + self.setStyleSheet(load_stylesheet()) + + def _on_ok_clicked(self): + self.done(1) diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json new file mode 100644 index 0000000000..b4d2ccc611 --- /dev/null +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -0,0 +1,3 @@ +{ + "number": 0 +} \ No newline at end of file diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index d74269922f..9cfaddecbe 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -137,6 +137,10 @@ } } }, + "kitsu": { + "enabled": false, + "kitsu_server": "" + }, "timers_manager": { "enabled": true, "auto_stop": true, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_main.json b/openpype/settings/entities/schemas/projects_schema/schema_main.json index dbddd18c80..6c07209de3 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_main.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_main.json @@ -62,6 +62,10 @@ "type": "schema", "name": "schema_project_ftrack" }, + { + "type": "schema", + "name": "schema_project_kitsu" + }, { "type": "schema", "name": "schema_project_deadline" diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json new file mode 100644 index 0000000000..93976cc03b --- /dev/null +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -0,0 +1,17 @@ +{ + "type": "dict", + "key": "kitsu", + "label": "Kitsu", + "collapsible": true, + "is_file": true, + "children": [ + { + "type": "number", + "key": "number", + "label": "This is your lucky number:", + "minimum": 7, + "maximum": 7, + "decimals": 0 + } + ] +} diff --git a/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json new file mode 100644 index 0000000000..8e496dc783 --- /dev/null +++ b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json @@ -0,0 +1,23 @@ +{ + "type": "dict", + "key": "kitsu", + "label": "Kitsu", + "collapsible": true, + "require_restart": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "text", + "key": "kitsu_server", + "label": "Server" + }, + { + "type": "splitter" + } + ] +} diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 52595914ed..d22b9016a7 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -44,6 +44,10 @@ "type": "schema", "name": "schema_ftrack" }, + { + "type": "schema", + "name": "schema_kitsu" + }, { "type": "dict", "key": "timers_manager", From 70440290cafcf02753f8c499951cc164b011c132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Feb 2022 15:38:02 +0100 Subject: [PATCH 02/82] Fist step to sync from Zou to local --- .../default_modules/kitsu/kitsu_module.py | 125 +++++++++++++++++- .../defaults/system_settings/modules.json | 4 +- .../module_settings/schema_kitsu.json | 12 +- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 81d7e56a81..55e4640fa2 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,20 +6,27 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ - +import collections import os import click +from avalon.api import AvalonMongoDB +import gazu +from openpype.api import get_project_basic_paths, create_project_folders +from openpype.lib import create_project +from openpype.lib.anatomy import Anatomy from openpype.modules import ( JsonFilesSettingsDef, OpenPypeModule, ModulesManager ) +from openpype.tools.project_manager.project_manager.model import AssetItem, ProjectItem, TaskItem # Import interface defined by this addon to be able find other addons using it from openpype_interfaces import ( IPluginPaths, ITrayAction ) +from pymongo import UpdateOne, DeleteOne # Settings definition of this addon using `JsonFilesSettingsDef` @@ -60,9 +67,27 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): def initialize(self, settings): """Initialization of addon.""" module_settings = settings[self.name] + # Enabled by settings self.enabled = module_settings.get("enabled", False) + # Add API URL schema + kitsu_url = module_settings["server"].strip() + if kitsu_url: + # Ensure web url + if not kitsu_url.startswith("http"): + kitsu_url = "https://" + kitsu_url + + # Check for "/api" url validity + if not kitsu_url.endswith("api"): + kitsu_url = f"{kitsu_url}{'' if kitsu_url.endswith('/') else '/'}api" + + self.server_url = kitsu_url + + # Set credentials + self.script_login = module_settings["script_login"] + self.script_pwd = module_settings["script_pwd"] + # Prepare variables that can be used or set afterwards self._connected_modules = None # UI which must not be created at this time @@ -76,6 +101,14 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): self._create_dialog() + def get_global_environments(self): + """Kitsu's global environments.""" + return { + "KITSU_SERVER": self.server_url, + "KITSU_LOGIN": self.script_login, + "KITSU_PWD": self.script_pwd + } + def _create_dialog(self): # Don't recreate dialog if already exists if self._dialog is not None: @@ -127,10 +160,94 @@ def cli_main(): @cli_main.command() -def nothing(): - """Does nothing but print a message.""" - print("You've triggered \"nothing\" command.") +def sync_local(): + """Synchronize local database from Zou sever database.""" + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + all_projects = gazu.project.all_projects() + for project in all_projects: + # Create project locally + # Try to find project document + project_name = project["name"] + project_code = project_name + dbcon.Session["AVALON_PROJECT"] = project_name + project_doc = dbcon.find_one({ + "type": "project" + }) + + # Get all assets from zou + all_assets = gazu.asset.all_assets_for_project(project) + + # Query all assets of the local project + project_col = dbcon.database[project_code] + asset_docs_zou_ids = { + asset_doc["data"]["zou_id"] + for asset_doc in project_col.find({"type": "asset"}) + } + + # Create project if is not available + # - creation is required to be able set project anatomy and attributes + if not project_doc: + print(f"Creating project '{project_name}'") + project_doc = create_project(project_name, project_code, dbcon=dbcon) + + # Create project item + insert_list = [] + + bulk_writes = [] + for zou_asset in all_assets: + doc_data = {"zou_id": zou_asset["id"]} + + # Create Asset + new_doc = { + "name": zou_asset["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": doc_data, + "parent": project_doc["_id"] + } + + if zou_asset["id"] not in asset_docs_zou_ids: # Item is new + insert_list.append(new_doc) + + # TODO tasks + + # elif item.data(REMOVED_ROLE): # TODO removal + # if item.data(HIERARCHY_CHANGE_ABLE_ROLE): + # bulk_writes.append(DeleteOne( + # {"_id": item.asset_id} + # )) + # else: + # bulk_writes.append(UpdateOne( + # {"_id": item.asset_id}, + # {"$set": {"type": "archived_asset"}} + # )) + + # else: TODO update data + # update_data = new_item.update_data() + # if update_data: + # bulk_writes.append(UpdateOne( + # {"_id": new_item.asset_id}, + # update_data + # )) + + # Insert new docs if created + if insert_list: + project_col.insert_many(insert_list) + + # Write into DB TODO + # if bulk_writes: + # project_col.bulk_write(bulk_writes) + + dbcon.uninstall() @cli_main.command() def show_dialog(): diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 9cfaddecbe..ddb2edc360 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -139,7 +139,9 @@ }, "kitsu": { "enabled": false, - "kitsu_server": "" + "server": "", + "script_login": "", + "script_pwd": "" }, "timers_manager": { "enabled": true, diff --git a/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json index 8e496dc783..ae2b52df0d 100644 --- a/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json +++ b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json @@ -13,9 +13,19 @@ }, { "type": "text", - "key": "kitsu_server", + "key": "server", "label": "Server" }, + { + "type": "text", + "key": "script_login", + "label": "Script Login" + }, + { + "type": "text", + "key": "script_pwd", + "label": "Script Password" + }, { "type": "splitter" } From 719afdcc8ae5f8710537d55da24040bd7ccfcd95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Feb 2022 15:41:02 +0100 Subject: [PATCH 03/82] black --- .../default_modules/kitsu/kitsu_module.py | 37 ++++++------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 55e4640fa2..6eb37dfaed 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,27 +6,17 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ -import collections import os import click from avalon.api import AvalonMongoDB import gazu -from openpype.api import get_project_basic_paths, create_project_folders from openpype.lib import create_project from openpype.lib.anatomy import Anatomy -from openpype.modules import ( - JsonFilesSettingsDef, - OpenPypeModule, - ModulesManager -) -from openpype.tools.project_manager.project_manager.model import AssetItem, ProjectItem, TaskItem +from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager + # Import interface defined by this addon to be able find other addons using it -from openpype_interfaces import ( - IPluginPaths, - ITrayAction -) -from pymongo import UpdateOne, DeleteOne +from openpype_interfaces import IPluginPaths, ITrayAction # Settings definition of this addon using `JsonFilesSettingsDef` @@ -47,10 +37,7 @@ class AddonSettingsDef(JsonFilesSettingsDef): Return directory path where json files defying addon settings are located. """ - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "settings" - ) + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "settings") class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): @@ -61,6 +48,7 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): - `IPluginPaths` to define custom plugin paths - `ITrayAction` to be shown in tray tool """ + label = "Kitsu" name = "kitsu" @@ -106,7 +94,7 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): return { "KITSU_SERVER": self.server_url, "KITSU_LOGIN": self.script_login, - "KITSU_PWD": self.script_pwd + "KITSU_PWD": self.script_pwd, } def _create_dialog(self): @@ -146,9 +134,7 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): """Implementation of abstract method for `IPluginPaths`.""" current_dir = os.path.dirname(os.path.abspath(__file__)) - return { - "publish": [os.path.join(current_dir, "plugins", "publish")] - } + return {"publish": [os.path.join(current_dir, "plugins", "publish")]} def cli(self, click_group): click_group.add_command(cli_main) @@ -179,10 +165,8 @@ def sync_local(): project_name = project["name"] project_code = project_name dbcon.Session["AVALON_PROJECT"] = project_name - project_doc = dbcon.find_one({ - "type": "project" - }) - + project_doc = dbcon.find_one({"type": "project"}) + # Get all assets from zou all_assets = gazu.asset.all_assets_for_project(project) @@ -212,7 +196,7 @@ def sync_local(): "type": "asset", "schema": "openpype:asset-3.0", "data": doc_data, - "parent": project_doc["_id"] + "parent": project_doc["_id"], } if zou_asset["id"] not in asset_docs_zou_ids: # Item is new @@ -249,6 +233,7 @@ def sync_local(): dbcon.uninstall() + @cli_main.command() def show_dialog(): """Show ExampleAddon dialog. From 7c63d3a374637ee8630d293b8ac8dcc8a34ffb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Feb 2022 16:28:06 +0100 Subject: [PATCH 04/82] Add tasks to asset --- .../default_modules/kitsu/kitsu_module.py | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 6eb37dfaed..9730437ec2 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -187,22 +187,43 @@ def sync_local(): insert_list = [] bulk_writes = [] - for zou_asset in all_assets: - doc_data = {"zou_id": zou_asset["id"]} + for asset in all_assets: + asset_data = {"zou_id": asset["id"]} + + # Set tasks + asset_tasks = gazu.task.all_tasks_for_asset(asset) + asset_data["tasks"] = { + t["task_type_name"]: {"type": t["task_type_name"]} for t in asset_tasks + } # Create Asset - new_doc = { - "name": zou_asset["name"], + asset_doc = { + "name": asset["name"], "type": "asset", "schema": "openpype:asset-3.0", - "data": doc_data, + "data": asset_data, "parent": project_doc["_id"], } - if zou_asset["id"] not in asset_docs_zou_ids: # Item is new - insert_list.append(new_doc) + if asset["id"] not in asset_docs_zou_ids: # Item is new + insert_list.append(asset_doc) + else: + asset_doc = project_col.find_one({"data": {"zou_id": asset["id"]}}) - # TODO tasks + # TODO update + # for task in asset_tasks: + # # print(task) + # task_data = {"zou_id": task["id"]} + + # # Create Task + # task_doc = { + # "name": task["name"], + # "type": "task", + # "schema": "openpype:asset-3.0", + # "data": task_data, + # "parent": asset_doc["_id"], + # } + # insert_list.append(task_doc) # elif item.data(REMOVED_ROLE): # TODO removal # if item.data(HIERARCHY_CHANGE_ABLE_ROLE): From 9a1dd4fc0630f94cff1ef554fd31ede714b8ad49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Feb 2022 19:26:26 +0100 Subject: [PATCH 05/82] All bulk write. Updating assets --- .../default_modules/kitsu/kitsu_module.py | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 9730437ec2..1a71a05a7e 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -12,10 +12,8 @@ import click from avalon.api import AvalonMongoDB import gazu from openpype.lib import create_project -from openpype.lib.anatomy import Anatomy from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager - -# Import interface defined by this addon to be able find other addons using it +from pymongo import DeleteOne, InsertOne, UpdateOne from openpype_interfaces import IPluginPaths, ITrayAction @@ -183,9 +181,6 @@ def sync_local(): print(f"Creating project '{project_name}'") project_doc = create_project(project_name, project_code, dbcon=dbcon) - # Create project item - insert_list = [] - bulk_writes = [] for asset in all_assets: asset_data = {"zou_id": asset["id"]} @@ -196,34 +191,33 @@ def sync_local(): t["task_type_name"]: {"type": t["task_type_name"]} for t in asset_tasks } - # Create Asset - asset_doc = { - "name": asset["name"], - "type": "asset", - "schema": "openpype:asset-3.0", - "data": asset_data, - "parent": project_doc["_id"], - } + # Update or create asset + if asset["id"] in asset_docs_zou_ids: # Update asset + asset_doc = project_col.find_one({"data.zou_id": asset["id"]}) - if asset["id"] not in asset_docs_zou_ids: # Item is new - insert_list.append(asset_doc) - else: - asset_doc = project_col.find_one({"data": {"zou_id": asset["id"]}}) + # Override all 'data' TODO filter data to update? + diff_data = { + k: asset_data[k] + for k in asset_data.keys() + if asset_doc["data"].get(k) != asset_data[k] + } + if diff_data: + bulk_writes.append( + UpdateOne( + {"_id": asset_doc["_id"]}, {"$set": {"data": asset_data}} + ) + ) + else: # Create + asset_doc = { + "name": asset["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": asset_data, + "parent": project_doc["_id"], + } - # TODO update - # for task in asset_tasks: - # # print(task) - # task_data = {"zou_id": task["id"]} - - # # Create Task - # task_doc = { - # "name": task["name"], - # "type": "task", - # "schema": "openpype:asset-3.0", - # "data": task_data, - # "parent": asset_doc["_id"], - # } - # insert_list.append(task_doc) + # Insert new doc + bulk_writes.append(InsertOne(asset_doc)) # elif item.data(REMOVED_ROLE): # TODO removal # if item.data(HIERARCHY_CHANGE_ABLE_ROLE): @@ -244,13 +238,9 @@ def sync_local(): # update_data # )) - # Insert new docs if created - if insert_list: - project_col.insert_many(insert_list) - - # Write into DB TODO - # if bulk_writes: - # project_col.bulk_write(bulk_writes) + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) dbcon.uninstall() From 206bf9f3c18873a6f81734841779eeab4f8829d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 9 Feb 2022 09:45:11 +0100 Subject: [PATCH 06/82] Delete assets --- .../default_modules/kitsu/kitsu_module.py | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 1a71a05a7e..1ce1bef6a2 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -170,10 +170,13 @@ def sync_local(): # Query all assets of the local project project_col = dbcon.database[project_code] - asset_docs_zou_ids = { - asset_doc["data"]["zou_id"] + asset_doc_ids = { + asset_doc["_id"]: asset_doc for asset_doc in project_col.find({"type": "asset"}) } + asset_docs_zou_ids = { + asset_doc["data"]["zou_id"] for asset_doc in asset_doc_ids.values() + } # Create project if is not available # - creation is required to be able set project anatomy and attributes @@ -182,6 +185,7 @@ def sync_local(): project_doc = create_project(project_name, project_code, dbcon=dbcon) bulk_writes = [] + sync_assets = set() for asset in all_assets: asset_data = {"zou_id": asset["id"]} @@ -195,13 +199,13 @@ def sync_local(): if asset["id"] in asset_docs_zou_ids: # Update asset asset_doc = project_col.find_one({"data.zou_id": asset["id"]}) - # Override all 'data' TODO filter data to update? - diff_data = { + # Override all 'data' + updated_data = { k: asset_data[k] for k in asset_data.keys() if asset_doc["data"].get(k) != asset_data[k] } - if diff_data: + if updated_data: bulk_writes.append( UpdateOne( {"_id": asset_doc["_id"]}, {"$set": {"data": asset_data}} @@ -219,24 +223,16 @@ def sync_local(): # Insert new doc bulk_writes.append(InsertOne(asset_doc)) - # elif item.data(REMOVED_ROLE): # TODO removal - # if item.data(HIERARCHY_CHANGE_ABLE_ROLE): - # bulk_writes.append(DeleteOne( - # {"_id": item.asset_id} - # )) - # else: - # bulk_writes.append(UpdateOne( - # {"_id": item.asset_id}, - # {"$set": {"type": "archived_asset"}} - # )) + # Keep synchronized asset for diff + sync_assets.add(asset_doc["_id"]) - # else: TODO update data - # update_data = new_item.update_data() - # if update_data: - # bulk_writes.append(UpdateOne( - # {"_id": new_item.asset_id}, - # update_data - # )) + # Delete from diff of assets in OP and synchronized assets to detect deleted assets + diff_assets = set(asset_doc_ids.keys()) - sync_assets + if diff_assets: + # Delete doc + bulk_writes.extend( + [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] + ) # Write into DB if bulk_writes: From e882de52f04d2ca79e30fb23801257cefd085e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 10 Feb 2022 17:35:03 +0100 Subject: [PATCH 07/82] Assets hierarchy --- .../default_modules/kitsu/kitsu_module.py | 165 ++++++++++++------ 1 file changed, 114 insertions(+), 51 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 1ce1bef6a2..92d724be67 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -7,13 +7,15 @@ in global space here until are required or used. - we still support Python 2 hosts where addon definition should available """ import os +from typing import Dict, List, Set import click from avalon.api import AvalonMongoDB import gazu from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager -from pymongo import DeleteOne, InsertOne, UpdateOne +from pymongo import DeleteOne, UpdateOne +from pymongo.collection import Collection from openpype_interfaces import IPluginPaths, ITrayAction @@ -144,8 +146,8 @@ def cli_main(): @cli_main.command() -def sync_local(): - """Synchronize local database from Zou sever database.""" +def sync_openpype(): + """Synchronize openpype database from Zou sever database.""" # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) @@ -167,69 +169,61 @@ def sync_local(): # Get all assets from zou all_assets = gazu.asset.all_assets_for_project(project) - - # Query all assets of the local project - project_col = dbcon.database[project_code] - asset_doc_ids = { - asset_doc["_id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) - } - asset_docs_zou_ids = { - asset_doc["data"]["zou_id"] for asset_doc in asset_doc_ids.values() - } + all_episodes = gazu.shot.all_episodes_for_project(project) + all_seqs = gazu.shot.all_sequences_for_project(project) + all_shots = gazu.shot.all_shots_for_project(project) # Create project if is not available # - creation is required to be able set project anatomy and attributes + to_insert = [] if not project_doc: print(f"Creating project '{project_name}'") project_doc = create_project(project_name, project_code, dbcon=dbcon) - bulk_writes = [] - sync_assets = set() - for asset in all_assets: - asset_data = {"zou_id": asset["id"]} + # Query all assets of the local project + project_col = dbcon.database[project_code] + asset_doc_ids = { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + asset_doc_ids[project["id"]] = project_doc - # Set tasks - asset_tasks = gazu.task.all_tasks_for_asset(asset) - asset_data["tasks"] = { - t["task_type_name"]: {"type": t["task_type_name"]} for t in asset_tasks - } - - # Update or create asset - if asset["id"] in asset_docs_zou_ids: # Update asset - asset_doc = project_col.find_one({"data.zou_id": asset["id"]}) - - # Override all 'data' - updated_data = { - k: asset_data[k] - for k in asset_data.keys() - if asset_doc["data"].get(k) != asset_data[k] - } - if updated_data: - bulk_writes.append( - UpdateOne( - {"_id": asset_doc["_id"]}, {"$set": {"data": asset_data}} - ) - ) - else: # Create - asset_doc = { - "name": asset["name"], + # Create + to_insert.extend( + [ + { + "name": item["name"], "type": "asset", "schema": "openpype:asset-3.0", - "data": asset_data, - "parent": project_doc["_id"], + "data": {"zou_id": item["id"], "tasks": {}}, } + for item in all_episodes + all_assets + all_seqs + all_shots + if item["id"] not in asset_doc_ids.keys() + ] + ) + if to_insert: + # Insert in doc + project_col.insert_many(to_insert) - # Insert new doc - bulk_writes.append(InsertOne(asset_doc)) + # Update existing docs + asset_doc_ids.update( + { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + ) - # Keep synchronized asset for diff - sync_assets.add(asset_doc["_id"]) + # Update + all_entities = all_assets + all_episodes + all_seqs + all_shots + bulk_writes = update_op_assets(project_col, all_entities, asset_doc_ids) - # Delete from diff of assets in OP and synchronized assets to detect deleted assets - diff_assets = set(asset_doc_ids.keys()) - sync_assets + # Delete + diff_assets = set(asset_doc_ids.keys()) - { + e["id"] for e in all_entities + [project] + } if diff_assets: - # Delete doc bulk_writes.extend( [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] ) @@ -241,6 +235,75 @@ def sync_local(): dbcon.uninstall() +def update_op_assets( + project_col: Collection, items_list: List[dict], asset_doc_ids: Dict[str, dict] +) -> List[UpdateOne]: + """Update OpenPype assets. + Set 'data' and 'parent' fields. + + :param project_col: Project collection to query data from + :param items_list: List of zou items to update + :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] + :return: List of UpdateOne objects + """ + bulk_writes = [] + for item in items_list: + # Update asset + item_doc = project_col.find_one({"data.zou_id": item["id"]}) + item_data = item_doc["data"].copy() + + # Tasks + tasks_list = None + if item["type"] == "Asset": + tasks_list = gazu.task.all_tasks_for_asset(item) + elif item["type"] == "Shot": + tasks_list = gazu.task.all_tasks_for_shot(item) + if tasks_list: + item_data["tasks"] = { + t["task_type_name"]: {"type": t["task_type_name"]} for t in tasks_list + } + + # Visual parent for hierarchy + direct_parent_id = item["parent_id"] or item["source_id"] + if direct_parent_id: + visual_parent_doc = asset_doc_ids[direct_parent_id] + item_data["visualParent"] = visual_parent_doc["_id"] + + # Add parents for hierarchy + parent_zou_id = item["parent_id"] + item_data["parents"] = [] + while parent_zou_id is not None: + parent_doc = asset_doc_ids[parent_zou_id] + item_data["parents"].insert(0, parent_doc["name"]) + + parent_zou_id = next( + i for i in items_list if i["id"] == parent_doc["data"]["zou_id"] + )["parent_id"] + + # TODO create missing tasks before + + # Update 'data' different in zou DB + updated_data = { + k: item_data[k] + for k in item_data.keys() + if item_doc["data"].get(k) != item_data[k] + } + if updated_data or not item_doc.get("parent"): + bulk_writes.append( + UpdateOne( + {"_id": item_doc["_id"]}, + { + "$set": { + "data": item_data, + "parent": asset_doc_ids[item["project_id"]]["_id"], + } + }, + ) + ) + + return bulk_writes + + @cli_main.command() def show_dialog(): """Show ExampleAddon dialog. From ee281f740d58821ab4d5e190e6b4863af5c2ac44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 10 Feb 2022 18:03:27 +0100 Subject: [PATCH 08/82] Project tasks --- .../default_modules/kitsu/kitsu_module.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 92d724be67..84709bc0a2 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -160,6 +160,8 @@ def sync_openpype(): dbcon.install() all_projects = gazu.project.all_projects() for project in all_projects: + bulk_writes = [] + # Create project locally # Try to find project document project_name = project["name"] @@ -180,6 +182,23 @@ def sync_openpype(): print(f"Creating project '{project_name}'") project_doc = create_project(project_name, project_code, dbcon=dbcon) + # Project tasks + bulk_writes.append( + UpdateOne( + {"_id": project_doc["_id"]}, + { + "$set": { + "config.tasks": { + t["name"]: { + "short_name": t.get("short_name", t["name"]) + } + for t in gazu.task.all_task_types_for_project(project) + } + } + }, + ) + ) + # Query all assets of the local project project_col = dbcon.database[project_code] asset_doc_ids = { @@ -217,7 +236,7 @@ def sync_openpype(): # Update all_entities = all_assets + all_episodes + all_seqs + all_shots - bulk_writes = update_op_assets(project_col, all_entities, asset_doc_ids) + bulk_writes.extend(update_op_assets(project_col, all_entities, asset_doc_ids)) # Delete diff_assets = set(asset_doc_ids.keys()) - { @@ -228,7 +247,7 @@ def sync_openpype(): [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] ) - # Write into DB + # Write into DB # TODO make it common for all projects if bulk_writes: project_col.bulk_write(bulk_writes) From d63c5fcae8ec1ee8c49c4a21dea86b3e9da157d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 11 Feb 2022 09:29:08 +0100 Subject: [PATCH 09/82] Optim: bulkwrite and queries mutualization --- .../default_modules/kitsu/kitsu_module.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 84709bc0a2..89312a344c 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -159,9 +159,8 @@ def sync_openpype(): dbcon = AvalonMongoDB() dbcon.install() all_projects = gazu.project.all_projects() + bulk_writes = [] for project in all_projects: - bulk_writes = [] - # Create project locally # Try to find project document project_name = project["name"] @@ -236,7 +235,7 @@ def sync_openpype(): # Update all_entities = all_assets + all_episodes + all_seqs + all_shots - bulk_writes.extend(update_op_assets(project_col, all_entities, asset_doc_ids)) + bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) # Delete diff_assets = set(asset_doc_ids.keys()) - { @@ -247,20 +246,19 @@ def sync_openpype(): [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] ) - # Write into DB # TODO make it common for all projects - if bulk_writes: - project_col.bulk_write(bulk_writes) + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) dbcon.uninstall() def update_op_assets( - project_col: Collection, items_list: List[dict], asset_doc_ids: Dict[str, dict] + items_list: List[dict], asset_doc_ids: Dict[str, dict] ) -> List[UpdateOne]: """Update OpenPype assets. Set 'data' and 'parent' fields. - :param project_col: Project collection to query data from :param items_list: List of zou items to update :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] :return: List of UpdateOne objects @@ -268,7 +266,7 @@ def update_op_assets( bulk_writes = [] for item in items_list: # Update asset - item_doc = project_col.find_one({"data.zou_id": item["id"]}) + item_doc = asset_doc_ids[item["id"]] item_data = item_doc["data"].copy() # Tasks @@ -299,8 +297,6 @@ def update_op_assets( i for i in items_list if i["id"] == parent_doc["data"]["zou_id"] )["parent_id"] - # TODO create missing tasks before - # Update 'data' different in zou DB updated_data = { k: item_data[k] From 4e68bcf55fd2552f53904dcd0a6c47b56946c927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 11 Feb 2022 09:31:14 +0100 Subject: [PATCH 10/82] cleaning --- openpype/modules/default_modules/kitsu/kitsu_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 89312a344c..e52d18b84b 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -15,7 +15,6 @@ import gazu from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager from pymongo import DeleteOne, UpdateOne -from pymongo.collection import Collection from openpype_interfaces import IPluginPaths, ITrayAction From 294b93f65a97d17f503a82962f121c3202002a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Sat, 12 Feb 2022 15:06:22 +0100 Subject: [PATCH 11/82] Create episode --- .../default_modules/kitsu/kitsu_module.py | 226 +++++++++++++++++- .../defaults/project_settings/kitsu.json | 6 +- .../projects_schema/schema_project_kitsu.json | 26 +- 3 files changed, 243 insertions(+), 15 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index e52d18b84b..c4f627d5ad 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,12 +6,14 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ -import os -from typing import Dict, List, Set import click +import os +import re +from typing import Dict, List from avalon.api import AvalonMongoDB import gazu +from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager from pymongo import DeleteOne, UpdateOne @@ -145,8 +147,8 @@ def cli_main(): @cli_main.command() -def sync_openpype(): - """Synchronize openpype database from Zou sever database.""" +def sync_zou(): + """Synchronize Zou server database (Kitsu backend) with openpype database.""" # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) @@ -157,8 +159,108 @@ def sync_openpype(): # Iterate projects dbcon = AvalonMongoDB() dbcon.install() - all_projects = gazu.project.all_projects() + + op_projects = [p for p in dbcon.projects()] bulk_writes = [] + for op_project in op_projects: + # Create project locally + # Try to find project document + project_name = op_project["name"] + project_code = op_project["data"]["code"] + dbcon.Session["AVALON_PROJECT"] = project_name + + # Get all entities from zou + zou_project = gazu.project.get_project_by_name(project_name) + + # Create project + if zou_project is None: + raise RuntimeError( + f"Project '{project_name}' doesn't exist in Zou database, please create it in Kitsu and add logged user to it before running synchronization." + ) + + # Update project settings and data + zou_project.update( + { + "code": op_project["data"]["code"], + "fps": op_project["data"]["fps"], + "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", + } + ) + gazu.project.update_project(zou_project) + gazu.project.update_project_data(zou_project, data=op_project["data"]) + + all_assets = gazu.asset.all_assets_for_project(zou_project) + all_episodes = gazu.shot.all_episodes_for_project(zou_project) + all_seqs = gazu.shot.all_sequences_for_project(zou_project) + all_shots = gazu.shot.all_shots_for_project(zou_project) + print(zou_project["name"]) + all_entities_ids = { + e["id"] for e in all_episodes + all_seqs + all_shots + all_assets + } + + project_module_settings = get_project_settings(project_name)["kitsu"] + + # Create new assets + # Query all assets of the local project + project_col = dbcon.database[project_name] + asset_docs = [asset_doc for asset_doc in project_col.find({"type": "asset"})] + + new_assets_docs = [ + doc + for doc in asset_docs + if doc["data"].get("zou_id") not in all_entities_ids + ] + naming_pattern = project_module_settings["entities_naming_pattern"] + regex_ep = re.compile( + r"({})|({})|({})".format( + naming_pattern["episode"].replace("#", "\d"), + naming_pattern["sequence"].replace("#", "\d"), + naming_pattern["shot"].replace("#", "\d"), + ), + re.IGNORECASE, + ) + for doc in new_assets_docs: + match = regex_ep.match(doc["name"]) + if not match: + # TODO asset + continue + + print(doc) + if match.group(1): # Episode + new_episode = gazu.shot.new_episode(zou_project, doc["name"]) + + # Update doc with zou id + bulk_writes.append( + UpdateOne( + {"_id": doc["_id"]}, + {"$set": {"data.zou_id": new_episode["id"]}}, + ) + ) + elif match.group(2): # Sequence + # TODO match zou episode + new_sequence = gazu.shot.new_sequence(zou_project, doc["name"]) + + # Update doc with zou id + bulk_writes.append( + UpdateOne( + {"_id": doc["_id"]}, + {"$set": {"data.zou_id": new_sequence["id"]}}, + ) + ) + elif match.group(3): # Shot + pass + + # Delete + # if gazu. + + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) + + dbcon.uninstall() + + return + for project in all_projects: # Create project locally # Try to find project document @@ -245,9 +347,117 @@ def sync_openpype(): [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] ) - # Write into DB - if bulk_writes: - project_col.bulk_write(bulk_writes) + +@cli_main.command() +def sync_openpype(): + """Synchronize openpype database from Zou sever database.""" + + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + all_projects = gazu.project.all_projects() + bulk_writes = [] + for project in all_projects: + # Create project locally + # Try to find project document + project_name = project["name"] + project_code = project_name + dbcon.Session["AVALON_PROJECT"] = project_name + project_doc = dbcon.find_one({"type": "project"}) + + # Get all assets from zou + all_assets = gazu.asset.all_assets_for_project(project) + all_episodes = gazu.shot.all_episodes_for_project(project) + all_seqs = gazu.shot.all_sequences_for_project(project) + all_shots = gazu.shot.all_shots_for_project(project) + + # Create project if is not available + # - creation is required to be able set project anatomy and attributes + to_insert = [] + if not project_doc: + print(f"Creating project '{project_name}'") + project_doc = create_project(project_name, project_code, dbcon=dbcon) + + # Project data and tasks + bulk_writes.append( + UpdateOne( + {"_id": project_doc["_id"]}, + { + "$set": { + "config.tasks": { + t["name"]: {"short_name": t.get("short_name", t["name"])} + for t in gazu.task.all_task_types_for_project(project) + }, + "data": project["data"].update( + { + "code": project["code"], + "fps": project_code["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ), + } + }, + ) + ) + + # Query all assets of the local project + project_col = dbcon.database[project_code] + asset_doc_ids = { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + asset_doc_ids[project["id"]] = project_doc + + # Create + to_insert.extend( + [ + { + "name": item["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": {"zou_id": item["id"], "tasks": {}}, + } + for item in all_episodes + all_assets + all_seqs + all_shots + if item["id"] not in asset_doc_ids.keys() + ] + ) + if to_insert: + # Insert in doc + project_col.insert_many(to_insert) + + # Update existing docs + asset_doc_ids.update( + { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + ) + + # Update + all_entities = all_assets + all_episodes + all_seqs + all_shots + bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) + + # Delete + diff_assets = set(asset_doc_ids.keys()) - { + e["id"] for e in all_entities + [project] + } + if diff_assets: + bulk_writes.extend( + [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] + ) + + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) dbcon.uninstall() diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index b4d2ccc611..435814a9d1 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -1,3 +1,7 @@ { - "number": 0 + "entities_naming_pattern": { + "episode": "E##", + "sequence": "SQ##", + "shot": "SH##" + } } \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index 93976cc03b..a504959001 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -6,12 +6,26 @@ "is_file": true, "children": [ { - "type": "number", - "key": "number", - "label": "This is your lucky number:", - "minimum": 7, - "maximum": 7, - "decimals": 0 + "type": "dict", + "key": "entities_naming_pattern", + "label": "Entities naming pattern", + "children": [ + { + "type": "text", + "key": "episode", + "label": "Episode:" + }, + { + "type": "text", + "key": "sequence", + "label": "Sequence:" + }, + { + "type": "text", + "key": "shot", + "label": "Shot:" + } + ] } ] } From bad16651e1b2ed2d307496941ac95adb1224040a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Mon, 14 Feb 2022 17:58:06 +0100 Subject: [PATCH 12/82] Create asset, ep, seq and shot in zou --- .../default_modules/kitsu/kitsu_module.py | 123 +++++++++++++----- 1 file changed, 93 insertions(+), 30 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index c4f627d5ad..453d1c5315 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -189,6 +189,7 @@ def sync_zou(): gazu.project.update_project(zou_project) gazu.project.update_project_data(zou_project, data=op_project["data"]) + all_asset_types = gazu.asset.all_asset_types() all_assets = gazu.asset.all_assets_for_project(zou_project) all_episodes = gazu.shot.all_episodes_for_project(zou_project) all_seqs = gazu.shot.all_sequences_for_project(zou_project) @@ -199,15 +200,17 @@ def sync_zou(): } project_module_settings = get_project_settings(project_name)["kitsu"] + project_col = dbcon.database[project_name] + asset_docs = { + asset_doc["_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + } # Create new assets # Query all assets of the local project - project_col = dbcon.database[project_name] - asset_docs = [asset_doc for asset_doc in project_col.find({"type": "asset"})] - new_assets_docs = [ doc - for doc in asset_docs + for doc in asset_docs.values() if doc["data"].get("zou_id") not in all_entities_ids ] naming_pattern = project_module_settings["entities_naming_pattern"] @@ -220,37 +223,95 @@ def sync_zou(): re.IGNORECASE, ) for doc in new_assets_docs: + visual_parent_id = doc["data"]["visualParent"] + match = regex_ep.match(doc["name"]) - if not match: - # TODO asset - continue - - print(doc) - if match.group(1): # Episode - new_episode = gazu.shot.new_episode(zou_project, doc["name"]) - - # Update doc with zou id - bulk_writes.append( - UpdateOne( - {"_id": doc["_id"]}, - {"$set": {"data.zou_id": new_episode["id"]}}, - ) + if not match: # Asset + new_entity = gazu.asset.new_asset( + zou_project, all_asset_types[0], doc["name"] ) + + elif match.group(1): # Episode + new_entity = gazu.shot.new_episode(zou_project, doc["name"]) + elif match.group(2): # Sequence - # TODO match zou episode - new_sequence = gazu.shot.new_sequence(zou_project, doc["name"]) - - # Update doc with zou id - bulk_writes.append( - UpdateOne( - {"_id": doc["_id"]}, - {"$set": {"data.zou_id": new_sequence["id"]}}, - ) + parent_doc = asset_docs[visual_parent_id] + new_entity = gazu.shot.new_sequence( + zou_project, doc["name"], episode=parent_doc["data"]["zou_id"] ) - elif match.group(3): # Shot - pass - # Delete + elif match.group(3): # Shot + # Match and check parent doc + parent_doc = asset_docs[visual_parent_id] + zou_parent_id = parent_doc["data"]["zou_id"] + if parent_doc["data"].get("zou", {}).get("type") != "Sequence": + # Warn + print( + f"Shot {doc['name']} must be parented to a Sequence in Kitsu. Creating automatically one substitute sequence..." + ) + + # Create new sequence + digits_padding = naming_pattern["sequence"].count("#") + substitute_sequence_name = f'{naming_pattern["sequence"].replace("#" * digits_padding, "1".zfill(digits_padding))}' + new_sequence = gazu.shot.new_sequence( + zou_project, substitute_sequence_name, episode=zou_parent_id + ) + + # Insert doc + inserted = project_col.insert_one( + { + "name": substitute_sequence_name, + "type": "asset", + "schema": "openpype:asset-3.0", + "data": { + "zou_id": new_sequence["id"], + "tasks": {}, + "parents": parent_doc["data"]["parents"] + + [parent_doc["name"]], + "visualParent": parent_doc["_id"], + }, + "parent": parent_doc["_id"], + } + ) + visual_parent_id = inserted.inserted_id + + # Update parent ID + zou_parent_id = new_sequence["id"] + + # Create shot + new_entity = gazu.shot.new_shot( + zou_project, + zou_parent_id, + doc["name"], + frame_in=doc["data"]["frameStart"], + frame_out=doc["data"]["frameEnd"], + nb_frames=doc["data"]["frameEnd"] - doc["data"]["frameStart"], + ) + + # Update doc with zou id + doc["data"].update( + { + "zou_id": new_entity["id"], + "visualParent": visual_parent_id, + "zou": new_entity, + } + ) + bulk_writes.append( + UpdateOne( + {"_id": doc["_id"]}, + { + "$set": { + "data.zou_id": new_entity["id"], + "data.visualParent": visual_parent_id, + "data.zou": new_entity, + } + }, + ) + ) + + # TODO update / tasks + + # Delete TODO # if gazu. # Write into DB @@ -477,6 +538,7 @@ def update_op_assets( # Update asset item_doc = asset_doc_ids[item["id"]] item_data = item_doc["data"].copy() + item_data["zou"] = item # Tasks tasks_list = None @@ -484,6 +546,7 @@ def update_op_assets( tasks_list = gazu.task.all_tasks_for_asset(item) elif item["type"] == "Shot": tasks_list = gazu.task.all_tasks_for_shot(item) + # TODO frame in and out if tasks_list: item_data["tasks"] = { t["task_type_name"]: {"type": t["task_type_name"]} for t in tasks_list From 9af6264d1e02f92244d678964da5c5902428016f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 15 Feb 2022 18:21:26 +0100 Subject: [PATCH 13/82] Update and delete in zou --- .../default_modules/kitsu/kitsu_module.py | 146 +++++++----------- 1 file changed, 53 insertions(+), 93 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 453d1c5315..e42edbc52b 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,6 +6,8 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ +from asyncio import all_tasks +from turtle import update import click import os import re @@ -170,6 +172,7 @@ def sync_zou(): dbcon.Session["AVALON_PROJECT"] = project_name # Get all entities from zou + print(f"Synchronizing {project_name}...") zou_project = gazu.project.get_project_by_name(project_name) # Create project @@ -194,11 +197,11 @@ def sync_zou(): all_episodes = gazu.shot.all_episodes_for_project(zou_project) all_seqs = gazu.shot.all_sequences_for_project(zou_project) all_shots = gazu.shot.all_shots_for_project(zou_project) - print(zou_project["name"]) all_entities_ids = { e["id"] for e in all_episodes + all_seqs + all_shots + all_assets } + # Query all assets of the local project project_module_settings = get_project_settings(project_name)["kitsu"] project_col = dbcon.database[project_name] asset_docs = { @@ -207,7 +210,6 @@ def sync_zou(): } # Create new assets - # Query all assets of the local project new_assets_docs = [ doc for doc in asset_docs.values() @@ -225,6 +227,7 @@ def sync_zou(): for doc in new_assets_docs: visual_parent_id = doc["data"]["visualParent"] + # Match asset type by it's name match = regex_ep.match(doc["name"]) if not match: # Asset new_entity = gazu.asset.new_asset( @@ -269,6 +272,7 @@ def sync_zou(): "parents": parent_doc["data"]["parents"] + [parent_doc["name"]], "visualParent": parent_doc["_id"], + "zou": new_sequence, }, "parent": parent_doc["_id"], } @@ -309,10 +313,54 @@ def sync_zou(): ) ) - # TODO update / tasks + # Update assets + all_tasks_types = {t["name"]: t for t in gazu.task.all_task_types()} + assets_docs_to_update = [ + doc + for doc in asset_docs.values() + if doc["data"].get("zou_id") in all_entities_ids + ] + for doc in assets_docs_to_update: + zou_id = doc["data"].get("zou", {}).get("id") + if zou_id: + # Data + entity_data = {} + frame_in = doc["data"].get("frameStart") + frame_out = doc["data"].get("frameEnd") + if frame_in or frame_out: + entity_data.update( + { + "data": {"frame_in": frame_in, "frame_out": frame_out}, + "nb_frames": frame_out - frame_in, + } + ) + entity = gazu.raw.update("entities", zou_id, entity_data) - # Delete TODO - # if gazu. + # Tasks + all_tasks_func = getattr( + gazu.task, f"all_tasks_for_{entity['type'].lower()}" + ) + entity_tasks = {t["name"] for t in all_tasks_func(entity)} + for task_name in doc["data"]["tasks"].keys(): + # Create only if new + if task_name not in entity_tasks: + task_type = all_tasks_types.get(task_name) + + # Create non existing task + if not task_type: + task_type = gazu.task.new_task_type(task_name) + all_tasks_types[task_name] = task_type + + # New task for entity + gazu.task.new_task(entity, task_type) + + # Delete + deleted_entities = all_entities_ids - { + asset_doc["data"].get("zou", {}).get("id") + for asset_doc in asset_docs.values() + } + for entity_id in deleted_entities: + gazu.raw.delete(f"data/entities/{entity_id}") # Write into DB if bulk_writes: @@ -320,94 +368,6 @@ def sync_zou(): dbcon.uninstall() - return - - for project in all_projects: - # Create project locally - # Try to find project document - project_name = project["name"] - project_code = project_name - dbcon.Session["AVALON_PROJECT"] = project_name - project_doc = dbcon.find_one({"type": "project"}) - - # Get all assets from zou - all_assets = gazu.asset.all_assets_for_project(project) - all_episodes = gazu.shot.all_episodes_for_project(project) - all_seqs = gazu.shot.all_sequences_for_project(project) - all_shots = gazu.shot.all_shots_for_project(project) - - # Create project if is not available - # - creation is required to be able set project anatomy and attributes - to_insert = [] - if not project_doc: - print(f"Creating project '{project_name}'") - project_doc = create_project(project_name, project_code, dbcon=dbcon) - - # Project tasks - bulk_writes.append( - UpdateOne( - {"_id": project_doc["_id"]}, - { - "$set": { - "config.tasks": { - t["name"]: { - "short_name": t.get("short_name", t["name"]) - } - for t in gazu.task.all_task_types_for_project(project) - } - } - }, - ) - ) - - # Query all assets of the local project - project_col = dbcon.database[project_code] - asset_doc_ids = { - asset_doc["data"]["zou_id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) - if asset_doc["data"].get("zou_id") - } - asset_doc_ids[project["id"]] = project_doc - - # Create - to_insert.extend( - [ - { - "name": item["name"], - "type": "asset", - "schema": "openpype:asset-3.0", - "data": {"zou_id": item["id"], "tasks": {}}, - } - for item in all_episodes + all_assets + all_seqs + all_shots - if item["id"] not in asset_doc_ids.keys() - ] - ) - if to_insert: - # Insert in doc - project_col.insert_many(to_insert) - - # Update existing docs - asset_doc_ids.update( - { - asset_doc["data"]["zou_id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) - if asset_doc["data"].get("zou_id") - } - ) - - # Update - all_entities = all_assets + all_episodes + all_seqs + all_shots - bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) - - # Delete - diff_assets = set(asset_doc_ids.keys()) - { - e["id"] for e in all_entities + [project] - } - if diff_assets: - bulk_writes.extend( - [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] - ) - @cli_main.command() def sync_openpype(): From b2ce0ac07a04ebdc2e61b4289ec2c2fcde624c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 15 Feb 2022 18:29:43 +0100 Subject: [PATCH 14/82] Shot naming matching --- .../default_modules/kitsu/kitsu_module.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index e42edbc52b..a67f0e2d58 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -233,16 +233,7 @@ def sync_zou(): new_entity = gazu.asset.new_asset( zou_project, all_asset_types[0], doc["name"] ) - - elif match.group(1): # Episode - new_entity = gazu.shot.new_episode(zou_project, doc["name"]) - - elif match.group(2): # Sequence - parent_doc = asset_docs[visual_parent_id] - new_entity = gazu.shot.new_sequence( - zou_project, doc["name"], episode=parent_doc["data"]["zou_id"] - ) - + # Match case in shot Date: Tue, 15 Feb 2022 18:30:06 +0100 Subject: [PATCH 15/82] cleaning --- openpype/modules/default_modules/kitsu/kitsu_module.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index a67f0e2d58..47af25ce60 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,8 +6,6 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ -from asyncio import all_tasks -from turtle import update import click import os import re From a8611c07a8d56942950a2294a95f06f07aa87d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 16 Feb 2022 15:42:23 +0100 Subject: [PATCH 16/82] Clean sync zou --- .../default_modules/kitsu/kitsu_module.py | 138 +++++++++--------- 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 47af25ce60..ce4c579607 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -13,6 +13,24 @@ from typing import Dict, List from avalon.api import AvalonMongoDB import gazu +from gazu.asset import all_assets_for_project, all_asset_types, new_asset +from gazu.shot import ( + all_episodes_for_project, + all_sequences_for_project, + all_shots_for_project, + new_episode, + new_sequence, + new_shot, + update_sequence, +) +from gazu.task import ( + all_tasks_for_asset, + all_tasks_for_shot, + all_task_types, + all_task_types_for_project, + new_task, + new_task_type, +) from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager @@ -176,7 +194,7 @@ def sync_zou(): # Create project if zou_project is None: raise RuntimeError( - f"Project '{project_name}' doesn't exist in Zou database, please create it in Kitsu and add logged user to it before running synchronization." + f"Project '{project_name}' doesn't exist in Zou database, please create it in Kitsu and add OpenPype user to it before running synchronization." ) # Update project settings and data @@ -190,11 +208,11 @@ def sync_zou(): gazu.project.update_project(zou_project) gazu.project.update_project_data(zou_project, data=op_project["data"]) - all_asset_types = gazu.asset.all_asset_types() - all_assets = gazu.asset.all_assets_for_project(zou_project) - all_episodes = gazu.shot.all_episodes_for_project(zou_project) - all_seqs = gazu.shot.all_sequences_for_project(zou_project) - all_shots = gazu.shot.all_shots_for_project(zou_project) + asset_types = all_asset_types() + all_assets = all_assets_for_project(zou_project) + all_episodes = all_episodes_for_project(zou_project) + all_seqs = all_sequences_for_project(zou_project) + all_shots = all_shots_for_project(zou_project) all_entities_ids = { e["id"] for e in all_episodes + all_seqs + all_shots + all_assets } @@ -211,14 +229,14 @@ def sync_zou(): new_assets_docs = [ doc for doc in asset_docs.values() - if doc["data"].get("zou_id") not in all_entities_ids + if doc["data"].get("zou", {}).get("id") not in all_entities_ids ] naming_pattern = project_module_settings["entities_naming_pattern"] regex_ep = re.compile( - r"({})|({})|({})".format( - naming_pattern["episode"].replace("#", "\d"), - naming_pattern["sequence"].replace("#", "\d"), - naming_pattern["shot"].replace("#", "\d"), + r"(.*{}.*)|(.*{}.*)|(.*{}.*)".format( + naming_pattern["shot"].replace("#", ""), + naming_pattern["sequence"].replace("#", ""), + naming_pattern["episode"].replace("#", ""), ), re.IGNORECASE, ) @@ -228,51 +246,38 @@ def sync_zou(): # Match asset type by it's name match = regex_ep.match(doc["name"]) if not match: # Asset - new_entity = gazu.asset.new_asset( - zou_project, all_asset_types[0], doc["name"] - ) + new_entity = new_asset(zou_project, asset_types[0], doc["name"]) # Match case in shot Date: Wed, 16 Feb 2022 17:30:35 +0100 Subject: [PATCH 17/82] Fix sync --- .../default_modules/kitsu/kitsu_module.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index ce4c579607..266aaa9139 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -184,7 +184,6 @@ def sync_zou(): # Create project locally # Try to find project document project_name = op_project["name"] - project_code = op_project["data"]["code"] dbcon.Session["AVALON_PROJECT"] = project_name # Get all entities from zou @@ -198,15 +197,16 @@ def sync_zou(): ) # Update project settings and data - zou_project.update( - { - "code": op_project["data"]["code"], - "fps": op_project["data"]["fps"], - "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", - } - ) + if op_project["data"]: + zou_project.update( + { + "code": op_project["data"]["code"], + "fps": op_project["data"]["fps"], + "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", + } + ) + gazu.project.update_project_data(zou_project, data=op_project["data"]) gazu.project.update_project(zou_project) - gazu.project.update_project_data(zou_project, data=op_project["data"]) asset_types = all_asset_types() all_assets = all_assets_for_project(zou_project) @@ -270,8 +270,9 @@ def sync_zou(): created_sequence = new_sequence( zou_project, substitute_sequence_name, episode=zou_parent_id ) - created_sequence["is_substitute"] = True - update_sequence(created_sequence) + gazu.shot.update_sequence_data( + created_sequence, {"is_substitute": True} + ) # Update parent ID zou_parent_id = created_sequence["id"] @@ -395,11 +396,18 @@ def sync_openpype(): dbcon.Session["AVALON_PROJECT"] = project_name project_doc = dbcon.find_one({"type": "project"}) + print(f"Synchronizing {project_name}...") + # Get all assets from zou all_assets = all_assets_for_project(project) all_episodes = all_episodes_for_project(project) all_seqs = all_sequences_for_project(project) all_shots = all_shots_for_project(project) + all_entities = [ + e + for e in all_assets + all_episodes + all_seqs + all_shots + if not e["data"].get("is_substitute") + ] # Create project if is not available # - creation is required to be able set project anatomy and attributes @@ -421,7 +429,7 @@ def sync_openpype(): "data": project["data"].update( { "code": project["code"], - "fps": project_code["fps"], + "fps": project["fps"], "resolutionWidth": project["resolution"].split("x")[0], "resolutionHeight": project["resolution"].split("x")[1], } @@ -449,9 +457,8 @@ def sync_openpype(): "schema": "openpype:asset-3.0", "data": {"zou": item, "tasks": {}}, } - for item in all_episodes + all_assets + all_seqs + all_shots + for item in all_entities if item["id"] not in asset_doc_ids.keys() - and not item.get("is_substitute") ] ) if to_insert: @@ -468,7 +475,6 @@ def sync_openpype(): ) # Update - all_entities = all_assets + all_episodes + all_seqs + all_shots bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) # Delete From 93fb1ac0f1331439852104189733671386366861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 17 Feb 2022 11:12:13 +0100 Subject: [PATCH 18/82] Use substitutes id --- .../default_modules/kitsu/kitsu_module.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 266aaa9139..0529d38a0e 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -242,6 +242,7 @@ def sync_zou(): ) for doc in new_assets_docs: visual_parent_id = doc["data"]["visualParent"] + parent_substitutes = [] # Match asset type by it's name match = regex_ep.match(doc["name"]) @@ -273,6 +274,7 @@ def sync_zou(): gazu.shot.update_sequence_data( created_sequence, {"is_substitute": True} ) + parent_substitutes.append(created_sequence) # Update parent ID zou_parent_id = created_sequence["id"] @@ -310,6 +312,7 @@ def sync_zou(): "$set": { "data.visualParent": visual_parent_id, "data.zou": new_entity, + "data.parent_substitutes": parent_substitutes, } }, ) @@ -494,17 +497,17 @@ def sync_openpype(): def update_op_assets( - items_list: List[dict], asset_doc_ids: Dict[str, dict] + entities_list: List[dict], asset_doc_ids: Dict[str, dict] ) -> List[UpdateOne]: """Update OpenPype assets. Set 'data' and 'parent' fields. - :param items_list: List of zou items to update + :param entities_list: List of zou entities to update :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] :return: List of UpdateOne objects """ bulk_writes = [] - for item in items_list: + for item in entities_list: # Update asset item_doc = asset_doc_ids[item["id"]] item_data = item_doc["data"].copy() @@ -523,20 +526,28 @@ def update_op_assets( } # Visual parent for hierarchy - direct_parent_id = item["parent_id"] or item["source_id"] - if direct_parent_id: - visual_parent_doc = asset_doc_ids[direct_parent_id] + substitute_parent_item = ( + item_data["parent_substitutes"][0] + if item_data.get("parent_substitutes") + else None + ) + parent_zou_id = item["parent_id"] or item["source_id"] + if substitute_parent_item: + parent_zou_id = ( + substitute_parent_item["parent_id"] + or substitute_parent_item["source_id"] + ) + visual_parent_doc = asset_doc_ids[parent_zou_id] item_data["visualParent"] = visual_parent_doc["_id"] # Add parents for hierarchy - parent_zou_id = item["parent_id"] item_data["parents"] = [] while parent_zou_id is not None: parent_doc = asset_doc_ids[parent_zou_id] item_data["parents"].insert(0, parent_doc["name"]) parent_zou_id = next( - i for i in items_list if i["id"] == parent_doc["data"]["zou"]["id"] + i for i in entities_list if i["id"] == parent_doc["data"]["zou"]["id"] )["parent_id"] # Update 'data' different in zou DB From c058ba54c2763b4563f5912f8a8a8930cceafc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 22 Feb 2022 18:27:06 +0100 Subject: [PATCH 19/82] Project sync and code DRYing --- .../modules/default_modules/kitsu/__init__.py | 10 +-- .../default_modules/kitsu/kitsu_module.py | 52 ++++++---------- .../default_modules/kitsu/listeners.py | 61 +++++++++++++++++++ .../default_modules/kitsu/utils/__init__.py | 0 .../default_modules/kitsu/utils/openpype.py | 47 ++++++++++++++ 5 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 openpype/modules/default_modules/kitsu/listeners.py create mode 100644 openpype/modules/default_modules/kitsu/utils/__init__.py create mode 100644 openpype/modules/default_modules/kitsu/utils/openpype.py diff --git a/openpype/modules/default_modules/kitsu/__init__.py b/openpype/modules/default_modules/kitsu/__init__.py index cd0c2ea8af..dc7c2dad50 100644 --- a/openpype/modules/default_modules/kitsu/__init__.py +++ b/openpype/modules/default_modules/kitsu/__init__.py @@ -4,12 +4,6 @@ If addon class or settings definition won't be here their definition won't be found by OpenPype discovery. """ -from .kitsu_module import ( - AddonSettingsDef, - KitsuModule -) +from .kitsu_module import AddonSettingsDef, KitsuModule -__all__ = ( - "AddonSettingsDef", - "KitsuModule" -) +__all__ = ("AddonSettingsDef", "KitsuModule") diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 0529d38a0e..3fddc48ee9 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -11,7 +11,6 @@ import os import re from typing import Dict, List -from avalon.api import AvalonMongoDB import gazu from gazu.asset import all_assets_for_project, all_asset_types, new_asset from gazu.shot import ( @@ -21,7 +20,6 @@ from gazu.shot import ( new_episode, new_sequence, new_shot, - update_sequence, ) from gazu.task import ( all_tasks_for_asset, @@ -31,11 +29,15 @@ from gazu.task import ( new_task, new_task_type, ) +from pymongo import DeleteOne, UpdateOne + +from avalon.api import AvalonMongoDB from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager -from pymongo import DeleteOne, UpdateOne +from openpype.modules.default_modules.kitsu.utils.openpype import sync_project from openpype_interfaces import IPluginPaths, ITrayAction +from .listeners import add_listeners # Settings definition of this addon using `JsonFilesSettingsDef` @@ -371,8 +373,6 @@ def sync_zou(): if bulk_writes: project_col.bulk_write(bulk_writes) - # TODO Create events daemons - dbcon.uninstall() @@ -412,35 +412,8 @@ def sync_openpype(): if not e["data"].get("is_substitute") ] - # Create project if is not available - # - creation is required to be able set project anatomy and attributes - to_insert = [] - if not project_doc: - print(f"Creating project '{project_name}'") - project_doc = create_project(project_name, project_code, dbcon=dbcon) - - # Project data and tasks - bulk_writes.append( - UpdateOne( - {"_id": project_doc["_id"]}, - { - "$set": { - "config.tasks": { - t["name"]: {"short_name": t.get("short_name", t["name"])} - for t in all_task_types_for_project(project) - }, - "data": project["data"].update( - { - "code": project["code"], - "fps": project["fps"], - "resolutionWidth": project["resolution"].split("x")[0], - "resolutionHeight": project["resolution"].split("x")[1], - } - ), - } - }, - ) - ) + # Sync project. Create if doesn't exist + bulk_writes.append(sync_project(project, dbcon)) # Query all assets of the local project project_col = dbcon.database[project_code] @@ -452,6 +425,7 @@ def sync_openpype(): asset_doc_ids[project["id"]] = project_doc # Create + to_insert = [] to_insert.extend( [ { @@ -572,6 +546,16 @@ def update_op_assets( return bulk_writes +@cli_main.command() +def listen(): + """Show ExampleAddon dialog. + + We don't have access to addon directly through cli so we have to create + it again. + """ + add_listeners() + + @cli_main.command() def show_dialog(): """Show ExampleAddon dialog. diff --git a/openpype/modules/default_modules/kitsu/listeners.py b/openpype/modules/default_modules/kitsu/listeners.py new file mode 100644 index 0000000000..5303933bce --- /dev/null +++ b/openpype/modules/default_modules/kitsu/listeners.py @@ -0,0 +1,61 @@ +from turtle import update +import gazu +import os + +from pymongo import DeleteOne, UpdateOne + +from avalon.api import AvalonMongoDB +from openpype.modules.default_modules.kitsu.utils.openpype import sync_project + + +def add_listeners(): + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + gazu.set_event_host(os.environ["KITSU_SERVER"].replace("api", "socket.io")) + + # Connect to DB + dbcon = AvalonMongoDB() + dbcon.install() + + def new_project(data): + """Create new project into DB.""" + + # Use update process to avoid duplicating code + update_project(data) + + def update_project(data): + """Update project into DB.""" + # Get project entity + project = gazu.project.get_project(data["project_id"]) + project_name = project["name"] + dbcon.Session["AVALON_PROJECT"] = project_name + + update_project = sync_project(project, dbcon) + + # Write into DB + if update_project: + project_col = dbcon.database[project_name] + project_col.bulk_write([update_project]) + + def delete_project(data): + # Get project entity + print(data) # TODO check bugfix + project = gazu.project.get_project(data["project_id"]) + + # Delete project collection + project_col = dbcon.database[project["name"]] + project_col.drop() + + def new_asset(data): + print("Asset created %s" % data) + + event_client = gazu.events.init() + gazu.events.add_listener(event_client, "project:new", new_project) + gazu.events.add_listener(event_client, "project:update", update_project) + gazu.events.add_listener(event_client, "project:delete", delete_project) + gazu.events.add_listener(event_client, "asset:new", new_asset) + gazu.events.run_client(event_client) + print("ll") diff --git a/openpype/modules/default_modules/kitsu/utils/__init__.py b/openpype/modules/default_modules/kitsu/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/default_modules/kitsu/utils/openpype.py new file mode 100644 index 0000000000..9e795bb8ca --- /dev/null +++ b/openpype/modules/default_modules/kitsu/utils/openpype.py @@ -0,0 +1,47 @@ +import gazu + +from pymongo import DeleteOne, UpdateOne + +from avalon.api import AvalonMongoDB +from openpype.lib import create_project + + +def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: + """Sync project with database. + Create project if doesn't exist. + + :param project: Gazu project + :param dbcon: DB to create project in + :return: Update instance for the project + """ + project_name = project["name"] + project_doc = dbcon.find_one({"type": "project"}) + if not project_doc: + print(f"Creating project '{project_name}'") + project_doc = create_project(project_name, project_name, dbcon=dbcon) + + print(f"Synchronizing {project_name}...") + + # Project data and tasks + if not project["data"]: # Sentinel + project["data"] = {} + + return UpdateOne( + {"_id": project_doc["_id"]}, + { + "$set": { + "config.tasks": { + t["name"]: {"short_name": t.get("short_name", t["name"])} + for t in gazu.task.all_task_types_for_project(project) + }, + "data": project["data"].update( + { + "code": project["code"], + "fps": project["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ), + } + }, + ) From 0d38ccc5127bd00157aa9d19157f5d7c3ec3e20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 23 Feb 2022 15:14:38 +0100 Subject: [PATCH 20/82] Listen asset and DRY --- .../default_modules/kitsu/kitsu_module.py | 117 ++++-------------- .../default_modules/kitsu/listeners.py | 70 +++++++++-- .../default_modules/kitsu/utils/openpype.py | 114 +++++++++++++++++ 3 files changed, 198 insertions(+), 103 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 3fddc48ee9..0bdd4b12e6 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -22,10 +22,7 @@ from gazu.shot import ( new_shot, ) from gazu.task import ( - all_tasks_for_asset, - all_tasks_for_shot, all_task_types, - all_task_types_for_project, new_task, new_task_type, ) @@ -33,9 +30,12 @@ from pymongo import DeleteOne, UpdateOne from avalon.api import AvalonMongoDB from openpype.api import get_project_settings -from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager -from openpype.modules.default_modules.kitsu.utils.openpype import sync_project +from openpype.modules.default_modules.kitsu.utils.openpype import ( + create_op_asset, + sync_project, + update_op_assets, +) from openpype_interfaces import IPluginPaths, ITrayAction from .listeners import add_listeners @@ -417,33 +417,28 @@ def sync_openpype(): # Query all assets of the local project project_col = dbcon.database[project_code] - asset_doc_ids = { + zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc for asset_doc in project_col.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } - asset_doc_ids[project["id"]] = project_doc + zou_ids_and_asset_docs[project["id"]] = project_doc # Create to_insert = [] to_insert.extend( [ - { - "name": item["name"], - "type": "asset", - "schema": "openpype:asset-3.0", - "data": {"zou": item, "tasks": {}}, - } + create_op_asset(item) for item in all_entities - if item["id"] not in asset_doc_ids.keys() + if item["id"] not in zou_ids_and_asset_docs.keys() ] ) if to_insert: - # Insert in doc + # Insert doc in DB project_col.insert_many(to_insert) # Update existing docs - asset_doc_ids.update( + zou_ids_and_asset_docs.update( { asset_doc["data"]["zou"]["id"]: asset_doc for asset_doc in project_col.find({"type": "asset"}) @@ -452,15 +447,23 @@ def sync_openpype(): ) # Update - bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) + bulk_writes.extend( + [ + UpdateOne({"_id": id}, update) + for id, update in update_op_assets(all_entities, zou_ids_and_asset_docs) + ] + ) # Delete - diff_assets = set(asset_doc_ids.keys()) - { + diff_assets = set(zou_ids_and_asset_docs.keys()) - { e["id"] for e in all_entities + [project] } if diff_assets: bulk_writes.extend( - [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] + [ + DeleteOne(zou_ids_and_asset_docs[asset_id]) + for asset_id in diff_assets + ] ) # Write into DB @@ -470,82 +473,6 @@ def sync_openpype(): dbcon.uninstall() -def update_op_assets( - entities_list: List[dict], asset_doc_ids: Dict[str, dict] -) -> List[UpdateOne]: - """Update OpenPype assets. - Set 'data' and 'parent' fields. - - :param entities_list: List of zou entities to update - :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] - :return: List of UpdateOne objects - """ - bulk_writes = [] - for item in entities_list: - # Update asset - item_doc = asset_doc_ids[item["id"]] - item_data = item_doc["data"].copy() - item_data["zou"] = item - - # Tasks - tasks_list = None - if item["type"] == "Asset": - tasks_list = all_tasks_for_asset(item) - elif item["type"] == "Shot": - tasks_list = all_tasks_for_shot(item) - # TODO frame in and out - if tasks_list: - item_data["tasks"] = { - t["task_type_name"]: {"type": t["task_type_name"]} for t in tasks_list - } - - # Visual parent for hierarchy - substitute_parent_item = ( - item_data["parent_substitutes"][0] - if item_data.get("parent_substitutes") - else None - ) - parent_zou_id = item["parent_id"] or item["source_id"] - if substitute_parent_item: - parent_zou_id = ( - substitute_parent_item["parent_id"] - or substitute_parent_item["source_id"] - ) - visual_parent_doc = asset_doc_ids[parent_zou_id] - item_data["visualParent"] = visual_parent_doc["_id"] - - # Add parents for hierarchy - item_data["parents"] = [] - while parent_zou_id is not None: - parent_doc = asset_doc_ids[parent_zou_id] - item_data["parents"].insert(0, parent_doc["name"]) - - parent_zou_id = next( - i for i in entities_list if i["id"] == parent_doc["data"]["zou"]["id"] - )["parent_id"] - - # Update 'data' different in zou DB - updated_data = { - k: item_data[k] - for k in item_data.keys() - if item_doc["data"].get(k) != item_data[k] - } - if updated_data or not item_doc.get("parent"): - bulk_writes.append( - UpdateOne( - {"_id": item_doc["_id"]}, - { - "$set": { - "data": item_data, - "parent": asset_doc_ids[item["project_id"]]["_id"], - } - }, - ) - ) - - return bulk_writes - - @cli_main.command() def listen(): """Show ExampleAddon dialog. diff --git a/openpype/modules/default_modules/kitsu/listeners.py b/openpype/modules/default_modules/kitsu/listeners.py index 5303933bce..fd4cf6dd3d 100644 --- a/openpype/modules/default_modules/kitsu/listeners.py +++ b/openpype/modules/default_modules/kitsu/listeners.py @@ -5,7 +5,12 @@ import os from pymongo import DeleteOne, UpdateOne from avalon.api import AvalonMongoDB -from openpype.modules.default_modules.kitsu.utils.openpype import sync_project +from openpype.modules.default_modules.kitsu.utils.openpype import ( + create_op_asset, + set_op_project, + sync_project, + update_op_assets, +) def add_listeners(): @@ -15,19 +20,22 @@ def add_listeners(): # Authenticate gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) gazu.set_event_host(os.environ["KITSU_SERVER"].replace("api", "socket.io")) + event_client = gazu.events.init() # Connect to DB dbcon = AvalonMongoDB() dbcon.install() + # == Project == + def new_project(data): - """Create new project into DB.""" + """Create new project into OP DB.""" # Use update process to avoid duplicating code update_project(data) def update_project(data): - """Update project into DB.""" + """Update project into OP DB.""" # Get project entity project = gazu.project.get_project(data["project_id"]) project_name = project["name"] @@ -41,6 +49,7 @@ def add_listeners(): project_col.bulk_write([update_project]) def delete_project(data): + """Delete project.""" # Get project entity print(data) # TODO check bugfix project = gazu.project.get_project(data["project_id"]) @@ -49,13 +58,58 @@ def add_listeners(): project_col = dbcon.database[project["name"]] project_col.drop() - def new_asset(data): - print("Asset created %s" % data) - - event_client = gazu.events.init() gazu.events.add_listener(event_client, "project:new", new_project) gazu.events.add_listener(event_client, "project:update", update_project) gazu.events.add_listener(event_client, "project:delete", delete_project) + + # == Asset == + + def new_asset(data): + """Create new asset into OP DB.""" + # Get project entity + project_col = set_op_project(dbcon, data["project_id"]) + + # Get gazu entity + asset = gazu.asset.get_asset(data["asset_id"]) + + # Insert doc in DB + project_col.insert_one(create_op_asset(asset)) + + # Update + update_asset(data) + + def update_asset(data): + """Update asset into OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + project_doc = dbcon.find_one({"type": "project"}) + + # Get gazu entity + asset = gazu.asset.get_asset(data["asset_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[asset["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets([asset], zou_ids_and_asset_docs)[ + 0 + ] + project_col.update_one({"_id": asset_doc_id}, asset_update) + + def delete_asset(data): + """Delete asset of OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + + # Delete + project_col.delete_one({"type": "asset", "data.zou.id": data["asset_id"]}) + gazu.events.add_listener(event_client, "asset:new", new_asset) + gazu.events.add_listener(event_client, "asset:update", update_asset) + gazu.events.add_listener(event_client, "asset:delete", delete_asset) + gazu.events.run_client(event_client) - print("ll") diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/default_modules/kitsu/utils/openpype.py index 9e795bb8ca..8ef987898d 100644 --- a/openpype/modules/default_modules/kitsu/utils/openpype.py +++ b/openpype/modules/default_modules/kitsu/utils/openpype.py @@ -1,11 +1,125 @@ +from typing import Dict, List import gazu from pymongo import DeleteOne, UpdateOne +from pymongo.collection import Collection + +from gazu.task import ( + all_tasks_for_asset, + all_tasks_for_shot, +) from avalon.api import AvalonMongoDB from openpype.lib import create_project +def create_op_asset(gazu_entity: dict) -> dict: + """Create OP asset dict from gazu entity. + + :param gazu_entity: + """ + return { + "name": gazu_entity["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": {"zou": gazu_entity, "tasks": {}}, + } + + +def set_op_project(dbcon, project_id) -> Collection: + """Set project context. + + :param dbcon: Connection to DB. + :param project_id: Project zou ID + """ + project = gazu.project.get_project(project_id) + project_name = project["name"] + dbcon.Session["AVALON_PROJECT"] = project_name + + return dbcon.database[project_name] + + +def update_op_assets( + entities_list: List[dict], asset_doc_ids: Dict[str, dict] +) -> List[Dict[str, dict]]: + """Update OpenPype assets. + Set 'data' and 'parent' fields. + + :param entities_list: List of zou entities to update + :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] + :return: List of (doc_id, update_dict) tuples + """ + assets_with_update = [] + for item in entities_list: + # Update asset + item_doc = asset_doc_ids[item["id"]] + item_data = item_doc["data"].copy() + item_data["zou"] = item + + # Tasks + tasks_list = [] + if item["type"] == "Asset": + tasks_list = all_tasks_for_asset(item) + elif item["type"] == "Shot": + tasks_list = all_tasks_for_shot(item) + # TODO frame in and out + item_data["tasks"] = { + t["task_type_name"]: {"type": t["task_type_name"]} for t in tasks_list + } + + # Get zou parent id for correct hierarchy + # Use parent substitutes if existing + substitute_parent_item = ( + item_data["parent_substitutes"][0] + if item_data.get("parent_substitutes") + else None + ) + if substitute_parent_item: + parent_zou_id = substitute_parent_item["id"] + else: + parent_zou_id = ( + item.get("parent_id") or item.get("episode_id") or item.get("source_id") + ) # TODO check consistency + + # Visual parent for hierarchy + visual_parent_doc_id = ( + asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None + ) + item_data["visualParent"] = visual_parent_doc_id + + # Add parents for hierarchy + item_data["parents"] = [] + while parent_zou_id is not None: + parent_doc = asset_doc_ids[parent_zou_id] + item_data["parents"].insert(0, parent_doc["name"]) + + # Get parent entity + parent_entity = parent_doc["data"]["zou"] + parent_zou_id = parent_entity["parent_id"] + + # Update 'data' different in zou DB + updated_data = { + k: item_data[k] + for k in item_data.keys() + if item_doc["data"].get(k) != item_data[k] + } + if updated_data or not item_doc.get("parent"): + assets_with_update.append( + ( + item_doc["_id"], + { + "$set": { + "name": item["name"], + "data": item_data, + "parent": asset_doc_ids[item["project_id"]]["_id"], + } + }, + ) + ) + + return assets_with_update + + def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: """Sync project with database. Create project if doesn't exist. From fe5486886712a77cd40f37cac6ec41d329c17748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 23 Feb 2022 17:04:21 +0100 Subject: [PATCH 21/82] Listen Episode, Sequence and Shot --- .../default_modules/kitsu/listeners.py | 147 ++++++++++++++++++ .../default_modules/kitsu/utils/openpype.py | 2 +- 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/kitsu/listeners.py b/openpype/modules/default_modules/kitsu/listeners.py index fd4cf6dd3d..bb217452df 100644 --- a/openpype/modules/default_modules/kitsu/listeners.py +++ b/openpype/modules/default_modules/kitsu/listeners.py @@ -112,4 +112,151 @@ def add_listeners(): gazu.events.add_listener(event_client, "asset:update", update_asset) gazu.events.add_listener(event_client, "asset:delete", delete_asset) + # == Episode == + def new_episode(data): + """Create new episode into OP DB.""" + # Get project entity + project_col = set_op_project(dbcon, data["project_id"]) + + # Get gazu entity + episode = gazu.shot.get_episode(data["episode_id"]) + + # Insert doc in DB + project_col.insert_one(create_op_asset(episode)) + + # Update + update_episode(data) + + def update_episode(data): + """Update episode into OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + project_doc = dbcon.find_one({"type": "project"}) + + # Get gazu entity + episode = gazu.shot.get_episode(data["episode_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[episode["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + [episode], zou_ids_and_asset_docs + )[0] + project_col.update_one({"_id": asset_doc_id}, asset_update) + + def delete_episode(data): + """Delete shot of OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + print("delete episode") # TODO check bugfix + + # Delete + project_col.delete_one({"type": "asset", "data.zou.id": data["episode_id"]}) + + gazu.events.add_listener(event_client, "episode:new", new_episode) + gazu.events.add_listener(event_client, "episode:update", update_episode) + gazu.events.add_listener(event_client, "episode:delete", delete_episode) + + # == Sequence == + def new_sequence(data): + """Create new sequnce into OP DB.""" + # Get project entity + project_col = set_op_project(dbcon, data["project_id"]) + + # Get gazu entity + sequence = gazu.shot.get_sequence(data["sequence_id"]) + + # Insert doc in DB + project_col.insert_one(create_op_asset(sequence)) + + # Update + update_sequence(data) + + def update_sequence(data): + """Update sequence into OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + project_doc = dbcon.find_one({"type": "project"}) + + # Get gazu entity + sequence = gazu.shot.get_sequence(data["sequence_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[sequence["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + [sequence], zou_ids_and_asset_docs + )[0] + project_col.update_one({"_id": asset_doc_id}, asset_update) + + def delete_sequence(data): + """Delete sequence of OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + print("delete sequence") # TODO check bugfix + + # Delete + project_col.delete_one({"type": "asset", "data.zou.id": data["sequence_id"]}) + + gazu.events.add_listener(event_client, "sequence:new", new_sequence) + gazu.events.add_listener(event_client, "sequence:update", update_sequence) + gazu.events.add_listener(event_client, "sequence:delete", delete_sequence) + + # == Shot == + def new_shot(data): + """Create new shot into OP DB.""" + # Get project entity + project_col = set_op_project(dbcon, data["project_id"]) + + # Get gazu entity + shot = gazu.shot.get_shot(data["shot_id"]) + + # Insert doc in DB + project_col.insert_one(create_op_asset(shot)) + + # Update + update_shot(data) + + def update_shot(data): + """Update shot into OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + project_doc = dbcon.find_one({"type": "project"}) + + # Get gazu entity + shot = gazu.shot.get_shot(data["shot_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[shot["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets([shot], zou_ids_and_asset_docs)[0] + project_col.update_one({"_id": asset_doc_id}, asset_update) + + def delete_shot(data): + """Delete shot of OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + + # Delete + project_col.delete_one({"type": "asset", "data.zou.id": data["shot_id"]}) + + gazu.events.add_listener(event_client, "shot:new", new_shot) + gazu.events.add_listener(event_client, "shot:update", update_shot) + gazu.events.add_listener(event_client, "shot:delete", delete_shot) + gazu.events.run_client(event_client) diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/default_modules/kitsu/utils/openpype.py index 8ef987898d..809748fa78 100644 --- a/openpype/modules/default_modules/kitsu/utils/openpype.py +++ b/openpype/modules/default_modules/kitsu/utils/openpype.py @@ -75,7 +75,7 @@ def update_op_assets( else None ) if substitute_parent_item: - parent_zou_id = substitute_parent_item["id"] + parent_zou_id = substitute_parent_item["parent_id"] else: parent_zou_id = ( item.get("parent_id") or item.get("episode_id") or item.get("source_id") From de153a0450d9cbdcf9229775fa8fa9afa94eef79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 24 Feb 2022 15:53:03 +0100 Subject: [PATCH 22/82] Listen tasks, clean code --- .../default_modules/kitsu/kitsu_module.py | 1 - .../default_modules/kitsu/listeners.py | 55 ++++++++++++++++++- .../default_modules/kitsu/utils/openpype.py | 2 +- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 0bdd4b12e6..10cbe76db2 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -9,7 +9,6 @@ in global space here until are required or used. import click import os import re -from typing import Dict, List import gazu from gazu.asset import all_assets_for_project, all_asset_types, new_asset diff --git a/openpype/modules/default_modules/kitsu/listeners.py b/openpype/modules/default_modules/kitsu/listeners.py index bb217452df..16c4e0e69e 100644 --- a/openpype/modules/default_modules/kitsu/listeners.py +++ b/openpype/modules/default_modules/kitsu/listeners.py @@ -1,9 +1,6 @@ -from turtle import update import gazu import os -from pymongo import DeleteOne, UpdateOne - from avalon.api import AvalonMongoDB from openpype.modules.default_modules.kitsu.utils.openpype import ( create_op_asset, @@ -259,4 +256,56 @@ def add_listeners(): gazu.events.add_listener(event_client, "shot:update", update_shot) gazu.events.add_listener(event_client, "shot:delete", delete_shot) + # == Task == + def new_task(data): + """Create new task into OP DB.""" + print("new", data) + # Get project entity + project_col = set_op_project(dbcon, data["project_id"]) + + # Get gazu entity + task = gazu.task.get_task(data["task_id"]) + + # Find asset doc + asset_doc = project_col.find_one( + {"type": "asset", "data.zou.id": task["entity"]["id"]} + ) + + # Update asset tasks with new one + asset_tasks = asset_doc["data"].get("tasks") + task_type_name = task["task_type"]["name"] + asset_tasks[task_type_name] = {"type": task_type_name, "zou": task} + project_col.update_one( + {"_id": asset_doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} + ) + + def update_task(data): + """Update task into OP DB.""" + # TODO is it necessary? + pass + + def delete_task(data): + """Delete task of OP DB.""" + project_col = set_op_project(dbcon, data["project_id"]) + + # Find asset doc + asset_docs = [doc for doc in project_col.find({"type": "asset"})] + for doc in asset_docs: + # Match task + for name, task in doc["data"]["tasks"].items(): + if task.get("zou") and data["task_id"] == task["zou"]["id"]: + # Pop task + asset_tasks = doc["data"].get("tasks") + asset_tasks.pop(name) + + # Delete task in DB + project_col.update_one( + {"_id": doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} + ) + return + + gazu.events.add_listener(event_client, "task:new", new_task) + gazu.events.add_listener(event_client, "task:update", update_task) + gazu.events.add_listener(event_client, "task:delete", delete_task) + gazu.events.run_client(event_client) diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/default_modules/kitsu/utils/openpype.py index 809748fa78..edcf233ea8 100644 --- a/openpype/modules/default_modules/kitsu/utils/openpype.py +++ b/openpype/modules/default_modules/kitsu/utils/openpype.py @@ -1,7 +1,7 @@ from typing import Dict, List import gazu -from pymongo import DeleteOne, UpdateOne +from pymongo import UpdateOne from pymongo.collection import Collection from gazu.task import ( From 6cf38e1a35ba73a5df64969befb71728ffec4813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 25 Feb 2022 16:23:14 +0100 Subject: [PATCH 23/82] Publish comment to kitsu --- .../default_modules/kitsu/kitsu_module.py | 2 +- .../kitsu/plugins/publish/example_plugin.py | 40 +++++++++++++++++++ .../default_modules/kitsu/utils/openpype.py | 24 +++++------ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index 10cbe76db2..f62a86f04d 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -408,7 +408,7 @@ def sync_openpype(): all_entities = [ e for e in all_assets + all_episodes + all_seqs + all_shots - if not e["data"].get("is_substitute") + if e["data"] and not e["data"].get("is_substitute") ] # Sync project. Create if doesn't exist diff --git a/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py b/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py index 61602f4e78..614d9ecc38 100644 --- a/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py +++ b/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py @@ -1,9 +1,49 @@ +import os + +import gazu + import pyblish.api +import debugpy + class CollectExampleAddon(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + 0.4 label = "Collect Kitsu" def process(self, context): + debugpy.breakpoint() self.log.info("I'm in Kitsu's plugin!") + + +class IntegrateRig(pyblish.api.InstancePlugin): + """Copy files to an appropriate location where others may reach it""" + + order = pyblish.api.IntegratorOrder + families = ["model"] + + def process(self, instance): + print(instance.data["version"]) + + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + + asset_data = instance.data["assetEntity"]["data"] + + # TODO Set local settings for login and password + + # Get task + task_type = gazu.task.get_task_type_by_name(instance.data["task"]) + entity_task = gazu.task.get_task_by_entity(asset_data["zou"], task_type) + + # Comment entity + gazu.task.add_comment( + entity_task, + entity_task["task_status_id"], + comment=f"Version {instance.data['version']} has been published!", + ) + + self.log.info("Copied successfully!") diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/default_modules/kitsu/utils/openpype.py index edcf233ea8..518872a71c 100644 --- a/openpype/modules/default_modules/kitsu/utils/openpype.py +++ b/openpype/modules/default_modules/kitsu/utils/openpype.py @@ -134,11 +134,18 @@ def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: print(f"Creating project '{project_name}'") project_doc = create_project(project_name, project_name, dbcon=dbcon) - print(f"Synchronizing {project_name}...") - # Project data and tasks - if not project["data"]: # Sentinel - project["data"] = {} + project_data = project["data"] or {} + + # Update data + project_data.update( + { + "code": project["code"], + "fps": project["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ) return UpdateOne( {"_id": project_doc["_id"]}, @@ -148,14 +155,7 @@ def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: t["name"]: {"short_name": t.get("short_name", t["name"])} for t in gazu.task.all_task_types_for_project(project) }, - "data": project["data"].update( - { - "code": project["code"], - "fps": project["fps"], - "resolutionWidth": project["resolution"].split("x")[0], - "resolutionHeight": project["resolution"].split("x")[1], - } - ), + "data": project_data, } }, ) From cb01f8338c3b651897d626479a1a1bd306a7e05f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 1 Mar 2022 17:56:08 +0100 Subject: [PATCH 24/82] Sign in dialog, credentials as local setting and cleaning --- .../default_modules/kitsu/kitsu_module.py | 72 +++---- .../default_modules/kitsu/kitsu_widgets.py | 193 ++++++++++++++++++ .../{example_plugin.py => kitsu_plugin.py} | 8 +- .../schemas/project_schemas/main.json | 30 --- .../schemas/project_schemas/the_template.json | 30 --- .../kitsu/{ => utils}/listeners.py | 2 +- .../modules/default_modules/kitsu/widgets.py | 31 --- .../defaults/system_settings/modules.json | 4 +- .../module_settings/schema_kitsu.json | 10 - pyproject.toml | 3 +- 10 files changed, 224 insertions(+), 159 deletions(-) create mode 100644 openpype/modules/default_modules/kitsu/kitsu_widgets.py rename openpype/modules/default_modules/kitsu/plugins/publish/{example_plugin.py => kitsu_plugin.py} (85%) delete mode 100644 openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json delete mode 100644 openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json rename openpype/modules/default_modules/kitsu/{ => utils}/listeners.py (99%) delete mode 100644 openpype/modules/default_modules/kitsu/widgets.py diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index f62a86f04d..51ab8aaa42 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -1,11 +1,5 @@ -"""Addon definition is located here. +"""Kitsu module.""" -Import of python packages that may not be available should not be imported -in global space here until are required or used. -- Qt related imports -- imports of Python 3 packages - - we still support Python 2 hosts where addon definition should available -""" import click import os import re @@ -35,20 +29,12 @@ from openpype.modules.default_modules.kitsu.utils.openpype import ( sync_project, update_op_assets, ) +from openpype.settings.lib import get_local_settings from openpype_interfaces import IPluginPaths, ITrayAction -from .listeners import add_listeners +from .utils.listeners import start_listeners -# Settings definition of this addon using `JsonFilesSettingsDef` -# - JsonFilesSettingsDef is prepared settings definition using json files -# to define settings and store default values class AddonSettingsDef(JsonFilesSettingsDef): - # This will add prefixes to every schema and template from `schemas` - # subfolder. - # - it is not required to fill the prefix but it is highly - # recommended as schemas and templates may have name clashes across - # multiple addons - # - it is also recommended that prefix has addon name in it schema_prefix = "kitsu" def get_settings_root_path(self): @@ -61,20 +47,15 @@ class AddonSettingsDef(JsonFilesSettingsDef): class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): - """This Addon has defined it's settings and interface. - - This example has system settings with an enabled option. And use - few other interfaces: - - `IPluginPaths` to define custom plugin paths - - `ITrayAction` to be shown in tray tool - """ + """Kitsu module class.""" label = "Kitsu" name = "kitsu" def initialize(self, settings): - """Initialization of addon.""" + """Initialization of module.""" module_settings = settings[self.name] + local_kitsu_settings = get_local_settings().get("kitsu", {}) # Enabled by settings self.enabled = module_settings.get("enabled", False) @@ -93,8 +74,8 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): self.server_url = kitsu_url # Set credentials - self.script_login = module_settings["script_login"] - self.script_pwd = module_settings["script_pwd"] + self.kitsu_login = local_kitsu_settings["login"] + self.kitsu_password = local_kitsu_settings["password"] # Prepare variables that can be used or set afterwards self._connected_modules = None @@ -113,8 +94,8 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): """Kitsu's global environments.""" return { "KITSU_SERVER": self.server_url, - "KITSU_LOGIN": self.script_login, - "KITSU_PWD": self.script_pwd, + "KITSU_LOGIN": self.kitsu_login, + "KITSU_PWD": self.kitsu_password, } def _create_dialog(self): @@ -122,9 +103,9 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): if self._dialog is not None: return - from .widgets import MyExampleDialog + from .kitsu_widgets import PasswordDialog - self._dialog = MyExampleDialog() + self._dialog = PasswordDialog() def show_dialog(self): """Show dialog with connected modules. @@ -376,7 +357,10 @@ def sync_zou(): @cli_main.command() -def sync_openpype(): +@click.option( + "-l", "--listen", is_flag=True, help="Listen Kitsu server after synchronization." +) +def sync_openpype(listen: bool): """Synchronize openpype database from Zou sever database.""" # Connect to server @@ -471,27 +455,23 @@ def sync_openpype(): dbcon.uninstall() + # Run listening + if listen: + start_listeners() + @cli_main.command() def listen(): - """Show ExampleAddon dialog. - - We don't have access to addon directly through cli so we have to create - it again. - """ - add_listeners() + """Listen to Kitsu server.""" + start_listeners() @cli_main.command() -def show_dialog(): - """Show ExampleAddon dialog. - - We don't have access to addon directly through cli so we have to create - it again. - """ +def sign_in(): + """Show credentials dialog.""" from openpype.tools.utils.lib import qt_app_context manager = ModulesManager() - example_addon = manager.modules_by_name[KitsuModule.name] + kitsu_addon = manager.modules_by_name[KitsuModule.name] with qt_app_context(): - example_addon.show_dialog() + kitsu_addon.show_dialog() diff --git a/openpype/modules/default_modules/kitsu/kitsu_widgets.py b/openpype/modules/default_modules/kitsu/kitsu_widgets.py new file mode 100644 index 0000000000..6bd436f460 --- /dev/null +++ b/openpype/modules/default_modules/kitsu/kitsu_widgets.py @@ -0,0 +1,193 @@ +import os + +import gazu +from Qt import QtWidgets, QtCore, QtGui + +from openpype import style +from openpype.resources import get_resource +from openpype.settings.lib import ( + get_local_settings, + get_system_settings, + save_local_settings, +) + +from openpype.widgets.password_dialog import PressHoverButton + + +class PasswordDialog(QtWidgets.QDialog): + """Stupidly simple dialog to compare password from general settings.""" + + finished = QtCore.Signal(bool) + + def __init__(self, parent=None): + super(PasswordDialog, self).__init__(parent) + + self.setWindowTitle("Kitsu Credentials") + self.resize(300, 120) + + system_settings = get_system_settings() + kitsu_settings = get_local_settings().get("kitsu", {}) + remembered = kitsu_settings.get("remember") + + self._final_result = None + self._connectable = bool( + system_settings["modules"].get("kitsu", {}).get("server") + ) + + # Server label + server_label = QtWidgets.QLabel( + f"Server: {system_settings['modules']['kitsu']['server'] if self._connectable else 'no server url set in Studio Settings...'}", + self, + ) + + # Login input + login_widget = QtWidgets.QWidget(self) + + login_label = QtWidgets.QLabel("Login:", login_widget) + + login_input = QtWidgets.QLineEdit( + login_widget, text=kitsu_settings.get("login") if remembered else None + ) + login_input.setPlaceholderText("Your Kitsu account login...") + + login_layout = QtWidgets.QHBoxLayout(login_widget) + login_layout.setContentsMargins(0, 0, 0, 0) + login_layout.addWidget(login_label) + login_layout.addWidget(login_input) + + # Password input + password_widget = QtWidgets.QWidget(self) + + password_label = QtWidgets.QLabel("Password:", password_widget) + + password_input = QtWidgets.QLineEdit( + password_widget, text=kitsu_settings.get("password") if remembered else None + ) + password_input.setPlaceholderText("Your password...") + password_input.setEchoMode(QtWidgets.QLineEdit.Password) + + show_password_icon_path = get_resource("icons", "eye.png") + show_password_icon = QtGui.QIcon(show_password_icon_path) + show_password_btn = PressHoverButton(password_widget) + show_password_btn.setObjectName("PasswordBtn") + show_password_btn.setIcon(show_password_icon) + show_password_btn.setFocusPolicy(QtCore.Qt.ClickFocus) + + password_layout = QtWidgets.QHBoxLayout(password_widget) + password_layout.setContentsMargins(0, 0, 0, 0) + password_layout.addWidget(password_label) + password_layout.addWidget(password_input) + password_layout.addWidget(show_password_btn) + + # Message label + message_label = QtWidgets.QLabel("", self) + + # Buttons + buttons_widget = QtWidgets.QWidget(self) + + remember_checkbox = QtWidgets.QCheckBox("Remember", buttons_widget) + remember_checkbox.setObjectName("RememberCheckbox") + remember_checkbox.setChecked(remembered if remembered is not None else True) + + ok_btn = QtWidgets.QPushButton("Ok", buttons_widget) + cancel_btn = QtWidgets.QPushButton("Cancel", buttons_widget) + + buttons_layout = QtWidgets.QHBoxLayout(buttons_widget) + buttons_layout.setContentsMargins(0, 0, 0, 0) + buttons_layout.addWidget(remember_checkbox) + buttons_layout.addStretch(1) + buttons_layout.addWidget(ok_btn) + buttons_layout.addWidget(cancel_btn) + + # Main layout + layout = QtWidgets.QVBoxLayout(self) + layout.addSpacing(5) + layout.addWidget(server_label, 0) + layout.addSpacing(5) + layout.addWidget(login_widget, 0) + layout.addWidget(password_widget, 0) + layout.addWidget(message_label, 0) + layout.addStretch(1) + layout.addWidget(buttons_widget, 0) + + ok_btn.clicked.connect(self._on_ok_click) + cancel_btn.clicked.connect(self._on_cancel_click) + show_password_btn.change_state.connect(self._on_show_password) + + self.login_input = login_input + self.password_input = password_input + self.remember_checkbox = remember_checkbox + self.message_label = message_label + + self.setStyleSheet(style.load_stylesheet()) + + def result(self): + return self._final_result + + def keyPressEvent(self, event): + if event.key() in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter): + self._on_ok_click() + return event.accept() + super(PasswordDialog, self).keyPressEvent(event) + + def closeEvent(self, event): + super(PasswordDialog, self).closeEvent(event) + self.finished.emit(self.result()) + + def _on_ok_click(self): + # Check if is connectable + if not self._connectable: + self.message_label.setText("Please set server url in Studio Settings!") + return + + # Collect values + login_value = self.login_input.text() + pwd_value = self.password_input.text() + remember = self.remember_checkbox.isChecked() + + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(login_value, pwd_value) + + # Set logging-in env vars + os.environ["KITSU_LOGIN"] = login_value + os.environ["KITSU_PWD"] = pwd_value + + # Get settings + local_settings = get_local_settings() + local_settings.setdefault("kitsu", {}) + + # Remember password cases + if remember: + # Set local settings + local_settings["kitsu"]["login"] = login_value + local_settings["kitsu"]["password"] = pwd_value + else: + # Clear local settings + local_settings["kitsu"]["login"] = None + local_settings["kitsu"]["password"] = None + + # Clear input fields + self.login_input.clear() + self.password_input.clear() + + # Keep 'remember' parameter + local_settings["kitsu"]["remember"] = remember + + # Save settings + save_local_settings(local_settings) + + self._final_result = True + self.close() + + def _on_show_password(self, show_password): + if show_password: + echo_mode = QtWidgets.QLineEdit.Normal + else: + echo_mode = QtWidgets.QLineEdit.Password + self.password_input.setEchoMode(echo_mode) + + def _on_cancel_click(self): + self.close() diff --git a/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py b/openpype/modules/default_modules/kitsu/plugins/publish/kitsu_plugin.py similarity index 85% rename from openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py rename to openpype/modules/default_modules/kitsu/plugins/publish/kitsu_plugin.py index 614d9ecc38..86ba40a5f4 100644 --- a/openpype/modules/default_modules/kitsu/plugins/publish/example_plugin.py +++ b/openpype/modules/default_modules/kitsu/plugins/publish/kitsu_plugin.py @@ -4,15 +4,12 @@ import gazu import pyblish.api -import debugpy - class CollectExampleAddon(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + 0.4 label = "Collect Kitsu" def process(self, context): - debugpy.breakpoint() self.log.info("I'm in Kitsu's plugin!") @@ -23,7 +20,6 @@ class IntegrateRig(pyblish.api.InstancePlugin): families = ["model"] def process(self, instance): - print(instance.data["version"]) # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) @@ -33,8 +29,6 @@ class IntegrateRig(pyblish.api.InstancePlugin): asset_data = instance.data["assetEntity"]["data"] - # TODO Set local settings for login and password - # Get task task_type = gazu.task.get_task_type_by_name(instance.data["task"]) entity_task = gazu.task.get_task_by_entity(asset_data["zou"], task_type) @@ -46,4 +40,4 @@ class IntegrateRig(pyblish.api.InstancePlugin): comment=f"Version {instance.data['version']} has been published!", ) - self.log.info("Copied successfully!") + self.log.info("Version published to Kitsu successfully!") diff --git a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json deleted file mode 100644 index 82e58ce9ab..0000000000 --- a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/main.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "dict", - "key": "kitsu", - "label": " Kitsu", - "collapsible": true, - "children": [ - { - "type": "number", - "key": "number", - "label": "This is your lucky number:", - "minimum": 7, - "maximum": 7, - "decimals": 0 - }, - { - "type": "template", - "name": "kitsu/the_template", - "template_data": [ - { - "name": "color_1", - "label": "Color 1" - }, - { - "name": "color_2", - "label": "Color 2" - } - ] - } - ] -} diff --git a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json b/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json deleted file mode 100644 index af8fd9dae4..0000000000 --- a/openpype/modules/default_modules/kitsu/settings/schemas/project_schemas/the_template.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "type": "list-strict", - "key": "{name}", - "label": "{label}", - "object_types": [ - { - "label": "Red", - "type": "number", - "minimum": 0, - "maximum": 1, - "decimal": 3 - }, - { - "label": "Green", - "type": "number", - "minimum": 0, - "maximum": 1, - "decimal": 3 - }, - { - "label": "Blue", - "type": "number", - "minimum": 0, - "maximum": 1, - "decimal": 3 - } - ] - } -] diff --git a/openpype/modules/default_modules/kitsu/listeners.py b/openpype/modules/default_modules/kitsu/utils/listeners.py similarity index 99% rename from openpype/modules/default_modules/kitsu/listeners.py rename to openpype/modules/default_modules/kitsu/utils/listeners.py index 16c4e0e69e..c99870fa36 100644 --- a/openpype/modules/default_modules/kitsu/listeners.py +++ b/openpype/modules/default_modules/kitsu/utils/listeners.py @@ -2,7 +2,7 @@ import gazu import os from avalon.api import AvalonMongoDB -from openpype.modules.default_modules.kitsu.utils.openpype import ( +from .openpype import ( create_op_asset, set_op_project, sync_project, diff --git a/openpype/modules/default_modules/kitsu/widgets.py b/openpype/modules/default_modules/kitsu/widgets.py deleted file mode 100644 index de232113fe..0000000000 --- a/openpype/modules/default_modules/kitsu/widgets.py +++ /dev/null @@ -1,31 +0,0 @@ -from Qt import QtWidgets - -from openpype.style import load_stylesheet - - -class MyExampleDialog(QtWidgets.QDialog): - def __init__(self, parent=None): - super(MyExampleDialog, self).__init__(parent) - - self.setWindowTitle("Connected modules") - - msg = "This is example dialog of Kitsu." - label_widget = QtWidgets.QLabel(msg, self) - - ok_btn = QtWidgets.QPushButton("OK", self) - btns_layout = QtWidgets.QHBoxLayout() - btns_layout.addStretch(1) - btns_layout.addWidget(ok_btn) - - layout = QtWidgets.QVBoxLayout(self) - layout.addWidget(label_widget) - layout.addLayout(btns_layout) - - ok_btn.clicked.connect(self._on_ok_clicked) - - self._label_widget = label_widget - - self.setStyleSheet(load_stylesheet()) - - def _on_ok_clicked(self): - self.done(1) diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index ddb2edc360..537e287366 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -139,9 +139,7 @@ }, "kitsu": { "enabled": false, - "server": "", - "script_login": "", - "script_pwd": "" + "server": "" }, "timers_manager": { "enabled": true, diff --git a/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json index ae2b52df0d..15a2ccc58d 100644 --- a/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json +++ b/openpype/settings/entities/schemas/system_schema/module_settings/schema_kitsu.json @@ -16,16 +16,6 @@ "key": "server", "label": "Server" }, - { - "type": "text", - "key": "script_login", - "label": "Script Login" - }, - { - "type": "text", - "key": "script_pwd", - "label": "Script Password" - }, { "type": "splitter" } diff --git a/pyproject.toml b/pyproject.toml index f32e385e80..93caa5ca70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ clique = "1.6.*" Click = "^7" dnspython = "^2.1.0" ftrack-python-api = "2.0.*" +gazu = "^0.8" google-api-python-client = "^1.12.8" # sync server google support (should be separate?) jsonschema = "^2.6.0" keyring = "^22.0.1" @@ -64,7 +65,7 @@ jinxed = [ python3-xlib = { version="*", markers = "sys_platform == 'linux'"} enlighten = "^1.9.0" slack-sdk = "^3.6.0" -requests = "2.25.1" +requests = "^2.25.1" pysftp = "^0.2.9" dropbox = "^11.20.0" From 6b3987708711e72d984b39096efec9a0421ffd0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 1 Mar 2022 18:11:42 +0100 Subject: [PATCH 25/82] Moved to modules --- openpype/modules/{default_modules => }/kitsu/__init__.py | 0 openpype/modules/{default_modules => }/kitsu/kitsu_module.py | 2 +- openpype/modules/{default_modules => }/kitsu/kitsu_widgets.py | 0 .../kitsu/plugins/publish/kitsu_plugin.py | 0 openpype/modules/{default_modules => }/kitsu/utils/__init__.py | 0 .../modules/{default_modules => }/kitsu/utils/listeners.py | 3 ++- openpype/modules/{default_modules => }/kitsu/utils/openpype.py | 0 7 files changed, 3 insertions(+), 2 deletions(-) rename openpype/modules/{default_modules => }/kitsu/__init__.py (100%) rename openpype/modules/{default_modules => }/kitsu/kitsu_module.py (99%) rename openpype/modules/{default_modules => }/kitsu/kitsu_widgets.py (100%) rename openpype/modules/{default_modules => }/kitsu/plugins/publish/kitsu_plugin.py (100%) rename openpype/modules/{default_modules => }/kitsu/utils/__init__.py (100%) rename openpype/modules/{default_modules => }/kitsu/utils/listeners.py (99%) rename openpype/modules/{default_modules => }/kitsu/utils/openpype.py (100%) diff --git a/openpype/modules/default_modules/kitsu/__init__.py b/openpype/modules/kitsu/__init__.py similarity index 100% rename from openpype/modules/default_modules/kitsu/__init__.py rename to openpype/modules/kitsu/__init__.py diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py similarity index 99% rename from openpype/modules/default_modules/kitsu/kitsu_module.py rename to openpype/modules/kitsu/kitsu_module.py index 51ab8aaa42..a17b509047 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -24,7 +24,7 @@ from pymongo import DeleteOne, UpdateOne from avalon.api import AvalonMongoDB from openpype.api import get_project_settings from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager -from openpype.modules.default_modules.kitsu.utils.openpype import ( +from openpype.modules.kitsu.utils.openpype import ( create_op_asset, sync_project, update_op_assets, diff --git a/openpype/modules/default_modules/kitsu/kitsu_widgets.py b/openpype/modules/kitsu/kitsu_widgets.py similarity index 100% rename from openpype/modules/default_modules/kitsu/kitsu_widgets.py rename to openpype/modules/kitsu/kitsu_widgets.py diff --git a/openpype/modules/default_modules/kitsu/plugins/publish/kitsu_plugin.py b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py similarity index 100% rename from openpype/modules/default_modules/kitsu/plugins/publish/kitsu_plugin.py rename to openpype/modules/kitsu/plugins/publish/kitsu_plugin.py diff --git a/openpype/modules/default_modules/kitsu/utils/__init__.py b/openpype/modules/kitsu/utils/__init__.py similarity index 100% rename from openpype/modules/default_modules/kitsu/utils/__init__.py rename to openpype/modules/kitsu/utils/__init__.py diff --git a/openpype/modules/default_modules/kitsu/utils/listeners.py b/openpype/modules/kitsu/utils/listeners.py similarity index 99% rename from openpype/modules/default_modules/kitsu/utils/listeners.py rename to openpype/modules/kitsu/utils/listeners.py index c99870fa36..961aa1691b 100644 --- a/openpype/modules/default_modules/kitsu/utils/listeners.py +++ b/openpype/modules/kitsu/utils/listeners.py @@ -10,7 +10,8 @@ from .openpype import ( ) -def add_listeners(): +def start_listeners(): + """Start listeners to keep OpenPype up-to-date with Kitsu.""" # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) diff --git a/openpype/modules/default_modules/kitsu/utils/openpype.py b/openpype/modules/kitsu/utils/openpype.py similarity index 100% rename from openpype/modules/default_modules/kitsu/utils/openpype.py rename to openpype/modules/kitsu/utils/openpype.py From a33b3d3b5cdfa37204da8570d175997be3544001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:30:33 +0100 Subject: [PATCH 26/82] Remove AddonSettingsDef --- openpype/modules/kitsu/__init__.py | 4 ++-- openpype/modules/kitsu/kitsu_module.py | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/openpype/modules/kitsu/__init__.py b/openpype/modules/kitsu/__init__.py index dc7c2dad50..6cb62bbb15 100644 --- a/openpype/modules/kitsu/__init__.py +++ b/openpype/modules/kitsu/__init__.py @@ -4,6 +4,6 @@ If addon class or settings definition won't be here their definition won't be found by OpenPype discovery. """ -from .kitsu_module import AddonSettingsDef, KitsuModule +from .kitsu_module import KitsuModule -__all__ = ("AddonSettingsDef", "KitsuModule") +__all__ = "KitsuModule" diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index a17b509047..9efcac9714 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -34,18 +34,6 @@ from openpype_interfaces import IPluginPaths, ITrayAction from .utils.listeners import start_listeners -class AddonSettingsDef(JsonFilesSettingsDef): - schema_prefix = "kitsu" - - def get_settings_root_path(self): - """Implemented abstract class of JsonFilesSettingsDef. - - Return directory path where json files defying addon settings are - located. - """ - return os.path.join(os.path.dirname(os.path.abspath(__file__)), "settings") - - class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): """Kitsu module class.""" From 4907ce1362602efef2692253d9b93fbc79d2e522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:31:02 +0100 Subject: [PATCH 27/82] Moved up module 'base' changes --- openpype/modules/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 0dd512ee8b..d77189be6c 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -860,6 +860,7 @@ class TrayModulesManager(ModulesManager): modules_menu_order = ( "user", "ftrack", + "kitsu", "muster", "launcher_tool", "avalon", From a15c4d2895a5eaf433ec9d3912200a271cac16a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:48:39 +0100 Subject: [PATCH 28/82] import gazu only at start --- openpype/modules/kitsu/kitsu_module.py | 63 ++++++++++------------- openpype/modules/kitsu/utils/listeners.py | 3 +- openpype/modules/kitsu/utils/openpype.py | 13 ++--- 3 files changed, 35 insertions(+), 44 deletions(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index 9efcac9714..502ed8ff96 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -4,34 +4,19 @@ import click import os import re -import gazu -from gazu.asset import all_assets_for_project, all_asset_types, new_asset -from gazu.shot import ( - all_episodes_for_project, - all_sequences_for_project, - all_shots_for_project, - new_episode, - new_sequence, - new_shot, -) -from gazu.task import ( - all_task_types, - new_task, - new_task_type, -) from pymongo import DeleteOne, UpdateOne from avalon.api import AvalonMongoDB from openpype.api import get_project_settings -from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager -from openpype.modules.kitsu.utils.openpype import ( +from openpype.modules import OpenPypeModule, ModulesManager +from openpype.settings.lib import get_local_settings +from openpype_interfaces import IPluginPaths, ITrayAction +from .utils.listeners import start_listeners +from .utils.openpype import ( create_op_asset, sync_project, update_op_assets, ) -from openpype.settings.lib import get_local_settings -from openpype_interfaces import IPluginPaths, ITrayAction -from .utils.listeners import start_listeners class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): @@ -137,6 +122,7 @@ def cli_main(): @cli_main.command() def sync_zou(): """Synchronize Zou server database (Kitsu backend) with openpype database.""" + import gazu # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) @@ -178,11 +164,11 @@ def sync_zou(): gazu.project.update_project_data(zou_project, data=op_project["data"]) gazu.project.update_project(zou_project) - asset_types = all_asset_types() - all_assets = all_assets_for_project(zou_project) - all_episodes = all_episodes_for_project(zou_project) - all_seqs = all_sequences_for_project(zou_project) - all_shots = all_shots_for_project(zou_project) + asset_types = gazu.asset.all_asset_types() + all_assets = gazu.asset.all_assets_for_project(zou_project) + all_episodes = gazu.shot.all_episodes_for_project(zou_project) + all_seqs = gazu.shot.all_sequences_for_project(zou_project) + all_shots = gazu.shot.all_shots_for_project(zou_project) all_entities_ids = { e["id"] for e in all_episodes + all_seqs + all_shots + all_assets } @@ -217,7 +203,9 @@ def sync_zou(): # Match asset type by it's name match = regex_ep.match(doc["name"]) if not match: # Asset - new_entity = new_asset(zou_project, asset_types[0], doc["name"]) + new_entity = gazu.asset.new_asset( + zou_project, asset_types[0], doc["name"] + ) # Match case in shot Collection: :param dbcon: Connection to DB. :param project_id: Project zou ID """ + import gazu + project = gazu.project.get_project(project_id) project_name = project["name"] dbcon.Session["AVALON_PROJECT"] = project_name @@ -49,6 +45,11 @@ def update_op_assets( :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] :return: List of (doc_id, update_dict) tuples """ + from gazu.task import ( + all_tasks_for_asset, + all_tasks_for_shot, + ) + assets_with_update = [] for item in entities_list: # Update asset From 44100b0da7d30bc34af0635b5e577e2a6df03c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:52:26 +0100 Subject: [PATCH 29/82] line length to 79 --- openpype/modules/kitsu/kitsu_module.py | 33 ++++++++++++++----- openpype/modules/kitsu/kitsu_widgets.py | 14 +++++--- .../kitsu/plugins/publish/kitsu_plugin.py | 4 ++- openpype/modules/kitsu/utils/listeners.py | 29 +++++++++++----- openpype/modules/kitsu/utils/openpype.py | 7 ++-- 5 files changed, 63 insertions(+), 24 deletions(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index 502ed8ff96..a7b3b17eb5 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -42,7 +42,9 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): # Check for "/api" url validity if not kitsu_url.endswith("api"): - kitsu_url = f"{kitsu_url}{'' if kitsu_url.endswith('/') else '/'}api" + kitsu_url = ( + f"{kitsu_url}{'' if kitsu_url.endswith('/') else '/'}api" + ) self.server_url = kitsu_url @@ -161,7 +163,9 @@ def sync_zou(): "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", } ) - gazu.project.update_project_data(zou_project, data=op_project["data"]) + gazu.project.update_project_data( + zou_project, data=op_project["data"] + ) gazu.project.update_project(zou_project) asset_types = gazu.asset.all_asset_types() @@ -227,7 +231,9 @@ def sync_zou(): # Create new sequence and set it as substitute created_sequence = gazu.shot.new_sequence( - zou_project, substitute_sequence_name, episode=zou_parent_id + zou_project, + substitute_sequence_name, + episode=zou_parent_id, ) gazu.shot.update_sequence_data( created_sequence, {"is_substitute": True} @@ -244,13 +250,16 @@ def sync_zou(): doc["name"], frame_in=doc["data"]["frameStart"], frame_out=doc["data"]["frameEnd"], - nb_frames=doc["data"]["frameEnd"] - doc["data"]["frameStart"], + nb_frames=doc["data"]["frameEnd"] + - doc["data"]["frameStart"], ) elif match.group(2): # Sequence parent_doc = asset_docs[visual_parent_id] new_entity = gazu.shot.new_sequence( - zou_project, doc["name"], episode=parent_doc["data"]["zou"]["id"] + zou_project, + doc["name"], + episode=parent_doc["data"]["zou"]["id"], ) elif match.group(3): # Episode @@ -293,7 +302,10 @@ def sync_zou(): if frame_in or frame_out: entity_data.update( { - "data": {"frame_in": frame_in, "frame_out": frame_out}, + "data": { + "frame_in": frame_in, + "frame_out": frame_out, + }, "nb_frames": frame_out - frame_in, } ) @@ -334,7 +346,10 @@ def sync_zou(): @cli_main.command() @click.option( - "-l", "--listen", is_flag=True, help="Listen Kitsu server after synchronization." + "-l", + "--listen", + is_flag=True, + help="Listen Kitsu server after synchronization.", ) def sync_openpype(listen: bool): """Synchronize openpype database from Zou sever database.""" @@ -410,7 +425,9 @@ def sync_openpype(listen: bool): bulk_writes.extend( [ UpdateOne({"_id": id}, update) - for id, update in update_op_assets(all_entities, zou_ids_and_asset_docs) + for id, update in update_op_assets( + all_entities, zou_ids_and_asset_docs + ) ] ) diff --git a/openpype/modules/kitsu/kitsu_widgets.py b/openpype/modules/kitsu/kitsu_widgets.py index 6bd436f460..1a32182795 100644 --- a/openpype/modules/kitsu/kitsu_widgets.py +++ b/openpype/modules/kitsu/kitsu_widgets.py @@ -46,7 +46,8 @@ class PasswordDialog(QtWidgets.QDialog): login_label = QtWidgets.QLabel("Login:", login_widget) login_input = QtWidgets.QLineEdit( - login_widget, text=kitsu_settings.get("login") if remembered else None + login_widget, + text=kitsu_settings.get("login") if remembered else None, ) login_input.setPlaceholderText("Your Kitsu account login...") @@ -61,7 +62,8 @@ class PasswordDialog(QtWidgets.QDialog): password_label = QtWidgets.QLabel("Password:", password_widget) password_input = QtWidgets.QLineEdit( - password_widget, text=kitsu_settings.get("password") if remembered else None + password_widget, + text=kitsu_settings.get("password") if remembered else None, ) password_input.setPlaceholderText("Your password...") password_input.setEchoMode(QtWidgets.QLineEdit.Password) @@ -87,7 +89,9 @@ class PasswordDialog(QtWidgets.QDialog): remember_checkbox = QtWidgets.QCheckBox("Remember", buttons_widget) remember_checkbox.setObjectName("RememberCheckbox") - remember_checkbox.setChecked(remembered if remembered is not None else True) + remember_checkbox.setChecked( + remembered if remembered is not None else True + ) ok_btn = QtWidgets.QPushButton("Ok", buttons_widget) cancel_btn = QtWidgets.QPushButton("Cancel", buttons_widget) @@ -137,7 +141,9 @@ class PasswordDialog(QtWidgets.QDialog): def _on_ok_click(self): # Check if is connectable if not self._connectable: - self.message_label.setText("Please set server url in Studio Settings!") + self.message_label.setText( + "Please set server url in Studio Settings!" + ) return # Collect values diff --git a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py index 86ba40a5f4..24f1e4e80c 100644 --- a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py +++ b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py @@ -31,7 +31,9 @@ class IntegrateRig(pyblish.api.InstancePlugin): # Get task task_type = gazu.task.get_task_type_by_name(instance.data["task"]) - entity_task = gazu.task.get_task_by_entity(asset_data["zou"], task_type) + entity_task = gazu.task.get_task_by_entity( + asset_data["zou"], task_type + ) # Comment entity gazu.task.add_comment( diff --git a/openpype/modules/kitsu/utils/listeners.py b/openpype/modules/kitsu/utils/listeners.py index 18f67b13e3..3768b4e8e6 100644 --- a/openpype/modules/kitsu/utils/listeners.py +++ b/openpype/modules/kitsu/utils/listeners.py @@ -95,9 +95,9 @@ def start_listeners(): zou_ids_and_asset_docs[asset["project_id"]] = project_doc # Update - asset_doc_id, asset_update = update_op_assets([asset], zou_ids_and_asset_docs)[ - 0 - ] + asset_doc_id, asset_update = update_op_assets( + [asset], zou_ids_and_asset_docs + )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) def delete_asset(data): @@ -105,7 +105,9 @@ def start_listeners(): project_col = set_op_project(dbcon, data["project_id"]) # Delete - project_col.delete_one({"type": "asset", "data.zou.id": data["asset_id"]}) + project_col.delete_one( + {"type": "asset", "data.zou.id": data["asset_id"]} + ) gazu.events.add_listener(event_client, "asset:new", new_asset) gazu.events.add_listener(event_client, "asset:update", update_asset) @@ -155,7 +157,9 @@ def start_listeners(): print("delete episode") # TODO check bugfix # Delete - project_col.delete_one({"type": "asset", "data.zou.id": data["episode_id"]}) + project_col.delete_one( + {"type": "asset", "data.zou.id": data["episode_id"]} + ) gazu.events.add_listener(event_client, "episode:new", new_episode) gazu.events.add_listener(event_client, "episode:update", update_episode) @@ -205,7 +209,9 @@ def start_listeners(): print("delete sequence") # TODO check bugfix # Delete - project_col.delete_one({"type": "asset", "data.zou.id": data["sequence_id"]}) + project_col.delete_one( + {"type": "asset", "data.zou.id": data["sequence_id"]} + ) gazu.events.add_listener(event_client, "sequence:new", new_sequence) gazu.events.add_listener(event_client, "sequence:update", update_sequence) @@ -244,7 +250,9 @@ def start_listeners(): zou_ids_and_asset_docs[shot["project_id"]] = project_doc # Update - asset_doc_id, asset_update = update_op_assets([shot], zou_ids_and_asset_docs)[0] + asset_doc_id, asset_update = update_op_assets( + [shot], zou_ids_and_asset_docs + )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) def delete_shot(data): @@ -252,7 +260,9 @@ def start_listeners(): project_col = set_op_project(dbcon, data["project_id"]) # Delete - project_col.delete_one({"type": "asset", "data.zou.id": data["shot_id"]}) + project_col.delete_one( + {"type": "asset", "data.zou.id": data["shot_id"]} + ) gazu.events.add_listener(event_client, "shot:new", new_shot) gazu.events.add_listener(event_client, "shot:update", update_shot) @@ -302,7 +312,8 @@ def start_listeners(): # Delete task in DB project_col.update_one( - {"_id": doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} + {"_id": doc["_id"]}, + {"$set": {"data.tasks": asset_tasks}}, ) return diff --git a/openpype/modules/kitsu/utils/openpype.py b/openpype/modules/kitsu/utils/openpype.py index 2443323893..8aabba6de0 100644 --- a/openpype/modules/kitsu/utils/openpype.py +++ b/openpype/modules/kitsu/utils/openpype.py @@ -65,7 +65,8 @@ def update_op_assets( tasks_list = all_tasks_for_shot(item) # TODO frame in and out item_data["tasks"] = { - t["task_type_name"]: {"type": t["task_type_name"]} for t in tasks_list + t["task_type_name"]: {"type": t["task_type_name"]} + for t in tasks_list } # Get zou parent id for correct hierarchy @@ -79,7 +80,9 @@ def update_op_assets( parent_zou_id = substitute_parent_item["parent_id"] else: parent_zou_id = ( - item.get("parent_id") or item.get("episode_id") or item.get("source_id") + item.get("parent_id") + or item.get("episode_id") + or item.get("source_id") ) # TODO check consistency # Visual parent for hierarchy From c26c2f09a8f271419e76cd7407f35b19d579d851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:54:47 +0100 Subject: [PATCH 30/82] fix import --- openpype/modules/kitsu/utils/openpype.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/modules/kitsu/utils/openpype.py b/openpype/modules/kitsu/utils/openpype.py index 8aabba6de0..56c99effff 100644 --- a/openpype/modules/kitsu/utils/openpype.py +++ b/openpype/modules/kitsu/utils/openpype.py @@ -132,6 +132,8 @@ def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: :param dbcon: DB to create project in :return: Update instance for the project """ + import gazu + project_name = project["name"] project_doc = dbcon.find_one({"type": "project"}) if not project_doc: From e8831947e6c2de902470e0fffdb69cad6429ce32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 09:57:00 +0100 Subject: [PATCH 31/82] Fix __all__ --- openpype/modules/kitsu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/__init__.py b/openpype/modules/kitsu/__init__.py index 6cb62bbb15..9737a054f6 100644 --- a/openpype/modules/kitsu/__init__.py +++ b/openpype/modules/kitsu/__init__.py @@ -6,4 +6,4 @@ be found by OpenPype discovery. from .kitsu_module import KitsuModule -__all__ = "KitsuModule" +__all__ = ("KitsuModule", ) From cde925c09b06130f6b4e1f36d993d21aba0fd7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 11:38:47 +0100 Subject: [PATCH 32/82] Use OpenPypeSecureRegistry for authentication --- openpype/modules/kitsu/__init__.py | 2 +- openpype/modules/kitsu/kitsu_module.py | 22 ++++++++++++++--- openpype/modules/kitsu/kitsu_widgets.py | 33 +++++++++++-------------- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/openpype/modules/kitsu/__init__.py b/openpype/modules/kitsu/__init__.py index 9737a054f6..9220cb1762 100644 --- a/openpype/modules/kitsu/__init__.py +++ b/openpype/modules/kitsu/__init__.py @@ -6,4 +6,4 @@ be found by OpenPype discovery. from .kitsu_module import KitsuModule -__all__ = ("KitsuModule", ) +__all__ = ("KitsuModule",) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index a7b3b17eb5..ebfa0dbeea 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -8,6 +8,7 @@ from pymongo import DeleteOne, UpdateOne from avalon.api import AvalonMongoDB from openpype.api import get_project_settings +from openpype.lib.local_settings import OpenPypeSecureRegistry from openpype.modules import OpenPypeModule, ModulesManager from openpype.settings.lib import get_local_settings from openpype_interfaces import IPluginPaths, ITrayAction @@ -28,7 +29,9 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): def initialize(self, settings): """Initialization of module.""" module_settings = settings[self.name] - local_kitsu_settings = get_local_settings().get("kitsu", {}) + + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") # Enabled by settings self.enabled = module_settings.get("enabled", False) @@ -49,8 +52,8 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): self.server_url = kitsu_url # Set credentials - self.kitsu_login = local_kitsu_settings["login"] - self.kitsu_password = local_kitsu_settings["password"] + self.kitsu_login = user_registry.get_item("login", None) + self.kitsu_password = user_registry.get_item("password", None) # Prepare variables that can be used or set afterwards self._connected_modules = None @@ -359,7 +362,13 @@ def sync_openpype(listen: bool): gazu.client.set_host(os.environ["KITSU_SERVER"]) # Authenticate - gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + kitsu_login = os.environ.get("KITSU_LOGIN") + kitsu_pwd = os.environ.get("KITSU_PWD") + if not kitsu_login or not kitsu_pwd: # Sentinel to log-in + log_in_dialog() + return + + gazu.log_in(kitsu_login, kitsu_pwd) # Iterate projects dbcon = AvalonMongoDB() @@ -462,6 +471,11 @@ def listen(): @cli_main.command() def sign_in(): + """Sign-in command.""" + log_in_dialog() + + +def log_in_dialog(): """Show credentials dialog.""" from openpype.tools.utils.lib import qt_app_context diff --git a/openpype/modules/kitsu/kitsu_widgets.py b/openpype/modules/kitsu/kitsu_widgets.py index 1a32182795..1a48e6dbc0 100644 --- a/openpype/modules/kitsu/kitsu_widgets.py +++ b/openpype/modules/kitsu/kitsu_widgets.py @@ -4,11 +4,10 @@ import gazu from Qt import QtWidgets, QtCore, QtGui from openpype import style +from openpype.lib.local_settings import OpenPypeSecureRegistry from openpype.resources import get_resource from openpype.settings.lib import ( - get_local_settings, get_system_settings, - save_local_settings, ) from openpype.widgets.password_dialog import PressHoverButton @@ -26,8 +25,11 @@ class PasswordDialog(QtWidgets.QDialog): self.resize(300, 120) system_settings = get_system_settings() - kitsu_settings = get_local_settings().get("kitsu", {}) - remembered = kitsu_settings.get("remember") + user_registry = OpenPypeSecureRegistry("kitsu_user") + remembered = bool( + user_registry.get_item("login", None) + or user_registry.get_item("password", None) + ) self._final_result = None self._connectable = bool( @@ -47,7 +49,7 @@ class PasswordDialog(QtWidgets.QDialog): login_input = QtWidgets.QLineEdit( login_widget, - text=kitsu_settings.get("login") if remembered else None, + text=user_registry.get_item("login") if remembered else None, ) login_input.setPlaceholderText("Your Kitsu account login...") @@ -63,7 +65,7 @@ class PasswordDialog(QtWidgets.QDialog): password_input = QtWidgets.QLineEdit( password_widget, - text=kitsu_settings.get("password") if remembered else None, + text=user_registry.get_item("password") if remembered else None, ) password_input.setPlaceholderText("Your password...") password_input.setEchoMode(QtWidgets.QLineEdit.Password) @@ -161,30 +163,23 @@ class PasswordDialog(QtWidgets.QDialog): os.environ["KITSU_LOGIN"] = login_value os.environ["KITSU_PWD"] = pwd_value - # Get settings - local_settings = get_local_settings() - local_settings.setdefault("kitsu", {}) + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") # Remember password cases if remember: # Set local settings - local_settings["kitsu"]["login"] = login_value - local_settings["kitsu"]["password"] = pwd_value + user_registry.set_item("login", login_value) + user_registry.set_item("password", pwd_value) else: # Clear local settings - local_settings["kitsu"]["login"] = None - local_settings["kitsu"]["password"] = None + user_registry.delete_item("login") + user_registry.delete_item("password") # Clear input fields self.login_input.clear() self.password_input.clear() - # Keep 'remember' parameter - local_settings["kitsu"]["remember"] = remember - - # Save settings - save_local_settings(local_settings) - self._final_result = True self.close() From c9ea0b3bb262484b43ebd5c1c5d649dcdaf75d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 2 Mar 2022 11:49:49 +0100 Subject: [PATCH 33/82] Line length max 79 --- openpype/modules/kitsu/kitsu_module.py | 28 +++++++++++++++++-------- openpype/modules/kitsu/kitsu_widgets.py | 7 ++++++- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index ebfa0dbeea..6a2e517832 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -10,7 +10,6 @@ from avalon.api import AvalonMongoDB from openpype.api import get_project_settings from openpype.lib.local_settings import OpenPypeSecureRegistry from openpype.modules import OpenPypeModule, ModulesManager -from openpype.settings.lib import get_local_settings from openpype_interfaces import IPluginPaths, ITrayAction from .utils.listeners import start_listeners from .utils.openpype import ( @@ -126,7 +125,7 @@ def cli_main(): @cli_main.command() def sync_zou(): - """Synchronize Zou server database (Kitsu backend) with openpype database.""" + """Synchronize Zou database (Kitsu backend) with openpype database.""" import gazu # Connect to server @@ -154,7 +153,9 @@ def sync_zou(): # Create project if zou_project is None: raise RuntimeError( - f"Project '{project_name}' doesn't exist in Zou database, please create it in Kitsu and add OpenPype user to it before running synchronization." + f"Project '{project_name}' doesn't exist in Zou database, " + "please create it in Kitsu and add OpenPype user to it before " + "running synchronization." ) # Update project settings and data @@ -163,7 +164,8 @@ def sync_zou(): { "code": op_project["data"]["code"], "fps": op_project["data"]["fps"], - "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", + "resolution": f"{op_project['data']['resolutionWidth']}" + f"x{op_project['data']['resolutionHeight']}", } ) gazu.project.update_project_data( @@ -213,7 +215,8 @@ def sync_zou(): new_entity = gazu.asset.new_asset( zou_project, asset_types[0], doc["name"] ) - # Match case in shot Date: Wed, 2 Mar 2022 12:11:05 +0100 Subject: [PATCH 34/82] Rename module to Kitsu Connect --- openpype/modules/kitsu/kitsu_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index 6a2e517832..d5e744ceb5 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -22,7 +22,7 @@ from .utils.openpype import ( class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): """Kitsu module class.""" - label = "Kitsu" + label = "Kitsu Connect" name = "kitsu" def initialize(self, settings): From c39bdee4330b1578cfe821d0506dbc7b59373621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 3 Mar 2022 15:50:35 +0100 Subject: [PATCH 35/82] Refactor for login system following recommendations. --- openpype/modules/kitsu/kitsu_module.py | 457 ++---------------- openpype/modules/kitsu/kitsu_widgets.py | 59 +-- .../kitsu/plugins/publish/kitsu_plugin.py | 2 +- openpype/modules/kitsu/utils/credentials.py | 92 ++++ .../utils/{listeners.py => sync_service.py} | 238 +++++---- .../{openpype.py => update_op_with_zou.py} | 153 +++++- .../modules/kitsu/utils/update_zou_with_op.py | 262 ++++++++++ 7 files changed, 717 insertions(+), 546 deletions(-) create mode 100644 openpype/modules/kitsu/utils/credentials.py rename openpype/modules/kitsu/utils/{listeners.py => sync_service.py} (54%) rename openpype/modules/kitsu/utils/{openpype.py => update_op_with_zou.py} (52%) create mode 100644 openpype/modules/kitsu/utils/update_zou_with_op.py diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index d5e744ceb5..dca6133e88 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -2,21 +2,9 @@ import click import os -import re -from pymongo import DeleteOne, UpdateOne - -from avalon.api import AvalonMongoDB -from openpype.api import get_project_settings -from openpype.lib.local_settings import OpenPypeSecureRegistry -from openpype.modules import OpenPypeModule, ModulesManager +from openpype.modules import OpenPypeModule from openpype_interfaces import IPluginPaths, ITrayAction -from .utils.listeners import start_listeners -from .utils.openpype import ( - create_op_asset, - sync_project, - update_op_assets, -) class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): @@ -29,9 +17,6 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): """Initialization of module.""" module_settings = settings[self.name] - # Get user registry - user_registry = OpenPypeSecureRegistry("kitsu_user") - # Enabled by settings self.enabled = module_settings.get("enabled", False) @@ -44,66 +29,58 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): # Check for "/api" url validity if not kitsu_url.endswith("api"): - kitsu_url = ( - f"{kitsu_url}{'' if kitsu_url.endswith('/') else '/'}api" + kitsu_url = "{}{}api".format( + kitsu_url, "" if kitsu_url.endswith("/") else "/" ) self.server_url = kitsu_url - # Set credentials - self.kitsu_login = user_registry.get_item("login", None) - self.kitsu_password = user_registry.get_item("password", None) - - # Prepare variables that can be used or set afterwards - self._connected_modules = None # UI which must not be created at this time self._dialog = None def tray_init(self): - """Implementation of abstract method for `ITrayAction`. - - We're definitely in tray tool so we can pre create dialog. - """ + """Tray init.""" self._create_dialog() + def tray_start(self): + """Tray start.""" + from .utils.credentials import ( + load_credentials, + validate_credentials, + set_credentials_envs, + ) + + username, password = load_credentials() + + # Check credentials, ask them if needed + if validate_credentials(username, password): + set_credentials_envs(username, password) + else: + self.show_dialog() + def get_global_environments(self): """Kitsu's global environments.""" - return { - "KITSU_SERVER": self.server_url, - "KITSU_LOGIN": self.kitsu_login, - "KITSU_PWD": self.kitsu_password, - } + return {"KITSU_SERVER": self.server_url} def _create_dialog(self): # Don't recreate dialog if already exists if self._dialog is not None: return - from .kitsu_widgets import PasswordDialog + from .kitsu_widgets import KitsuPasswordDialog - self._dialog = PasswordDialog() + self._dialog = KitsuPasswordDialog() def show_dialog(self): - """Show dialog with connected modules. + """Show dialog to log-in.""" - This can be called from anywhere but can also crash in headless mode. - There is no way to prevent addon to do invalid operations if he's - not handling them. - """ # Make sure dialog is created self._create_dialog() + # Show dialog self._dialog.open() - def get_connected_modules(self): - """Custom implementation of addon.""" - names = set() - if self._connected_modules is not None: - for module in self._connected_modules: - names.add(module.name) - return names - def on_action_trigger(self): """Implementation of abstract method for `ITrayAction`.""" self.show_dialog() @@ -124,372 +101,36 @@ def cli_main(): @cli_main.command() -def sync_zou(): - """Synchronize Zou database (Kitsu backend) with openpype database.""" - import gazu - - # Connect to server - gazu.client.set_host(os.environ["KITSU_SERVER"]) - - # Authenticate - gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) - - # Iterate projects - dbcon = AvalonMongoDB() - dbcon.install() - - op_projects = [p for p in dbcon.projects()] - bulk_writes = [] - for op_project in op_projects: - # Create project locally - # Try to find project document - project_name = op_project["name"] - dbcon.Session["AVALON_PROJECT"] = project_name - - # Get all entities from zou - print(f"Synchronizing {project_name}...") - zou_project = gazu.project.get_project_by_name(project_name) - - # Create project - if zou_project is None: - raise RuntimeError( - f"Project '{project_name}' doesn't exist in Zou database, " - "please create it in Kitsu and add OpenPype user to it before " - "running synchronization." - ) - - # Update project settings and data - if op_project["data"]: - zou_project.update( - { - "code": op_project["data"]["code"], - "fps": op_project["data"]["fps"], - "resolution": f"{op_project['data']['resolutionWidth']}" - f"x{op_project['data']['resolutionHeight']}", - } - ) - gazu.project.update_project_data( - zou_project, data=op_project["data"] - ) - gazu.project.update_project(zou_project) - - asset_types = gazu.asset.all_asset_types() - all_assets = gazu.asset.all_assets_for_project(zou_project) - all_episodes = gazu.shot.all_episodes_for_project(zou_project) - all_seqs = gazu.shot.all_sequences_for_project(zou_project) - all_shots = gazu.shot.all_shots_for_project(zou_project) - all_entities_ids = { - e["id"] for e in all_episodes + all_seqs + all_shots + all_assets - } - - # Query all assets of the local project - project_module_settings = get_project_settings(project_name)["kitsu"] - project_col = dbcon.database[project_name] - asset_docs = { - asset_doc["_id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) - } - - # Create new assets - new_assets_docs = [ - doc - for doc in asset_docs.values() - if doc["data"].get("zou", {}).get("id") not in all_entities_ids - ] - naming_pattern = project_module_settings["entities_naming_pattern"] - regex_ep = re.compile( - r"(.*{}.*)|(.*{}.*)|(.*{}.*)".format( - naming_pattern["shot"].replace("#", ""), - naming_pattern["sequence"].replace("#", ""), - naming_pattern["episode"].replace("#", ""), - ), - re.IGNORECASE, - ) - for doc in new_assets_docs: - visual_parent_id = doc["data"]["visualParent"] - parent_substitutes = [] - - # Match asset type by it's name - match = regex_ep.match(doc["name"]) - if not match: # Asset - new_entity = gazu.asset.new_asset( - zou_project, asset_types[0], doc["name"] - ) - # Match case in shot bool: + """Validate credentials by trying to connect to Kitsu host URL. + + :param login: Kitsu Login + :param password: Kitsu Password + :param kitsu_url: Kitsu host URL + :return: Are credentials valid? + """ + # Connect to server + validate_host(kitsu_url) + + # Authenticate + try: + gazu.log_in(login, password) + except gazu.exception.AuthFailedException: + return False + + return True + + +def validate_host(kitsu_url: str) -> bool: + """Validate credentials by trying to connect to Kitsu host URL. + + :param kitsu_url: Kitsu host URL + :return: Is host valid? + """ + # Connect to server + gazu.set_host(kitsu_url) + + # Test host + if gazu.client.host_is_valid(): + return True + else: + raise gazu.exception.HostException(f"Host '{kitsu_url}' is invalid.") + + +def clear_credentials(): + """Clear credentials in Secure Registry.""" + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + # Set local settings + user_registry.delete_item("login") + user_registry.delete_item("password") + + +def save_credentials(login: str, password: str): + """Save credentials in Secure Registry. + + :param login: Kitsu Login + :param password: Kitsu Password + """ + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + # Set local settings + user_registry.set_item("login", login) + user_registry.set_item("password", password) + + +def load_credentials() -> Tuple[str, str]: + """Load registered credentials. + + :return: Login, Password + """ + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + return user_registry.get_item("login", None), user_registry.get_item( + "password", None + ) + + +def set_credentials_envs(login: str, password: str): + """Set environment variables with Kitsu login and password. + + :param login: Kitsu Login + :param password: Kitsu Password + """ + os.environ["KITSU_LOGIN"] = login + os.environ["KITSU_PWD"] = password diff --git a/openpype/modules/kitsu/utils/listeners.py b/openpype/modules/kitsu/utils/sync_service.py similarity index 54% rename from openpype/modules/kitsu/utils/listeners.py rename to openpype/modules/kitsu/utils/sync_service.py index 3768b4e8e6..831673ec0d 100644 --- a/openpype/modules/kitsu/utils/listeners.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -1,72 +1,143 @@ import os +import gazu + from avalon.api import AvalonMongoDB -from .openpype import ( +from .credentials import load_credentials, validate_credentials +from .update_op_with_zou import ( create_op_asset, set_op_project, - sync_project, + write_project_to_op, update_op_assets, ) -def start_listeners(): - """Start listeners to keep OpenPype up-to-date with Kitsu.""" - import gazu +class Listener: + """Host Kitsu listener.""" - # Connect to server - gazu.client.set_host(os.environ["KITSU_SERVER"]) + def __init__(self, login, password): + """Create client and add listeners to events without starting it. - # Authenticate - gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) - gazu.set_event_host(os.environ["KITSU_SERVER"].replace("api", "socket.io")) - event_client = gazu.events.init() + Run `listener.start()` to actually start the service. - # Connect to DB - dbcon = AvalonMongoDB() - dbcon.install() + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + AuthFailedException: Wrong user login and/or password + """ + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + gazu.set_event_host( + os.environ["KITSU_SERVER"].replace("api", "socket.io") + ) + self.event_client = gazu.events.init() + + gazu.events.add_listener( + self.event_client, "project:new", self._new_project + ) + gazu.events.add_listener( + self.event_client, "project:update", self._update_project + ) + gazu.events.add_listener( + self.event_client, "project:delete", self._delete_project + ) + + gazu.events.add_listener( + self.event_client, "asset:new", self._new_asset + ) + gazu.events.add_listener( + self.event_client, "asset:update", self._update_asset + ) + gazu.events.add_listener( + self.event_client, "asset:delete", self._delete_asset + ) + + gazu.events.add_listener( + self.event_client, "episode:new", self._new_episode + ) + gazu.events.add_listener( + self.event_client, "episode:update", self._update_episode + ) + gazu.events.add_listener( + self.event_client, "episode:delete", self._delete_episode + ) + + gazu.events.add_listener( + self.event_client, "sequence:new", self._new_sequence + ) + gazu.events.add_listener( + self.event_client, "sequence:update", self._update_sequence + ) + gazu.events.add_listener( + self.event_client, "sequence:delete", self._delete_sequence + ) + + gazu.events.add_listener(self.event_client, "shot:new", self._new_shot) + gazu.events.add_listener( + self.event_client, "shot:update", self._update_shot + ) + gazu.events.add_listener( + self.event_client, "shot:delete", self._delete_shot + ) + + gazu.events.add_listener(self.event_client, "task:new", self._new_task) + gazu.events.add_listener( + self.event_client, "task:update", self._update_task + ) + gazu.events.add_listener( + self.event_client, "task:delete", self._delete_task + ) + + def start(self): + gazu.events.run_client(self.event_client) # == Project == - - def new_project(data): + def _new_project(self, data): """Create new project into OP DB.""" # Use update process to avoid duplicating code - update_project(data) + self._update_project(data) - def update_project(data): + def _update_project(self, data): """Update project into OP DB.""" # Get project entity project = gazu.project.get_project(data["project_id"]) project_name = project["name"] - dbcon.Session["AVALON_PROJECT"] = project_name - update_project = sync_project(project, dbcon) + update_project = write_project_to_op(project, self.dbcon) # Write into DB if update_project: - project_col = dbcon.database[project_name] + project_col = self.dbcon.database[project_name] project_col.bulk_write([update_project]) - def delete_project(data): + def _delete_project(self, data): """Delete project.""" # Get project entity print(data) # TODO check bugfix project = gazu.project.get_project(data["project_id"]) # Delete project collection - project_col = dbcon.database[project["name"]] + project_col = self.dbcon.database[project["name"]] project_col.drop() - gazu.events.add_listener(event_client, "project:new", new_project) - gazu.events.add_listener(event_client, "project:update", update_project) - gazu.events.add_listener(event_client, "project:delete", delete_project) - # == Asset == - def new_asset(data): + def _new_asset(self, data): """Create new asset into OP DB.""" # Get project entity - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Get gazu entity asset = gazu.asset.get_asset(data["asset_id"]) @@ -75,12 +146,12 @@ def start_listeners(): project_col.insert_one(create_op_asset(asset)) # Update - update_asset(data) + self._update_asset(data) - def update_asset(data): + def _update_asset(self, data): """Update asset into OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) - project_doc = dbcon.find_one({"type": "project"}) + project_col = set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity asset = gazu.asset.get_asset(data["asset_id"]) @@ -100,24 +171,20 @@ def start_listeners(): )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) - def delete_asset(data): + def _delete_asset(self, data): """Delete asset of OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Delete project_col.delete_one( {"type": "asset", "data.zou.id": data["asset_id"]} ) - gazu.events.add_listener(event_client, "asset:new", new_asset) - gazu.events.add_listener(event_client, "asset:update", update_asset) - gazu.events.add_listener(event_client, "asset:delete", delete_asset) - # == Episode == - def new_episode(data): + def _new_episode(self, data): """Create new episode into OP DB.""" # Get project entity - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Get gazu entity episode = gazu.shot.get_episode(data["episode_id"]) @@ -126,12 +193,12 @@ def start_listeners(): project_col.insert_one(create_op_asset(episode)) # Update - update_episode(data) + self._update_episode(data) - def update_episode(data): + def _update_episode(self, data): """Update episode into OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) - project_doc = dbcon.find_one({"type": "project"}) + project_col = set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity episode = gazu.shot.get_episode(data["episode_id"]) @@ -151,9 +218,9 @@ def start_listeners(): )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) - def delete_episode(data): + def _delete_episode(self, data): """Delete shot of OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) print("delete episode") # TODO check bugfix # Delete @@ -161,15 +228,11 @@ def start_listeners(): {"type": "asset", "data.zou.id": data["episode_id"]} ) - gazu.events.add_listener(event_client, "episode:new", new_episode) - gazu.events.add_listener(event_client, "episode:update", update_episode) - gazu.events.add_listener(event_client, "episode:delete", delete_episode) - # == Sequence == - def new_sequence(data): + def _new_sequence(self, data): """Create new sequnce into OP DB.""" # Get project entity - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Get gazu entity sequence = gazu.shot.get_sequence(data["sequence_id"]) @@ -178,12 +241,12 @@ def start_listeners(): project_col.insert_one(create_op_asset(sequence)) # Update - update_sequence(data) + self._update_sequence(data) - def update_sequence(data): + def _update_sequence(self, data): """Update sequence into OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) - project_doc = dbcon.find_one({"type": "project"}) + project_col = set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity sequence = gazu.shot.get_sequence(data["sequence_id"]) @@ -203,9 +266,9 @@ def start_listeners(): )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) - def delete_sequence(data): + def _delete_sequence(self, data): """Delete sequence of OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) print("delete sequence") # TODO check bugfix # Delete @@ -213,15 +276,11 @@ def start_listeners(): {"type": "asset", "data.zou.id": data["sequence_id"]} ) - gazu.events.add_listener(event_client, "sequence:new", new_sequence) - gazu.events.add_listener(event_client, "sequence:update", update_sequence) - gazu.events.add_listener(event_client, "sequence:delete", delete_sequence) - # == Shot == - def new_shot(data): + def _new_shot(self, data): """Create new shot into OP DB.""" # Get project entity - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Get gazu entity shot = gazu.shot.get_shot(data["shot_id"]) @@ -230,12 +289,12 @@ def start_listeners(): project_col.insert_one(create_op_asset(shot)) # Update - update_shot(data) + self._update_shot(data) - def update_shot(data): + def _update_shot(self, data): """Update shot into OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) - project_doc = dbcon.find_one({"type": "project"}) + project_col = set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity shot = gazu.shot.get_shot(data["shot_id"]) @@ -255,25 +314,20 @@ def start_listeners(): )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) - def delete_shot(data): + def _delete_shot(self, data): """Delete shot of OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Delete project_col.delete_one( {"type": "asset", "data.zou.id": data["shot_id"]} ) - gazu.events.add_listener(event_client, "shot:new", new_shot) - gazu.events.add_listener(event_client, "shot:update", update_shot) - gazu.events.add_listener(event_client, "shot:delete", delete_shot) - # == Task == - def new_task(data): + def _new_task(self, data): """Create new task into OP DB.""" - print("new", data) # Get project entity - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Get gazu entity task = gazu.task.get_task(data["task_id"]) @@ -291,14 +345,14 @@ def start_listeners(): {"_id": asset_doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} ) - def update_task(data): + def _update_task(self, data): """Update task into OP DB.""" # TODO is it necessary? pass - def delete_task(data): + def _delete_task(self, data): """Delete task of OP DB.""" - project_col = set_op_project(dbcon, data["project_id"]) + project_col = set_op_project(self.dbcon, data["project_id"]) # Find asset doc asset_docs = [doc for doc in project_col.find({"type": "asset"})] @@ -307,7 +361,7 @@ def start_listeners(): for name, task in doc["data"]["tasks"].items(): if task.get("zou") and data["task_id"] == task["zou"]["id"]: # Pop task - asset_tasks = doc["data"].get("tasks") + asset_tasks = doc["data"].get("tasks", {}) asset_tasks.pop(name) # Delete task in DB @@ -317,8 +371,20 @@ def start_listeners(): ) return - gazu.events.add_listener(event_client, "task:new", new_task) - gazu.events.add_listener(event_client, "task:update", update_task) - gazu.events.add_listener(event_client, "task:delete", delete_task) - gazu.events.run_client(event_client) +def start_listeners(login: str, password: str): + """Start listeners to keep OpenPype up-to-date with Kitsu. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + + # Connect to server + listener = Listener(login, password) + listener.start() + + +if __name__ == "__main__": + # TODO not sure when this can be run and if this system is reliable + start_listeners(load_credentials()) diff --git a/openpype/modules/kitsu/utils/openpype.py b/openpype/modules/kitsu/utils/update_op_with_zou.py similarity index 52% rename from openpype/modules/kitsu/utils/openpype.py rename to openpype/modules/kitsu/utils/update_op_with_zou.py index 56c99effff..eb675ad09e 100644 --- a/openpype/modules/kitsu/utils/openpype.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -1,10 +1,17 @@ +"""Functions to update OpenPype data using Kitsu DB (a.k.a Zou).""" from typing import Dict, List -from pymongo import UpdateOne +from pymongo import DeleteOne, UpdateOne from pymongo.collection import Collection +import gazu +from gazu.task import ( + all_tasks_for_asset, + all_tasks_for_shot, +) from avalon.api import AvalonMongoDB from openpype.lib import create_project +from openpype.modules.kitsu.utils.credentials import validate_credentials def create_op_asset(gazu_entity: dict) -> dict: @@ -26,8 +33,6 @@ def set_op_project(dbcon, project_id) -> Collection: :param dbcon: Connection to DB. :param project_id: Project zou ID """ - import gazu - project = gazu.project.get_project(project_id) project_name = project["name"] dbcon.Session["AVALON_PROJECT"] = project_name @@ -45,11 +50,6 @@ def update_op_assets( :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] :return: List of (doc_id, update_dict) tuples """ - from gazu.task import ( - all_tasks_for_asset, - all_tasks_for_shot, - ) - assets_with_update = [] for item in entities_list: # Update asset @@ -124,18 +124,19 @@ def update_op_assets( return assets_with_update -def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: - """Sync project with database. +def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: + """Write gazu project to OP database. Create project if doesn't exist. - :param project: Gazu project - :param dbcon: DB to create project in - :return: Update instance for the project - """ - import gazu + Args: + project (dict): Gazu project + dbcon (AvalonMongoDB): DB to create project in + Returns: + UpdateOne: Update instance for the project + """ project_name = project["name"] - project_doc = dbcon.find_one({"type": "project"}) + project_doc = dbcon.database[project_name].find_one({"type": "project"}) if not project_doc: print(f"Creating project '{project_name}'") project_doc = create_project(project_name, project_name, dbcon=dbcon) @@ -165,3 +166,123 @@ def sync_project(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: } }, ) + + +def sync_all_project(login: str, password: str): + """Update all OP projects in DB with Zou data. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + gazu.exception.AuthFailedException: Wrong user login and/or password + """ + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + all_projects = gazu.project.all_projects() + for project in all_projects: + sync_project_from_kitsu(project["name"], dbcon, project) + + +def sync_project_from_kitsu( + project_name: str, dbcon: AvalonMongoDB, project: dict = None +): + """Update OP project in DB with Zou data. + + Args: + project_name (str): Name of project to sync + dbcon (AvalonMongoDB): MongoDB connection + project (dict, optional): Project dict got using gazu. + Defaults to None. + """ + bulk_writes = [] + + # Get project from zou + if not project: + project = gazu.project.get_project_by_name(project_name) + project_code = project_name + + # Try to find project document + project_col = dbcon.database[project_code] + project_doc = project_col.find_one({"type": "project"}) + + print(f"Synchronizing {project_name}...") + + # Get all assets from zou + all_assets = gazu.asset.all_assets_for_project(project) + all_episodes = gazu.shot.all_episodes_for_project(project) + all_seqs = gazu.shot.all_sequences_for_project(project) + all_shots = gazu.shot.all_shots_for_project(project) + all_entities = [ + e + for e in all_assets + all_episodes + all_seqs + all_shots + if e["data"] and not e["data"].get("is_substitute") + ] + + # Sync project. Create if doesn't exist + bulk_writes.append(write_project_to_op(project, dbcon)) + + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[project["id"]] = project_doc + + # Create + to_insert = [] + to_insert.extend( + [ + create_op_asset(item) + for item in all_entities + if item["id"] not in zou_ids_and_asset_docs.keys() + ] + ) + if to_insert: + # Insert doc in DB + project_col.insert_many(to_insert) + + # Update existing docs + zou_ids_and_asset_docs.update( + { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou") + } + ) + + # Update + bulk_writes.extend( + [ + UpdateOne({"_id": id}, update) + for id, update in update_op_assets( + all_entities, zou_ids_and_asset_docs + ) + ] + ) + + # Delete + diff_assets = set(zou_ids_and_asset_docs.keys()) - { + e["id"] for e in all_entities + [project] + } + if diff_assets: + bulk_writes.extend( + [ + DeleteOne(zou_ids_and_asset_docs[asset_id]) + for asset_id in diff_assets + ] + ) + + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) diff --git a/openpype/modules/kitsu/utils/update_zou_with_op.py b/openpype/modules/kitsu/utils/update_zou_with_op.py new file mode 100644 index 0000000000..d1fcde5601 --- /dev/null +++ b/openpype/modules/kitsu/utils/update_zou_with_op.py @@ -0,0 +1,262 @@ +"""Functions to update Kitsu DB (a.k.a Zou) using OpenPype Data.""" + +import re +from typing import List + +import gazu +from pymongo import UpdateOne + +from avalon.api import AvalonMongoDB +from openpype.api import get_project_settings +from openpype.modules.kitsu.utils.credentials import validate_credentials + + +def sync_zou(login: str, password: str): + """Synchronize Zou database (Kitsu backend) with openpype database. + This is an utility function to help updating zou data with OP's, it may not + handle correctly all cases, a human intervention might + be required after all. + Will work better if OP DB has been previously synchronized from zou/kitsu. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + gazu.exception.AuthFailedException: Wrong user login and/or password + """ + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + + op_projects = [p for p in dbcon.projects()] + for project_doc in op_projects: + sync_zou_from_op_project(project_doc["name"], dbcon, project_doc) + + +def sync_zou_from_op_project( + project_name: str, dbcon: AvalonMongoDB, project_doc: dict = None +) -> List[UpdateOne]: + """Update OP project in DB with Zou data. + + Args: + project_name (str): Name of project to sync + dbcon (AvalonMongoDB): MongoDB connection + project_doc (str, optional): Project doc to sync + """ + # Get project doc if not provided + if not project_doc: + project_doc = dbcon.database[project_name].find_one( + {"type": "project"} + ) + + # Get all entities from zou + print(f"Synchronizing {project_name}...") + zou_project = gazu.project.get_project_by_name(project_name) + + # Create project + if zou_project is None: + raise RuntimeError( + f"Project '{project_name}' doesn't exist in Zou database, " + "please create it in Kitsu and add OpenPype user to it before " + "running synchronization." + ) + + # Update project settings and data + if project_doc["data"]: + zou_project.update( + { + "code": project_doc["data"]["code"], + "fps": project_doc["data"]["fps"], + "resolution": f"{project_doc['data']['resolutionWidth']}" + f"x{project_doc['data']['resolutionHeight']}", + } + ) + gazu.project.update_project_data(zou_project, data=project_doc["data"]) + gazu.project.update_project(zou_project) + + asset_types = gazu.asset.all_asset_types() + all_assets = gazu.asset.all_assets_for_project(zou_project) + all_episodes = gazu.shot.all_episodes_for_project(zou_project) + all_seqs = gazu.shot.all_sequences_for_project(zou_project) + all_shots = gazu.shot.all_shots_for_project(zou_project) + all_entities_ids = { + e["id"] for e in all_episodes + all_seqs + all_shots + all_assets + } + + # Query all assets of the local project + project_module_settings = get_project_settings(project_name)["kitsu"] + project_col = dbcon.database[project_name] + asset_docs = { + asset_doc["_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + } + + # Create new assets + new_assets_docs = [ + doc + for doc in asset_docs.values() + if doc["data"].get("zou", {}).get("id") not in all_entities_ids + ] + naming_pattern = project_module_settings["entities_naming_pattern"] + regex_ep = re.compile( + r"(.*{}.*)|(.*{}.*)|(.*{}.*)".format( + naming_pattern["shot"].replace("#", ""), + naming_pattern["sequence"].replace("#", ""), + naming_pattern["episode"].replace("#", ""), + ), + re.IGNORECASE, + ) + bulk_writes = [] + for doc in new_assets_docs: + visual_parent_id = doc["data"]["visualParent"] + parent_substitutes = [] + + # Match asset type by it's name + match = regex_ep.match(doc["name"]) + if not match: # Asset + new_entity = gazu.asset.new_asset( + zou_project, asset_types[0], doc["name"] + ) + # Match case in shot Date: Thu, 3 Mar 2022 15:55:56 +0100 Subject: [PATCH 36/82] Cleaning. --- openpype/modules/kitsu/utils/credentials.py | 36 ++++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/openpype/modules/kitsu/utils/credentials.py b/openpype/modules/kitsu/utils/credentials.py index b4dd5ee4a2..0529380d6d 100644 --- a/openpype/modules/kitsu/utils/credentials.py +++ b/openpype/modules/kitsu/utils/credentials.py @@ -8,15 +8,21 @@ from openpype.lib.local_settings import OpenPypeSecureRegistry def validate_credentials( - login: str, password: str, kitsu_url: str = os.environ.get("KITSU_SERVER") + login: str, password: str, kitsu_url: str = None ) -> bool: """Validate credentials by trying to connect to Kitsu host URL. - :param login: Kitsu Login - :param password: Kitsu Password - :param kitsu_url: Kitsu host URL - :return: Are credentials valid? + Args: + login (str): Kitsu user login + password (str): Kitsu user password + kitsu_url (str, optional): Kitsu host URL. Defaults to None. + + Returns: + bool: Are credentials valid? """ + if kitsu_url is None: + kitsu_url = os.environ.get("KITSU_SERVER") + # Connect to server validate_host(kitsu_url) @@ -32,8 +38,11 @@ def validate_credentials( def validate_host(kitsu_url: str) -> bool: """Validate credentials by trying to connect to Kitsu host URL. - :param kitsu_url: Kitsu host URL - :return: Is host valid? + Args: + kitsu_url (str, optional): Kitsu host URL. + + Returns: + bool: Is host valid? """ # Connect to server gazu.set_host(kitsu_url) @@ -58,8 +67,9 @@ def clear_credentials(): def save_credentials(login: str, password: str): """Save credentials in Secure Registry. - :param login: Kitsu Login - :param password: Kitsu Password + Args: + login (str): Kitsu user login + password (str): Kitsu user password """ # Get user registry user_registry = OpenPypeSecureRegistry("kitsu_user") @@ -72,7 +82,8 @@ def save_credentials(login: str, password: str): def load_credentials() -> Tuple[str, str]: """Load registered credentials. - :return: Login, Password + Returns: + Tuple[str, str]: (Login, Password) """ # Get user registry user_registry = OpenPypeSecureRegistry("kitsu_user") @@ -85,8 +96,9 @@ def load_credentials() -> Tuple[str, str]: def set_credentials_envs(login: str, password: str): """Set environment variables with Kitsu login and password. - :param login: Kitsu Login - :param password: Kitsu Password + Args: + login (str): Kitsu user login + password (str): Kitsu user password """ os.environ["KITSU_LOGIN"] = login os.environ["KITSU_PWD"] = password From 5db0c1dfa7cbf483976b6e52f6d4cde5813daedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 4 Mar 2022 10:12:08 +0100 Subject: [PATCH 37/82] Python 2 compat and cleaning --- openpype/modules/kitsu/kitsu_module.py | 4 ++-- openpype/modules/kitsu/utils/sync_service.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index dca6133e88..53edfddf9a 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -105,7 +105,7 @@ def cli_main(): @click.option( "--password", envvar="KITSU_PWD", help="Password for kitsu username" ) -def push_to_zou(login: str, password: str): +def push_to_zou(login, password): """Synchronize Zou database (Kitsu backend) with openpype database. Args: @@ -122,7 +122,7 @@ def push_to_zou(login: str, password: str): @click.option( "-p", "--password", envvar="KITSU_PWD", help="Password for kitsu username" ) -def sync_service(login: str, password: str): +def sync_service(login, password): """Synchronize openpype database from Zou sever database. Args: diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 831673ec0d..6bf98cf308 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -383,8 +383,3 @@ def start_listeners(login: str, password: str): # Connect to server listener = Listener(login, password) listener.start() - - -if __name__ == "__main__": - # TODO not sure when this can be run and if this system is reliable - start_listeners(load_credentials()) From 8c3b510887564d52d7230c7114a17f9044157be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 4 Mar 2022 10:12:58 +0100 Subject: [PATCH 38/82] Cleaning --- openpype/modules/kitsu/utils/sync_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 6bf98cf308..2e8fbf77f5 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -3,7 +3,7 @@ import os import gazu from avalon.api import AvalonMongoDB -from .credentials import load_credentials, validate_credentials +from .credentials import validate_credentials from .update_op_with_zou import ( create_op_asset, set_op_project, From e5ae5459e135d10d01e1549f8ac3f8a8cb83e689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 4 Mar 2022 15:58:06 +0100 Subject: [PATCH 39/82] Kitsu docs --- website/docs/artist_kitsu.md | 17 ++++++++ .../docs/assets/kitsu/kitsu_credentials.png | Bin 0 -> 15798 bytes website/docs/module_kitsu.md | 37 ++++++++++++++++++ website/sidebars.js | 2 + 4 files changed, 56 insertions(+) create mode 100644 website/docs/artist_kitsu.md create mode 100644 website/docs/assets/kitsu/kitsu_credentials.png create mode 100644 website/docs/module_kitsu.md diff --git a/website/docs/artist_kitsu.md b/website/docs/artist_kitsu.md new file mode 100644 index 0000000000..9ef782c297 --- /dev/null +++ b/website/docs/artist_kitsu.md @@ -0,0 +1,17 @@ +--- +id: artist_kitsu +title: Kitsu +sidebar_label: Kitsu +--- + +# How to use Kitsu in OpenPype + +## Login to Kitsu module in OpenPype +1. Launch OpenPype, the `Kitsu Credentials` window will open automatically, if not, or if you want to log-in with another account, go to systray OpenPype icon and click on `Kitsu Connect`. +2. Enter your credentials and press *Ok*: + + ![kitsu-login](assets/kitsu/kitsu_credentials.png) + +:::tip +In Kitsu, All the publish actions executed by `pyblish` will be attributed to the currently logged-in user. +::: \ No newline at end of file diff --git a/website/docs/assets/kitsu/kitsu_credentials.png b/website/docs/assets/kitsu/kitsu_credentials.png new file mode 100644 index 0000000000000000000000000000000000000000..25c1ad93c4d966a083d1fbdcd15e78a54e659b76 GIT binary patch literal 15798 zcmdVBbx`HN_vi@>?t?S9ySuww+#N0sgUiKT1_pO`nZe!N-3NDfcZbLKx4ZA{R=ujX z_5RqZO?91gk~-<6lXN~uI!sAH3JLxPJQx@l5`c67#J}aKwL!4 zJ@b6s!W~1MD0JJxT(*`bLk6BAQAt#tHxNS^RYD}3G#>-6`0j5$#&+10A$Y5!5p+UN)Z`Pd-vsjerg*()?Xr{T3ftDs$Op!;mxMP`hFwRd;=`0y>wG znAk9}s>&X(U?$a)8HW_as}B%{%q51EIEe>~92*&d<4Ti@U}I$!ggArx(tylmWM>CO ze`z>;Ek14Q7q(5S${70#bl6|bh{cnW0MK}|kh2t064KLK1^D>*IA~~&&ERnY(*rN) ziHWyN{QN${!NI{}(k9G1`nHAD=T{jGP$9P!N=n0LkNeH*j(vQ5Wa+}==>aN*%QgnO zv4_~yU;_O7v`|n`NGOPi<>ob7cWxTB#IT~<&fNI%F)@nO>vkwjTJ&XE3KlC9*<{$J zeH$AaOwTVblq)uzmwZgBsE|DzT%>Hg()eT34!HR48t*0Em~dVo!%ay2LDop$5b^q_1a$H?D^^4E6lGT#ixSRYs&wDk* zW1+t&7rmF|{_DwD87g7Lc<92{S84>NygM;V!`HDQu$Q~Z!h^nxqh3%hSJAs<2+0<4 zHo&%P#%S@xmMVgBb*mOF4Q>n=ZZ5ngEg2o2^h<+SVl2ETei_6($*$YCTU|elcWE`9 z;qgiwxNVumfXQeS3#T6HS58RW-FvcZM|{7#REy{q-FR?Wz4&qNEw)nHe$~@*(WW? zKW9yO+IVUB8p3l|VJnWV69lac?jAaqFJfkIx;q=G-w`fk?k8PVo&W4CBN)ejJxJZD zTbdilESb_u#W53N@ob%gWtNzP`t zaOO_6Pm&SDttZKk@2n^h35O^xtOu*`0c6jp34D)+3qR9V#ISDH5eAqUE~HKs8x4Gf zNND^Iz6s^(Yb(yJ;b1A^OubDE0h~R*>vU2kEdeqihX1(dPw9fnfl)UULDw@{0HJF- zqV@*@uSDZHpB9vk6(Je5>X5U#S8cXeJW3`Z$&Kg4IPbFQJfL4IzebXh50)c?>BgT1(t}VKT&5k?eby1w$LFSB_ogM#h)}*Tt*b{!1;#9ZwV0Mf=Yv+zZT-KOlv-D#w@f+eLn`z*lc5f2$^a#4dyY zcbbB7)KV9$l{uX6{*U-@pU~fMfdz!ywP6IC$c9V#2}^bl?DH~`qNLic4&>(BkcC?d z?XM$c{8kQz;X;7&`Oqt4-f|vV9J_+rTyy?Kb};RRTCWo&L7?}AH+9^ z^ivB}@j22%J0dH6_7ks@w6mvj*;JF@$h;Pp~hYNBPDAxB|%pUbP2}ntfBl(OVMce8%$)qKNqEu&E|} zCt8=N#hdU|Z3a{sFFkb=;-6>Dt0NVy22HKuN4HxJXQH^UT)dT`g0-T0CvEQGq$l0L)VAJgV-?T`>{pt=aj2} zPOUy3-}6s?bY!gEX8;1O3Rm#pMCsLiV-~mFw5z6b_2H>2zo0^leXfu}z{InIj^9aA z=}6yzDKc<(Q1&KjWpl!PtstoQ>s#TW<;)<+Qc$d+dS$bGr4jsK`)J%?n^e_Z!hc0G z8|5@S${*lwaD_uKXEK}!%jB!PS%|j4Y+DcLH0H?XCnqoAy`*0o=SwNi3r-@I3mIp{ zq%KQRQfaDlb;F`HY|{5-tz_tSjc*BLOFIezP`iw65n=d;=#G={1c%tim>waxz_Ja0 zcKC*38!SFEJDR<2+M^LHb_u&3r*8gI-I5)GtV-ObZ;$Z#&$TKLD|SGBr3DwG!=TPa5|Mo4*oHZqdZ_V0OFs^l7^XvMz}{8IpEq0=7UB>)ID z(5&WGog8qSEJ3Se5IbMV8|P1RxHZ`NV~X@cxPj2*{w?Zx_ZN1f?TT!C{HK)~SVi<{ zvU=>f-mbt-GiqV{!A{?zZ5y3C*Kyyl=84fbPj~)ES1nT$DM1J*?{D_0-;cH8ca%+# z(r!Tf_FQbys5>Bqh$SUt^G|@=Ht1;-bsRMM>t&vs1?hmWP)FhFHS`^)iGx5L7;-51 z60EQfTGZwW#czXm!ih8Ui3QUxoA~Urih`tOq0eGZOj&W=eYJ#%&x4T1OBQb|Oh_*9 z*9)|o4?+MDQ`2Nu7`4VPBbwq1{ddI?ht6qhHw)^m)o;>Q>4sh5cS3C4aMQX~D6Zmq z98G!He6>eP)mOPqT1@E`)g5@%yQM9QHGL^Jir*15cE0uf?On;FqtumQ-H>06_B}Fo ziTNJ)uTA^FbfUH8D%kO|4>0e%j`5H+FM8@P`L?n^$%Xg=+i^dXoVndN3Ffo=Tcb+q zf8vP-x%ur+E!NIFL2W+GY4D5@0p_eoTmHWJ*^dZkrtyr88EGQuhy*Buv4!+EruUO3CRr6A_^L zX=?AysMBTqbUI#`$%Il6 z&^VpYMEk<^7z}PO0q^jZT4WK_U{RBXL2OF_M|gVJz*V%bYy1B~x&yBYC;2vSL?XZn zWwEVkXI$QJcoWh5Bt@DDcCwst2{^obhEkXkMVy;tTM`@T=V{Fyj;i7A;|H9sX~h97 z_`F5Lp$+DDk%DuJ`tP6?cRIn$rO*(`;}aN!sNl_l_f+n33ZtE3vlaX7FJLJ!`v3H&aWZu0L=r>>6+=fB>a9RCVVs}J=bSM$E zz%)}_U-lA-7NDNbOeoNQEE}Y7Lt1JDP$1wBIXBO!rf@WTRc6IlTsW1BG4cYjx?$=5 z_m)zc3t_17g5Z)(v>z^i%4u|aMw9VP>U-;#l`VicQ_i1?56jyb;FOxFXJ(=B{iKa%=V z^d>KK#_w_#&@Ga#>{$Jo@;}UOmOa~F*2JD{a)fh^;rJguELX@CjjrzA^-dR~F-S z58NM&_1@Xun93lKWL|f6D@Yv2y#Kzprhp`R%e-YhLG}{&sNa6f%l+#VO67O?7G`jN zEYL`|($be&ef)M@xhJodgqFGHC^3CJs@mX93Ug>y(7SIEbVR+JPb8KtZFEpnw`}j8 zuAGy6IMw7w-t?YJcS6vE!m@Aq>1mO!j@YlNG0fAJ)PjCc&2~uQsc<3skaEd#Den7X zMGD*-%aio=J%oASWn3=`c1sA5v%;E~j?s)GTuJ|anQS}-jl^N~YkT)Rh|>of;qRo0 zRB7Oz=J|<_f?ni%Nr2}3<2!O1t*ycS0A(D;?X@uHP>si=+QFHe*i7Q#pQ*O(8pd>g zUUk@qhaO!S!#+i!D=!chZ7!T=>ccv4tEP0@y^JSQS14+x?$Ko7m;T{W%S&qW21^F^ zT(AgH6N%5LwE4x9#Jto^Z*hRy+7-=gZMJc@+0YofV5T$d(0m5=1zHMZ?(Ldg<6u*v z&(wk`gVxvw*JJ*|F%2SKQ%+{Cr&`P*V7MCW1Y1H{^v+)RXma{|g zao4a=+NJ`rzJK%v?(kG-zAishcz&|oP7R9-ZpILrHD1r>GV$0kDU^X<&UTqVTuul; zOFu0fi^IG^P$H!%ln#V{Q$#b#51Slso(!N?t+m5A6(W$&VKZ;1buoTwW16v9q$*!y zO&}VRWSREj{N}rq^XdIePe&J%w^Qo&?3wWEjTo#?tO4%H6p;-1Ft7$%K(ce-6hOr6 z+7a@XsbR=%9p=s;D_oUGSe%-iE_X$h=Q}|ZzGc7;RYYZ;?RKeqliZTmrRS) z6W{%)K|{ziWe3h2ZN_uX`?x_uxDEGbYLRJt=A# zU|#`7bbrfpZYFYj_B$2JuGqOj&ee&t*D~KbGlj0cNlx0bslGb!W0QP;*wef+r}tPc z$v6gODGs{#}Xc49p1B}E0W;FOxSwgVf;G?B|RPMdIo3#(5Q55hke9Y zyc;Jd(tcZ}dh=wAS0Ed_ohat=Xp}_N5k{_(+GVABx{di}%H5OQp2X3nMp&u)@%Ab& z8m>K}_?by8z=d7M5ZZzFxTF60owUIpL%Kt9pQr8`h*s#tZe?kDS&NB3rmR$PUH*Q^ z{Lx^A=J+vDw(?NxexYH^<~&lzEj+Q^ZJ+S$E04d3$2+!hk6kvkmzN*!<;^womsm8& zuA(IIiNJMv%OF+KAt|&cMt9)=~m2oY)ej0(wAWYy*!ow*B>bc z?4|-5)nDP-QrwBY%ljz3rQ&%pJkoD}m_(sWDlQOzGS!)$#n{L*3>PN^(y-v#{*vT3 zC?gk)|CKmjI9RzW5t?ZNr| zIUp1kC3ec?G@0CGCn<+GB9e!ZAi`*nA|-=ww#>u_VCQp8pXE-mlAK?ATWZ-F94eO? zTv#6G)rr!CLCm$-#W+v|O4Z_AwJ|jxk-`>xINE4GA?ae_(ySJLv%e#dtO+B!Nkog#3xAxmaaaYrzNi z-I#NKm!;H)h0u;Me#M@;Jq(tSfdDkIQ~!xgNOKwSZwSmE9}*BNTj( zU7V_zn5MYT8jr`{6*??duCyAM5lg>GU`xY2-%-5Q#AZ1IK4sLEDd6B@-`=h`iW2Nv zcI>HM*YB2Ep5ou_y~lay=pX1;rH7_gwyzIy8vm?cnyg>+nSr~m>5z%Nz6LAaqfO%Q2AX@g2!j#i1b8RTzvXhAenP>x zFjaCqaQ0yL8pd>+=LDT8UDl>Tes8AIju69;msl+5oCHm2P`kkI%`fE(@FjROE4;#2LP+#v~@b1JJ zOEeoC><-cXp6P|y@1ft}FXb=-zC8=-h^QBiO`@CH6SX7Y2Xa}yot`^?Cf)y}ezzg= z)L0))tr2pdRg_?y31J~hVbp@s*53Ouin>J-up4ijh)2MuY10QPFu|G*{?{(+ywsju zFJ-WIjFeMD{_xQic#f>+1WzBTo@cO+28mNIU7PI>KjB^X^AY__vvQg&!D! ziUJpVjYuzF(v642&BN1h46G&cjK@BKskZ3rd?R)wwAel1KWX#}DGlHy7v5EmtfhFb z+*FS9F242#ZdiSH#1qq_ZQ=32kx8ahgNY{(NS9Jz3{Il?MMa#oS%bggU-Tnnn*`vV zXgTUEABn%>_?v$u0u6Hvt$51M?q7+!+|l$;uo!8b$7A_bne1mg7PWk<5;pN6Jwe*4 zh;o_dL^1qMR^}x7e{BDj?`KhLXdX!Ih0uG{XCbOeHG8W-6f(FC(%d5e?tvD3+$;lV ziljnzvi&jaF?%|B)cXv<;Hd!{T9MmqpX0mg>tx;#haew=0)Y;n4rM1+7H#!uD3#so zmqh9mY$Sd<8z_%*uf$Ou03p8G7^ zyx(cdz4rAc2&|$VgD>(M1bcNT@#4~_@3uT$X_7rqgc;n0_;_DP$RHQ_= zn%)|h-|F5WO2x==$zs5T4U5co3K!BbuHr*>&!UxHst$H(XuG|hD;i#T`zB_7X7c)} zF@06hE=2Oq~JGeS}MPM?we zb#KiK{c!VL&l%mx`)`SKDtC(`u<&EdvqqcAH@*ZSH*H~n;Y6lcqTjv#AjcHNQ073G z?JBp%GM%4pEzx&%TGg9+@USyoODVXj+QsD}tB(B6IPaz#Y}Y&AxEfU;G1DzQA6Au- z>^UveY7$%WAM_%Vqm4Ah&dX4d2@>CXHNkYZtB|2+6888}Wx$g)$z;02x(VmZBh7|Q z+5VJbisR@-a6{cw#RIITpr3!7v+>t2@T{Znr>LkA?X;ANxOhL0iPDIJCibmiGc&;+ zn3&Kc2htke*ACZEBK6Sa*DzniYyWHAr(z0kxb6V%Pij$=e&5eHfApVx3TH8lhF zjoGO=$E0=2Lmu0#YOHjI zumu(|eFOrpz|g?8j?|pEyeG==E?fx$iPs{zp|Bu(Y{S4X{`ci21W;5t^XQ)N=#Fss zCg`dzGSI=#iI;#o3tk}WH+WXw__+jSwjqctzYfcx>TnY2ZM;G zkC-(OX_n%_q^dnjFaEz9maS6|O1+mezH;~r@zt;8H++9e9w=vhy%Ve?1RY0$RxaPV zn;9|fc0>1|wl@yan-cb9>+*Jg6260zrBFGp3PpKgCS{8Gv2w8mu`fS@Mj=4pLcVK|eyS0*kxEX&eihbV51c3>^H2>ci5)djZI_*N~oFl3@{60bp zx-_b~VHcZs{mw4zB1okw<3}-?3ycTv?V-OL&)thtQ6!nzV6Y;0+c+#7!P0YA+Mire zbTr^ZtpUcD@g;KFPQmz?{AC?-m(ZbU%*Nka8rp6?o^&9sINVPdY|_~i9f4Ljw{|eJ z=Q$(aVUEZM52H4iW~h7pIqqRUZH9YG4)p02(e*+V1ln%I=TX~-DO_KPAQ2GsQ#Vb(5&Dx7NLC6O z@TgzaZC@*gkaP;$Z1f+Q{h(ZU;wFmp$*zYBN^7NpQ3u8`0+EYu(v7^WJ)<8DN<^wV z@l}Z$P=N6%4O74By zC7MHpp%H!2ciORGA_IcTw!?Q=?~K9Mi{VXA^5!t+zC`{p*Z-A73~t zRYOS;(T$D{+9bZ27T-9zL90S}#k8J){*3Y({bR{jFXS>gVd$`kef)L;t1)bg^Ppe? zG0ByPe*?>5i-60~EY$I;a9}x?5qWen@Q)=3cj=X2&<6R>7~KSHBAA-ob(^-JBeT7$ zSDL0&`B0OOn@)1`!!}7Mdwzf`OAaEg@MaSs~VYC=t^q2l^bJ|-3QYBQlxEsXlfJh9uZ3TI|rekmtf6QhOL z8;_uyfx}kQh<6ud^lHYw1%>CjWITa?`oMJo+)}ii zp}x&W;m_L|vi=iM*!z@NS@wMG7Mx_pHL=!jXMl_+=Qf#hXk|-wv){DnQg=plFE{Gc z&cq=wv+-ViNGJ~UYC6+5klb?mOAj|Zpo;(ctb;XsXyObGWj{WqEMuAdidxObLfQr z)MNf!nak1d`s|6bftq&;>H6dq7GC#ETqrKIf62Z?5g6CK^b^h*ihuff10)Z>)z5>) zyr!v+Wa)-YOEWdVKT@eNH8tdT*`>l`{=s2NIQeA1CN)7yb_<;Xov2sq{OP5qt`@uB z3viv6QJDPI85*8nT8!oFyVU#a$b$=rkA;A^UeqjDeL4npURm8|7kdfJ$go0JVAkmm z#KlS4UDt}c+Z@O#eX~lLNjNQ#!dvif-GI?{&=9HIiK*g=0*(1qjU}f6CZ9h?eFJbN za831`6spV%<49i7kEj9&l$=LerWs4&OT&4l-Zr1|OY?_WYayjK61~dF&WXxIe=GFl z{eVS^Uv})UYf2Y$ousg>G$@)d^MiQvDPDf%`wO&`jDiVw5y(c%O~JP z;4IqkO7Rbd(fxYsL^WFBN}^xhXJoa-7PjbeYP)Bva=y5;Jx_g-r!+=qcMahXt1R|oD9xhVDK%*C%*UlCOYWki- zRTuyZ=!hO*cTyfDYfs9d)OCL?79n=_^eA`R(owR?AD8DWMU=c&IdY-Mb2E7HL*ac; z-p>%$>)(P$D6CFhg2!GAN)YvAH9ufz6-ZRg=J9VbrVagvgrp&8_FsW*@YFnrsA$@1X+vZ0&k?Z@-<1-;=$<o7d8(pTZA}L#m4Ny{F!U)D~RVQzg_Zomv@&1k%|Qmj7BXNxT+n*iILO zkW7ilpuMnuYHO&CBg- zjH8qJjDz5lEyC>J^Oa0Wp7(4Y%yRm8UW4kBt2!yX;R`T> zE3v)e2V$_eq%zL{5kpNA53-i-ETZ&(jIUI6z8^^I3JWP=}}0^2e(b zl5r}33eQ-RGwFIr={0%tS>GH}b}tZ5!J#H*k9)8RI6=CV)g$LK_)T{7y9Sf?!1hka z?JMulO%2eW}f(`elb*Pm{a zPRZw8K0^ZqzF(!6Nd;F+ngDEG^)&Z|@}W446Y`m$vTLIm|IOcF6G!9|Y+ z4$DoT5jiMa&%3zhs*4)DdT5OZT2|EfZHJOe4bPbY_34l5O=XIBa@Tjjx&p1ka^W#cWxR5T>wPYD1$26?P;Ye;D%x( zErs7O@HNf&SED^a`WE~exNg^H#gY)09c)n4wYrbN``uqc{Bk>BBr$s4s&n-ZY%hcJ z(z&UHMeeNE+PMmR5^OhV{_Kg~!!J7#YIx)fq>6{0R-@lvdr3!Xxz8~#tQ7{r=!LZ6 zt_?eWe8rk{i7O!x0+qe+2pW(d5<7ELrtWe?B@AE%*_q!Z$hQOa$Dp8>q>jh^qQ@r#Jr--5E0dw=Q3D_!@aB)5aJ0#0OXA_1 zRbn3~UVT>Q<)DwsQ5dx2omfdje?HGmK$AB(n0IZUJ}!W==4BZxVtLl`5yah@YVTBC zHiK$;s*C)n6*$O&zCPYOE{UMUs}l(`+tSmyA!Y z8uvd7>HLI&D_#xbGO&^>tO`87qk^V4;-UC`` zd`w2ib7O!=w#zduc%uqOk|JyF458_=Yk`%avnOC!nRW7P{OEYj!S?B?SW^lg_|4^RGcGI@~RmOnuiqkjK(2k@;wq{zO8@Qfu5AweOZFSxsdx zL<#a03}!3j5%5|dci9NN^X%XvnpDVD>0s4+!m6@T+!qJ24<%GI5&EidD@J8bMjo@J zu~vWgMU(c8!g+d^`#X_7c~#UK*Zjab^~3&ZV*L1xdsmh>X{_qt#7u`2h(8v?_Lt4% z-|5`d-pW{|@=(?sB`g>LUp>j5_Y*idIb~dItoUnVMTuGse-dqSkeWTDsmhTn40vdmg+p;J$x9e?OEGnyzXOU#^M+MW zIjo_aO_w(_X*Cn94A9bR)k{nT#P77By3&zKGOI&`8mUawd$L<&ByW)bi*dbDW-P%Kl56}2dm7kC)v-6_Xa-tMhVhBqfB%ZtBr3kk_kJaR*$S@RZ0R; z;tNS~RVaEnh(k5=^Sht{H+d@iDwZt^tZBLMm~is|>Fh6jKl}x%FpCdhO_JHA0UU|u z7o(E*H6;I^FX7GkN(Xk?{Q}h|A3pK}0ryd0c;&$gOZPJovy;cac>%H*hZBL>kfG9F zMoI+Kt->Qgaym^u0pHKt(7TRNd_TQ0-SR=_7` zSC<^>d2D41>+DPrk%c>~(nSANBAt0r22!6QbK(z+NOX9n!O!4*?&~GB9wAp*XvRZW zP0o*HYAyzV+8W?QJN*{y|Eh(dvD^Mb9q!ut{xy7E$|lct+FBETVl=y7Gxv&#F$voH z-5D*EuWONWU>!wLkdp3XvJ{)T_cWdSA1)f73a~k)*4?4V;?^VNBVJ_(G21xQi{YI}Y63zBlZ^s0=w*g-L?>v?Qz?R6IQ0H}li=B>O=3 zSI>eyQ{&pwT!g~G-Fq5GUnO~XT3Izmdv<>Tw(MCz@*5HT;xCm>V-`f#;sthvp})Ix zKWD(KR!H(^$8Jp{V52X&f8!?A`@9bSwvwp(J$oi(BPlUZ?}ZK{N>n7)$IMSY4J9NO zT13XvVFnYnn3+)!-{QwrB+FGKKOtN%?H(*7I_4?z$&o2VUh#VO*Op=`-;!6PxoU|3 z`R_z#U%c6~^V_IY@*ls;?7K^^58!2XVhlY(_e<0#tH)@J8@_MT4kO|+cbN7w0-tBs z@KOm=Y>G!RLBwM9AJ-3jl4Y&5!wp_0)nCKCY9 z@J^bavj#Y~?46P2;}UK%!LS6>T%pfdlgnPqSrk6|{-enJID;JCVIQ4-$c;P-k*PIn z=z_xbs(Kjp;xQs!SXmlx1&+PzYa-ExYrMw`@T19`XkKPti19d(TD66>b=|WNCnA-Y z3|5mltyb&qRYf6_e`0sy2fB78qB76?D zZ4rmEqHaxAcTbg4HMsVUj@a1Yv9aB=mVVL8d(1lwExmel$h_fH2CQg8KF6l9X6ZE~ zNmR%t3W)08+*t}Blr4qnd6GvX6@-PKbu!!IP;lzR8K@Ok)tQxq9sxw}~!RSd9 zLNN`U%6fv{0U5Aw<39WmojXjLi4?FB zl2wb?%bg(47_a;i1)@ykhSo z4r)yOcq`L5$YI?Sh*Yw$Dt@TedEri?zb^95wyS{Y zSi`?+zV%094h^fj%$;!rENG8$V=Ko<`;kfN!|P(vlT@D6)t7ID#>^y)6WpfqTWEfO z(|D-7)TDUax2z;GQ=8^&^alP!SWZ3@Mz)^#?Iv4epAdb561sDG81;VSbYOdz@3FcP z{QR_2kZnHjD0Fo=g3zG{6B2%od*ehl3gc<0xHdr~G-*C@{5E8-_xk_HvHE+ze=6_M z^zMch9v&SYBcO2HjOl$nWXI?&xw`XEcSaSqm8W>?KH&4_)bIE_y9Z|F;+s@d$RU^2 z)r1NmBbIicc@ z&SwL5cMBOwSv~0bhBcRizncBq`#RIAeCX$({H1(Ujy_eP<+E z@m89L?8lciahg(XDMs(%R>HMa3pRHQ@_s(19!(D0D!We5ZZcwv{cOlI5_?5gFM$A? zwU{(|A7pTA7aAdp4QIepWjSRoL(e1*ovC$WvdYTtt#*7JfwK~?+V(k0zamt1TLni_ zzL%|W>bEpcH5JwA7)yEgs=IuR^Jb^R9MH-U1h&hao_nSF^oQk1Vne$y*WTTNmq#xu zb?%~uJotv!6(l<$#|E7$^*wygL5k_SP|wm7!W*APqrzuH43aTne7*pG>i&Fi7Ugm`)RyPj3`(P@^}y?|H+l>dQ)$OiO0;6X!_lG%_q+9M zi>?p7tl!S=I)J-9=*q>kV6=zb;~Abd{Up$wCQScU2dI(e{%%}E52|wl>ze43p#y4M%67)P8;d3;VaT(v2L&)Qn zmekA18nQLo5p*WSU+sFQ^L<36QEU7#&`eB>dXGx#1ZLSMlA&Ar<8K~#fM~iLP@4VUBec@Ulhnd_p6>Cvqj70B=NUN-{ufAPfuKD4g-vOsT!TjV&!0KIgDk)GS2%* zO62<+nP@oF?Z)(n&1Ox;m!leH>yqO%jmPVQ;9_9p`oByA)&ZUNvD6_FGBP0~@=vfg zkTU24#7{6)#c<58gyL?1*xjDl{)r4GAyK1Kt@jqJuB0AI4hOg)CGHN|3VmhWjzb_y z`*9ZyLdG?%hL9m_!G5_eg{RdTPr*Mb4JRG>e9u!y;8{)~PZ2;+&E}FMf*ZLtZ=f?J zr`#vrR5g!>G+%y9tm!^5|L;)o0 zY^~+zbD*x6WJ+l8oGTz)t2ka^B={x=kT+SnV)1@nriS?T#uTy{o#mDKbpH6OnJQ_| zrK|cNEU*2opHfge?Y9-HQSI5TpBPomy!ZY~>+KZn(ZNKjoW^9uOE?PFfPu>|ZDhW# z23z(Icre>%TjR=4e^fy<1peQLNC?r0 zcX74SRRWrBwM{5I`&<#_`L4tZzatSJd&4rJhfLW{&HhuqzfM-Wn(;rAH!Z_bV>>s} zEU$~T#=anMjjqt|JN{jCv~fTp^L#vGd50sbR^+m1LG5*5zkvVBv@^FxFULcr&0>%K z&m_y8_FqtM8dfTBsN_nceW3&#Gu)$K@#OjjB~K3E$Ii=y$ji5!=D{6%MQ;%Ef(ieU zWt>z__R2%B;;+#V{*}06*%1VXGF?DUWN2Z`Lujx5i`eJG@q8PY>G0X+riTqU%MOdO z81^bC23BFwX;$=DHMANs(YAQ`9>&a?xaXN5xhifBuwH zDBndsu5*-rHy}KsQP}d;b#lz^lnjE1*1r2GR+~~ffEJA@D|p-|XqX@BPSe`*5#M>6 zw&(gK@P|8ZO@slflj9@iI*nGn5k9);hOv%&W~gm0EVCLHyHoSZAm4YNlhs>kp^hZK z#>_kq@L5gy&7QSoL6G+YlBC;>Xl-H5Ytu3PT1s!oGQxDDwZvr{(@uqQO<|7FZk13% ztcP3lDMUx%@87?j2Xo`Mq1KNE{WJ_~w3gd0yBRQVm7=V05VP5R&yICHvx(QNMf+s2 zjz8kjb}qxd*w(Y^TBr8k4!W_4NyMg&K*}AM5D{O%7j<^+Rx_`6(ggITU&$8`eSFu*2 zOY^@dgWatEOe|oGH5nCmQLfroDb4??wx50-oA|$uZK-8`fwWOgs=m|9ff@!q)~&oB z!^FHxvx*4tOWkb`0}|VjRzVK;r(0X3*WYNbxFc}*z^z7cu>Tzi|EAttUjGECiA4$< hNc_JnYyF?#;2yg)xbjvzUlXsv01^t~m7<1${|y+=wcG#z literal 0 HcmV?d00001 diff --git a/website/docs/module_kitsu.md b/website/docs/module_kitsu.md new file mode 100644 index 0000000000..ec38cce5e1 --- /dev/null +++ b/website/docs/module_kitsu.md @@ -0,0 +1,37 @@ +--- +id: module_kitsu +title: Kitsu Administration +sidebar_label: Kitsu +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Kitsu is a great open source production tracker and can be used for project management instead of Ftrack. This documentation assumes that you are familiar with Kitsu and it's basic principles. If you're new to Kitsu, we recommend having a thorough look at [Kitsu Official Documentation](https://kitsu.cg-wire.com/). + +## Prepare Kitsu for OpenPype + +### Server URL +If you want to connect Kitsu to OpenPype you have to set the `Server` url in Kitsu settings. And that's all! +This setting is available for all the users of the OpenPype instance. + +## Synchronize +Updating OP with Kitsu data is executed running the `sync-service`, which requires to provide your Kitsu credentials with `-l, --login` and `-p, --password` or by setting the environment variables `KITSU_LOGIN` and `KITSU_PWD`. This process will request data from Kitsu and create/delete/update OP assets. +Once this sync is done, the thread will automatically start a loop to listen to Kitsu events. + +```bash +openpype_console module kitsu sync-service -l me@domain.ext -p my_password +``` + +### Events listening +Listening to Kitsu events is the key to automation of many tasks like _project/episode/sequence/shot/asset/task create/update/delete_ and some more. Events listening should run at all times to perform the required processing as it is not possible to catch some of them retrospectively with strong reliability. If such timeout has been encountered, you must relaunch the `sync-service` command to run the synchronization step again. + +### Push to Kitsu +An utility function is provided to help update Kitsu data (a.k.a Zou database) with OpenPype data if the publishing to the production tracker hasn't been possible for some time. Running `push-to-zou` will create the data on behalf of the user. +:::caution +This functionality cannot deal with all cases and is not error proof, some intervention by a human being might be required. +::: + +```bash +openpype_console module kitsu push-to-zou -l me@domain.ext -p my_password +``` diff --git a/website/sidebars.js b/website/sidebars.js index 105afc30eb..eecdcc6e9a 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -28,6 +28,7 @@ module.exports = { "artist_hosts_photoshop", "artist_hosts_tvpaint", "artist_hosts_unreal", + "artist_kitsu", { type: "category", label: "Ftrack", @@ -75,6 +76,7 @@ module.exports = { label: "Modules", items: [ "module_ftrack", + "module_kitsu", "module_site_sync", "module_deadline", "module_muster", From 05158585c778a3bed91c023626bcf5ee3baf29a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Mar 2022 09:22:16 +0100 Subject: [PATCH 40/82] Add pyblish comment to kitsu --- openpype/modules/kitsu/kitsu_module.py | 6 +++--- openpype/modules/kitsu/plugins/publish/kitsu_plugin.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py index 53edfddf9a..8e7ab6f78c 100644 --- a/openpype/modules/kitsu/kitsu_module.py +++ b/openpype/modules/kitsu/kitsu_module.py @@ -51,11 +51,11 @@ class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): set_credentials_envs, ) - username, password = load_credentials() + login, password = load_credentials() # Check credentials, ask them if needed - if validate_credentials(username, password): - set_credentials_envs(username, password) + if validate_credentials(login, password): + set_credentials_envs(login, password) else: self.show_dialog() diff --git a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py index b556f2b91f..5fce123d7e 100644 --- a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py +++ b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py @@ -39,7 +39,9 @@ class IntegrateRig(pyblish.api.InstancePlugin): gazu.task.add_comment( entity_task, entity_task["task_status_id"], - comment=f"Version {instance.data['version']} has been published!", - ) # TODO add comment from pyblish + comment=f"Version {instance.data['version']} has been published!\n" + "\n" # Add written comment in Pyblish + f"{instance.data['versionEntity']['data']['comment']}".strip(), + ) self.log.info("Version published to Kitsu successfully!") From 1591b91c54611e2bfe55902ad31929e72416515f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 8 Mar 2022 09:57:37 +0100 Subject: [PATCH 41/82] Python2 compat --- openpype/modules/kitsu/plugins/publish/kitsu_plugin.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py index 5fce123d7e..5d6c76bc3f 100644 --- a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py +++ b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py @@ -39,9 +39,11 @@ class IntegrateRig(pyblish.api.InstancePlugin): gazu.task.add_comment( entity_task, entity_task["task_status_id"], - comment=f"Version {instance.data['version']} has been published!\n" - "\n" # Add written comment in Pyblish - f"{instance.data['versionEntity']['data']['comment']}".strip(), + comment="Version {} has been published!\n".format( + instance.data["version"] + ) + # Add written comment in Pyblish + + "\n{}".format(instance.data["versionEntity"]["data"]["comment"]), ) self.log.info("Version published to Kitsu successfully!") From 9c0c43a0612c70fa7fa26e71c1f91b7605b3792d Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Tue, 15 Mar 2022 18:01:04 +0100 Subject: [PATCH 42/82] fix first sync crash --- .../modules/kitsu/utils/update_op_with_zou.py | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index eb675ad09e..e76d54d1ad 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -145,14 +145,15 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: project_data = project["data"] or {} # Update data - project_data.update( - { - "code": project["code"], - "fps": project["fps"], - "resolutionWidth": project["resolution"].split("x")[0], - "resolutionHeight": project["resolution"].split("x")[1], - } - ) + if project_data: + project_data.update( + { + "code": project["code"], + "fps": project["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ) return UpdateOne( {"_id": project_doc["_id"]}, @@ -211,10 +212,6 @@ def sync_project_from_kitsu( project = gazu.project.get_project_by_name(project_name) project_code = project_name - # Try to find project document - project_col = dbcon.database[project_code] - project_doc = project_col.find_one({"type": "project"}) - print(f"Synchronizing {project_name}...") # Get all assets from zou @@ -222,15 +219,15 @@ def sync_project_from_kitsu( all_episodes = gazu.shot.all_episodes_for_project(project) all_seqs = gazu.shot.all_sequences_for_project(project) all_shots = gazu.shot.all_shots_for_project(project) - all_entities = [ - e - for e in all_assets + all_episodes + all_seqs + all_shots - if e["data"] and not e["data"].get("is_substitute") - ] + all_entities = all_assets + all_episodes + all_seqs + all_shots # Sync project. Create if doesn't exist bulk_writes.append(write_project_to_op(project, dbcon)) + # Try to find project document + project_col = dbcon.database[project_code] + project_doc = project_col.find_one({"type": "project"}) + # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc From 17dd018aa060f8fbaf043e19e86a9a0e318f14dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 16 Mar 2022 17:01:30 +0100 Subject: [PATCH 43/82] Build project code --- openpype/modules/kitsu/utils/update_op_with_zou.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index e76d54d1ad..98f263efe1 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -144,11 +144,20 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: # Project data and tasks project_data = project["data"] or {} + # Build project code and update Kitsu + project_code = project.get("code") + if not project_code: + project_code = project["name"].replace(" ", "_").lower() + project["code"] = project_code + + # Update Zou + gazu.project.update_project(project) + # Update data if project_data: project_data.update( { - "code": project["code"], + "code": project_code, "fps": project["fps"], "resolutionWidth": project["resolution"].split("x")[0], "resolutionHeight": project["resolution"].split("x")[1], From 093e801b0b02074e4d5bd7ecd0ff6d791ba6ea92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 16 Mar 2022 17:31:22 +0100 Subject: [PATCH 44/82] Sync project FPS and resolution --- .../modules/kitsu/utils/update_op_with_zou.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 98f263efe1..cb2c79b942 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -154,15 +154,14 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: gazu.project.update_project(project) # Update data - if project_data: - project_data.update( - { - "code": project_code, - "fps": project["fps"], - "resolutionWidth": project["resolution"].split("x")[0], - "resolutionHeight": project["resolution"].split("x")[1], - } - ) + project_data.update( + { + "code": project_code, + "fps": project["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ) return UpdateOne( {"_id": project_doc["_id"]}, From bd06809ffaf8ff8ae8bde463a5ad990b486288c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 16 Mar 2022 18:44:26 +0100 Subject: [PATCH 45/82] Fix shot syncs --- openpype/modules/kitsu/utils/update_op_with_zou.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index cb2c79b942..6aea2e3930 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -1,4 +1,5 @@ """Functions to update OpenPype data using Kitsu DB (a.k.a Zou).""" +from copy import deepcopy from typing import Dict, List from pymongo import DeleteOne, UpdateOne @@ -54,9 +55,14 @@ def update_op_assets( for item in entities_list: # Update asset item_doc = asset_doc_ids[item["id"]] - item_data = item_doc["data"].copy() + item_data = deepcopy(item_doc["data"]) + item_data.update(item.get("data") or {}) item_data["zou"] = item + # Asset settings + item_data["frameStart"] = item_data.get("frame_in") + item_data["frameEnd"] = item_data.get("frame_out") + # Tasks tasks_list = [] if item["type"] == "Asset": @@ -103,9 +109,7 @@ def update_op_assets( # Update 'data' different in zou DB updated_data = { - k: item_data[k] - for k in item_data.keys() - if item_doc["data"].get(k) != item_data[k] + k: v for k, v in item_data.items() if item_doc["data"].get(k) != v } if updated_data or not item_doc.get("parent"): assets_with_update.append( From 84a13e4eb6ff99b433ea1d2fddce32d106381e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 17 Mar 2022 15:52:14 +0100 Subject: [PATCH 46/82] shots and assets custom folders root --- openpype/modules/kitsu/utils/sync_service.py | 8 +- .../modules/kitsu/utils/update_op_with_zou.py | 82 +++++++++++++++++-- .../defaults/project_settings/kitsu.json | 4 + .../projects_schema/schema_project_kitsu.json | 17 ++++ 4 files changed, 99 insertions(+), 12 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 2e8fbf77f5..746cb843e9 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -167,7 +167,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - [asset], zou_ids_and_asset_docs + project_col[asset], zou_ids_and_asset_docs )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) @@ -214,7 +214,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - [episode], zou_ids_and_asset_docs + project_col, [episode], zou_ids_and_asset_docs )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) @@ -262,7 +262,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - [sequence], zou_ids_and_asset_docs + project_col, [sequence], zou_ids_and_asset_docs )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) @@ -310,7 +310,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - [shot], zou_ids_and_asset_docs + project_col, [shot], zou_ids_and_asset_docs )[0] project_col.update_one({"_id": asset_doc_id}, asset_update) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 6aea2e3930..e2ad29bfa0 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -11,6 +11,7 @@ from gazu.task import ( ) from avalon.api import AvalonMongoDB +from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules.kitsu.utils.credentials import validate_credentials @@ -42,15 +43,23 @@ def set_op_project(dbcon, project_id) -> Collection: def update_op_assets( - entities_list: List[dict], asset_doc_ids: Dict[str, dict] + project_col: Collection, + entities_list: List[dict], + asset_doc_ids: Dict[str, dict], ) -> List[Dict[str, dict]]: """Update OpenPype assets. Set 'data' and 'parent' fields. - :param entities_list: List of zou entities to update - :param asset_doc_ids: Dicts of [{zou_id: asset_doc}, ...] - :return: List of (doc_id, update_dict) tuples + Args: + project_col (Collection): Mongo project collection to sync + entities_list (List[dict]): List of zou entities to update + asset_doc_ids (Dict[str, dict]): Dicts of [{zou_id: asset_doc}, ...] + + Returns: + List[Dict[str, dict]]: List of (doc_id, update_dict) tuples """ + project_name = project_col.name + assets_with_update = [] for item in entities_list: # Update asset @@ -65,9 +74,10 @@ def update_op_assets( # Tasks tasks_list = [] - if item["type"] == "Asset": + item_type = item["type"] + if item_type == "Asset": tasks_list = all_tasks_for_asset(item) - elif item["type"] == "Shot": + elif item_type == "Shot": tasks_list = all_tasks_for_shot(item) # TODO frame in and out item_data["tasks"] = { @@ -91,10 +101,39 @@ def update_op_assets( or item.get("source_id") ) # TODO check consistency - # Visual parent for hierarchy + # Substitute Episode and Sequence by Shot + project_module_settings = get_project_settings(project_name)["kitsu"] + substitute_item_type = ( + "shots" + if item_type in ["Episode", "Sequence"] + else f"{item_type.lower()}s" + ) + entity_parent_folders = [ + f + for f in project_module_settings["entities_root"] + .get(substitute_item_type) + .split("/") + if f + ] + + # Root parent folder if exist visual_parent_doc_id = ( asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None ) + if visual_parent_doc_id is None: + # Find root folder doc + root_folder_doc = project_col.find_one( + { + "type": "asset", + "name": entity_parent_folders[-1], + "data.root_of": substitute_item_type, + }, + ["_id"], + ) + if root_folder_doc: + visual_parent_doc_id = root_folder_doc["_id"] + + # Visual parent for hierarchy item_data["visualParent"] = visual_parent_doc_id # Add parents for hierarchy @@ -107,6 +146,9 @@ def update_op_assets( parent_entity = parent_doc["data"]["zou"] parent_zou_id = parent_entity["parent_id"] + # Set root folders parents + item_data["parents"] = entity_parent_folders + item_data["parents"] + # Update 'data' different in zou DB updated_data = { k: v for k, v in item_data.items() if item_doc["data"].get(k) != v @@ -248,6 +290,30 @@ def sync_project_from_kitsu( } zou_ids_and_asset_docs[project["id"]] = project_doc + # Create entities root folders + project_module_settings = get_project_settings(project_name)["kitsu"] + for entity_type, root in project_module_settings["entities_root"].items(): + parent_folders = root.split("/") + direct_parent_doc = None + for i, folder in enumerate(parent_folders, 1): + parent_doc = project_col.find_one( + {"type": "asset", "name": folder, "data.root_of": entity_type} + ) + if not parent_doc: + direct_parent_doc = project_col.insert_one( + { + "name": folder, + "type": "asset", + "schema": "openpype:asset-3.0", + "data": { + "root_of": entity_type, + "parents": parent_folders[:i], + "visualParent": direct_parent_doc, + "tasks": {}, + }, + } + ) + # Create to_insert = [] to_insert.extend( @@ -275,7 +341,7 @@ def sync_project_from_kitsu( [ UpdateOne({"_id": id}, update) for id, update in update_op_assets( - all_entities, zou_ids_and_asset_docs + project_col, all_entities, zou_ids_and_asset_docs ) ] ) diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index 435814a9d1..a37146e1d2 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -1,4 +1,8 @@ { + "entities_root": { + "assets": "Assets", + "shots": "Shots" + }, "entities_naming_pattern": { "episode": "E##", "sequence": "SQ##", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index a504959001..8d71d0ecd6 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -5,6 +5,23 @@ "collapsible": true, "is_file": true, "children": [ + { + "type": "dict", + "key": "entities_root", + "label": "Entities root folder", + "children": [ + { + "type": "text", + "key": "assets", + "label": "Assets:" + }, + { + "type": "text", + "key": "shots", + "label": "Shots (includes Episodes & Sequences if any):" + } + ] + }, { "type": "dict", "key": "entities_naming_pattern", From 3970229ce5908abfd62c90a88b656f08f232dbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 17 Mar 2022 16:09:15 +0100 Subject: [PATCH 47/82] Fix fps fallback to project's value --- openpype/modules/kitsu/utils/update_op_with_zou.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index e2ad29bfa0..288efe30da 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -71,6 +71,10 @@ def update_op_assets( # Asset settings item_data["frameStart"] = item_data.get("frame_in") item_data["frameEnd"] = item_data.get("frame_out") + # Sentinel for fps, fallback to project's value when entity fps is deleted + if not item_data.get("fps") and item_doc["data"].get("fps"): + project_doc = project_col.find_one({"type": "project"}) + item_data["fps"] = project_doc["data"]["fps"] # Tasks tasks_list = [] From 78bda28da4ec26cf4b8b8673cabc01316f578678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 17 Mar 2022 16:49:13 +0100 Subject: [PATCH 48/82] frame_in/out fallbacks --- .../modules/kitsu/utils/update_op_with_zou.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 288efe30da..f5c6406722 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -68,10 +68,19 @@ def update_op_assets( item_data.update(item.get("data") or {}) item_data["zou"] = item - # Asset settings - item_data["frameStart"] = item_data.get("frame_in") - item_data["frameEnd"] = item_data.get("frame_out") - # Sentinel for fps, fallback to project's value when entity fps is deleted + # == Asset settings == + # Frame in, fallback on 0 + frame_in = int(item_data.get("frame_in") or 0) + item_data["frameStart"] = frame_in + # Frame out, fallback on frame_in + duration + frames_duration = int(item.get("nb_frames") or 1) + frame_out = ( + item_data["frame_out"] + if item_data.get("frame_out") + else frame_in + frames_duration + ) + item_data["frameEnd"] = int(frame_out) + # Fps, fallback to project's value when entity fps is deleted if not item_data.get("fps") and item_doc["data"].get("fps"): project_doc = project_col.find_one({"type": "project"}) item_data["fps"] = project_doc["data"]["fps"] From 527f4a710dbddd21feac630bdafb7b986a354c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Fri, 25 Mar 2022 10:03:22 +0100 Subject: [PATCH 49/82] Sync only open projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Hector --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index f5c6406722..f43223cdf7 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -256,7 +256,7 @@ def sync_all_project(login: str, password: str): # Iterate projects dbcon = AvalonMongoDB() dbcon.install() - all_projects = gazu.project.all_projects() + all_projects = gazu.project.all_open_projects() for project in all_projects: sync_project_from_kitsu(project["name"], dbcon, project) From 63096b4e6b819081bfc3ea962ea82ab8d2ef507e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 4 May 2022 18:26:58 +0200 Subject: [PATCH 50/82] change avalon API --- openpype/modules/kitsu/utils/sync_service.py | 2 +- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 746cb843e9..01596e2667 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -2,7 +2,7 @@ import os import gazu -from avalon.api import AvalonMongoDB +from openpype.pipeline import AvalonMongoDB from .credentials import validate_credentials from .update_op_with_zou import ( create_op_asset, diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index f43223cdf7..25c89800d4 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -10,7 +10,7 @@ from gazu.task import ( all_tasks_for_shot, ) -from avalon.api import AvalonMongoDB +from openpype.pipeline import AvalonMongoDB from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules.kitsu.utils.credentials import validate_credentials From 0359ec6da4e03bf9c7df49060a423c028195d86c Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Thu, 7 Apr 2022 15:46:40 +0200 Subject: [PATCH 51/82] create kitsu collector --- .../publish/collect_kitsu_credential.py | 18 ++++++ .../plugins/publish/collect_kitsu_entities.py | 64 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py create mode 100644 openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py new file mode 100644 index 0000000000..bd0af16c8b --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py @@ -0,0 +1,18 @@ +import os + +import gazu + +import pyblish.api + + +class CollectKitsuSession(pyblish.api.ContextPlugin): + """Collect Kitsu session using user credentials""" + + order = pyblish.api.CollectorOrder + label = "Kitsu user session" + + + def process(self, context): + + gazu.client.set_host(os.environ["KITSU_SERVER"]) + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py new file mode 100644 index 0000000000..e4773f7b2a --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -0,0 +1,64 @@ +import os + +import gazu + +import pyblish.api + + +class CollectKitsuEntities(pyblish.api.ContextPlugin): + """Collect Kitsu entities according to the current context""" + + order = pyblish.api.CollectorOrder + 0.499 + label = "Kitsu entities" + + def process(self, context): + + os.environ["AVALON_PROJECT"], + os.environ["AVALON_ASSET"], + os.environ["AVALON_TASK"], + os.environ["AVALON_APP_NAME"] + + asset_data = context.data["assetEntity"]["data"] + zoo_asset_data = asset_data.get("zou") + if not zoo_asset_data: + raise + + kitsu_project = gazu.project.get_project(zoo_asset_data["project_id"]) + if not kitsu_project: + raise + context.data["kitsu_project"] = kitsu_project + + kitsu_asset = gazu.asset.get_asset(zoo_asset_data["entity_type_id"]) + if not kitsu_asset: + raise + context.data["kitsu_asset"] = kitsu_asset + + # kitsu_task_type = gazu.task.get_task_type_by_name(instance.data["task"]) + # if not kitsu_task_type: + # raise + # context.data["kitsu_task_type"] = kitsu_task_type + + zoo_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") + kitsu_task = gazu.task.get_task( + asset_data["zou"], + kitsu_task_type + ) + if not kitsu_task: + raise + context.data["kitsu_task"] = kitsu_task + + wip = gazu.task.get_task_status_by_short_name("wip") + + task = gazu.task.get_task_by_name(asset, modeling) + comment = gazu.task.add_comment(task, wip, "Change status to work in progress") + + person = gazu.person.get_person_by_desktop_login("john.doe") + + # task_type = gazu.task.get_task_type_by_name(instance.data["task"]) + + # entity_task = gazu.task.get_task_by_entity( + # asset_data["zou"], + # task_type + # ) + + raise \ No newline at end of file From 196c81ab78dffdafcca28c00657d11c799a9d7f4 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Fri, 8 Apr 2022 12:03:43 +0200 Subject: [PATCH 52/82] collect all kitsu entities in context --- .../plugins/publish/collect_kitsu_entities.py | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index e4773f7b2a..f599fd0c14 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -13,52 +13,31 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): def process(self, context): - os.environ["AVALON_PROJECT"], - os.environ["AVALON_ASSET"], - os.environ["AVALON_TASK"], - os.environ["AVALON_APP_NAME"] - asset_data = context.data["assetEntity"]["data"] zoo_asset_data = asset_data.get("zou") if not zoo_asset_data: - raise + raise AssertionError("Zoo asset data not found in OpenPype!") + self.log.debug("Collected zoo asset data: {}".format(zoo_asset_data)) + + zoo_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") + if not zoo_task_data: + raise AssertionError("Zoo task data not found in OpenPype!") + self.log.debug("Collected zoo task data: {}".format(zoo_task_data)) kitsu_project = gazu.project.get_project(zoo_asset_data["project_id"]) if not kitsu_project: - raise + raise AssertionError("Project not not found in kitsu!") context.data["kitsu_project"] = kitsu_project + self.log.debug("Collect kitsu project: {}".format(kitsu_project)) - kitsu_asset = gazu.asset.get_asset(zoo_asset_data["entity_type_id"]) + kitsu_asset = gazu.asset.get_asset(zoo_asset_data["id"]) if not kitsu_asset: - raise + raise AssertionError("Asset not not found in kitsu!") context.data["kitsu_asset"] = kitsu_asset + self.log.debug("Collect kitsu asset: {}".format(kitsu_asset)) - # kitsu_task_type = gazu.task.get_task_type_by_name(instance.data["task"]) - # if not kitsu_task_type: - # raise - # context.data["kitsu_task_type"] = kitsu_task_type - - zoo_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") - kitsu_task = gazu.task.get_task( - asset_data["zou"], - kitsu_task_type - ) + kitsu_task = gazu.task.get_task(zoo_task_data["id"]) if not kitsu_task: - raise + raise AssertionError("Task not not found in kitsu!") context.data["kitsu_task"] = kitsu_task - - wip = gazu.task.get_task_status_by_short_name("wip") - - task = gazu.task.get_task_by_name(asset, modeling) - comment = gazu.task.add_comment(task, wip, "Change status to work in progress") - - person = gazu.person.get_person_by_desktop_login("john.doe") - - # task_type = gazu.task.get_task_type_by_name(instance.data["task"]) - - # entity_task = gazu.task.get_task_by_entity( - # asset_data["zou"], - # task_type - # ) - - raise \ No newline at end of file + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) From eb288959f50666e0c9a4751d7908decd540de9de Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Sat, 9 Apr 2022 11:13:48 +0200 Subject: [PATCH 53/82] integrate note and status --- .../publish/collect_kitsu_credential.py | 2 +- .../plugins/publish/integrate_kitsu_note.py | 36 +++++++++++++++++++ .../plugins/publish/integrate_kitsu_review.py | 16 +++++++++ .../plugins/publish/validate_kitsu_intent.py | 27 ++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py create mode 100644 openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py create mode 100644 openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py index bd0af16c8b..c9d94d128a 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py @@ -10,7 +10,7 @@ class CollectKitsuSession(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder label = "Kitsu user session" - + # families = ["kitsu"] def process(self, context): diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py new file mode 100644 index 0000000000..5601dea586 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -0,0 +1,36 @@ +import gazu +import pyblish.api + + +class IntegrateKitsuNote(pyblish.api.ContextPlugin): + """Integrate Kitsu Note""" + + order = pyblish.api.IntegratorOrder + label = "Kitsu Note and Status" + # families = ["kitsu"] + optional = True + + def process(self, context): + + publish_comment = context.data.get("comment") + if not publish_comment: + self.log.info("Comment is not set.") + + publish_status = context.data.get("intent", {}).get("value") + if not publish_status: + self.log.info("Status is not set.") + + self.log.debug("Comment is `{}`".format(publish_comment)) + self.log.debug("Status is `{}`".format(publish_status)) + + kitsu_status = context.data.get("kitsu_status") + if not kitsu_status: + self.log.info("The status will not be changed") + kitsu_status = context.data["kitsu_task"].get("task_status") + self.log.debug("Kitsu status: {}".format(kitsu_status)) + + gazu.task.add_comment( + context.data["kitsu_task"], + kitsu_status, + comment = publish_comment + ) \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py new file mode 100644 index 0000000000..1853bf569f --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -0,0 +1,16 @@ +# import gazu +import pyblish.api + + +class IntegrateKitsuVersion(pyblish.api.InstancePlugin): + """Integrate Kitsu Review""" + + order = pyblish.api.IntegratorOrder + label = "Kitsu Review" + # families = ["kitsu"] + + def process(self, instance): + pass + + # gazu.task.upload_preview_file(preview, file_path, normalize_movie=True, client=) + # gazu.task.add_preview(task, comment, preview_file_path, normalize_movie=True, client=) \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py new file mode 100644 index 0000000000..9708ebb0dd --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -0,0 +1,27 @@ +from typing import Optional +import pyblish.api +import gazu + + +class IntegrateKitsuNote(pyblish.api.ContextPlugin): + """Integrate Kitsu Note""" + + order = pyblish.api.ValidatorOrder + label = "Kitsu Intent/Status" + # families = ["kitsu"] + optional = True + + def process(self, context): + + publish_status = context.data.get("intent", {}).get("value") + if not publish_status: + self.log.info("Status is not set.") + + kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) + if not kitsu_status: + raise AssertionError( + "Status `{}` not not found in kitsu!".format(kitsu_status) + ) + self.log.debug("Collect kitsu status: {}".format(kitsu_status)) + + context.data["kitsu_status"] = kitsu_status \ No newline at end of file From ff6c8a6a54b2146fce11b78ea7dccf048536d1e8 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Sun, 10 Apr 2022 10:47:11 +0200 Subject: [PATCH 54/82] Add kitsu log out --- .../plugins/publish/collect_kitsu_credential.py | 4 ++-- .../plugins/publish/collect_kitsu_entities.py | 2 +- .../plugins/publish/integrate_kitsu_file.py | 1 + .../plugins/publish/integrate_kitsu_note.py | 1 + .../plugins/publish/integrate_kitsu_review.py | 3 ++- .../kitsu/plugins/publish/other_kitsu_log_out.py | 16 ++++++++++++++++ 6 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py create mode 100644 openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py index c9d94d128a..4a27117e03 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py @@ -1,11 +1,11 @@ +# -*- coding: utf-8 -*- import os import gazu - import pyblish.api -class CollectKitsuSession(pyblish.api.ContextPlugin): +class CollectKitsuSession(pyblish.api.ContextPlugin): #rename log in """Collect Kitsu session using user credentials""" order = pyblish.api.CollectorOrder diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index f599fd0c14..c5df20b349 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -1,7 +1,7 @@ +# -*- coding: utf-8 -*- import os import gazu - import pyblish.api diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py new file mode 100644 index 0000000000..7c68785e9d --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 5601dea586..8844581237 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import gazu import pyblish.api diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 1853bf569f..a800ca9b57 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -1,4 +1,5 @@ -# import gazu +# -*- coding: utf-8 -*- +import gazu import pyblish.api diff --git a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py new file mode 100644 index 0000000000..ff76d9c4f6 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""".""" +import gazu +import pyblish.api + + +class KitsuLogOut(pyblish.api.ContextPlugin): + """ + Log out from Kitsu API + """ + + order = pyblish.api.IntegratorOrder + 10 + label = "Kitsu Log Out" + + def process(self, context): + gazu.client.log_out() From ed8c01c2639321126b43d84ad83bd06496cb1bad Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 11:57:33 +0200 Subject: [PATCH 55/82] upload file to kitsu --- .../plugins/publish/integrate_kitsu_note.py | 8 ++++--- .../plugins/publish/integrate_kitsu_review.py | 24 +++++++++++++++---- .../plugins/publish/validate_kitsu_intent.py | 5 ++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 8844581237..afe388fd82 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -9,7 +9,7 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder label = "Kitsu Note and Status" # families = ["kitsu"] - optional = True + # optional = True def process(self, context): @@ -30,8 +30,10 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): kitsu_status = context.data["kitsu_task"].get("task_status") self.log.debug("Kitsu status: {}".format(kitsu_status)) - gazu.task.add_comment( + kitsu_comment = gazu.task.add_comment( context.data["kitsu_task"], kitsu_status, comment = publish_comment - ) \ No newline at end of file + ) + + context.data["kitsu_comment"] = kitsu_comment \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index a800ca9b57..e69937e9bf 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import gazu import pyblish.api @@ -6,12 +7,27 @@ import pyblish.api class IntegrateKitsuVersion(pyblish.api.InstancePlugin): """Integrate Kitsu Review""" - order = pyblish.api.IntegratorOrder + order = pyblish.api.IntegratorOrder + 0.01 label = "Kitsu Review" # families = ["kitsu"] def process(self, instance): - pass - # gazu.task.upload_preview_file(preview, file_path, normalize_movie=True, client=) - # gazu.task.add_preview(task, comment, preview_file_path, normalize_movie=True, client=) \ No newline at end of file + context = instance.context + task = context.data["kitsu_task"] + comment = context.data["kitsu_comment"] + + for representation in instance.data.get("representations", []): + + local_path = representation.get("published_path") + self.log.info("*"*40) + self.log.info(local_path) + self.log.info(representation.get("tags", [])) + + # code = os.path.basename(local_path) + + if representation.get("tags", []): + continue + + # gazu.task.upload_preview_file(preview, file_path, normalize_movie=True) + gazu.task.add_preview(task, comment, local_path, normalize_movie=True) \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py index 9708ebb0dd..0597e8546a 100644 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -1,10 +1,9 @@ -from typing import Optional import pyblish.api import gazu -class IntegrateKitsuNote(pyblish.api.ContextPlugin): - """Integrate Kitsu Note""" +class ValidateKitsuIntent(pyblish.api.ContextPlugin): + """Validate Kitsu Status""" order = pyblish.api.ValidatorOrder label = "Kitsu Intent/Status" From 2e0f6ce42d631674d910dc3caffc1654b6d781f2 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 17:47:17 +0200 Subject: [PATCH 56/82] use task type if no task data in OP --- .../plugins/publish/collect_kitsu_entities.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index c5df20b349..c907c22e0f 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -21,7 +21,7 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): zoo_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") if not zoo_task_data: - raise AssertionError("Zoo task data not found in OpenPype!") + self.log.warning("Zoo task data not found in OpenPype!") self.log.debug("Collected zoo task data: {}".format(zoo_task_data)) kitsu_project = gazu.project.get_project(zoo_asset_data["project_id"]) @@ -36,8 +36,29 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): context.data["kitsu_asset"] = kitsu_asset self.log.debug("Collect kitsu asset: {}".format(kitsu_asset)) - kitsu_task = gazu.task.get_task(zoo_task_data["id"]) - if not kitsu_task: - raise AssertionError("Task not not found in kitsu!") - context.data["kitsu_task"] = kitsu_task - self.log.debug("Collect kitsu task: {}".format(kitsu_task)) + if zoo_task_data: + kitsu_task = gazu.task.get_task(zoo_task_data["id"]) + if not kitsu_task: + raise AssertionError("Task not not found in kitsu!") + context.data["kitsu_task"] = kitsu_task + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) + + else: + kitsu_task_type = gazu.task.get_task_type_by_name( + os.environ["AVALON_TASK"] + ) + if not kitsu_task_type: + raise AssertionError( + "Task type {} not found in Kitsu!".format( + os.environ["AVALON_TASK"] + ) + ) + + kitsu_task = gazu.task.get_task_by_name( + kitsu_asset, + kitsu_task_type + ) + if not kitsu_task: + raise AssertionError("Task not not found in kitsu!") + context.data["kitsu_task"] = kitsu_task + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) \ No newline at end of file From f72cb8ba9e6201be23f09515e3e3190408fc30a0 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 17:56:39 +0200 Subject: [PATCH 57/82] fix log out --- openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py index ff76d9c4f6..d7e1616f8d 100644 --- a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py +++ b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py @@ -13,4 +13,4 @@ class KitsuLogOut(pyblish.api.ContextPlugin): label = "Kitsu Log Out" def process(self, context): - gazu.client.log_out() + gazu.log_out() From 3d6a6fcbf065c5ed000aea43c7ce4b5db91cd47a Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 18:10:58 +0200 Subject: [PATCH 58/82] upload review --- .../plugins/publish/integrate_kitsu_review.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index e69937e9bf..23ee4a668e 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- import os +from typing import Optional import gazu import pyblish.api -class IntegrateKitsuVersion(pyblish.api.InstancePlugin): +class IntegrateKitsuReview(pyblish.api.InstancePlugin): """Integrate Kitsu Review""" order = pyblish.api.IntegratorOrder + 0.01 label = "Kitsu Review" # families = ["kitsu"] + optional = True def process(self, instance): @@ -20,14 +22,16 @@ class IntegrateKitsuVersion(pyblish.api.InstancePlugin): for representation in instance.data.get("representations", []): local_path = representation.get("published_path") - self.log.info("*"*40) - self.log.info(local_path) - self.log.info(representation.get("tags", [])) - # code = os.path.basename(local_path) - - if representation.get("tags", []): + if 'review' not in representation.get("tags", []): continue - # gazu.task.upload_preview_file(preview, file_path, normalize_movie=True) - gazu.task.add_preview(task, comment, local_path, normalize_movie=True) \ No newline at end of file + self.log.debug("Found review at: {}".format(local_path)) + + gazu.task.add_preview( + task, + comment, + local_path, + normalize_movie=True + ) + self.log.info("Review upload on comment") From db8719b895cf553de6f24ad80f610fff28acfc21 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 18:11:38 +0200 Subject: [PATCH 59/82] remove unused import --- .../modules/kitsu/plugins/publish/integrate_kitsu_review.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 23ee4a668e..59b3bcf53e 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -import os -from typing import Optional import gazu import pyblish.api From 7787056c969b0ca91f2f1eee1d58c6c92de2e4f8 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Wed, 20 Apr 2022 18:14:14 +0200 Subject: [PATCH 60/82] Do some cleanup --- .../plugins/publish/collect_kitsu_entities.py | 37 +++++++------- .../plugins/publish/integrate_kitsu_file.py | 1 - .../plugins/publish/integrate_kitsu_note.py | 5 +- .../plugins/publish/integrate_kitsu_review.py | 8 +-- .../kitsu/plugins/publish/kitsu_plugin.py | 49 ------------------- .../plugins/publish/other_kitsu_log_out.py | 1 - .../plugins/publish/validate_kitsu_intent.py | 5 +- 7 files changed, 28 insertions(+), 78 deletions(-) delete mode 100644 openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py delete mode 100644 openpype/modules/kitsu/plugins/publish/kitsu_plugin.py diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index c907c22e0f..935b020641 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -14,35 +14,36 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): def process(self, context): asset_data = context.data["assetEntity"]["data"] - zoo_asset_data = asset_data.get("zou") - if not zoo_asset_data: - raise AssertionError("Zoo asset data not found in OpenPype!") - self.log.debug("Collected zoo asset data: {}".format(zoo_asset_data)) + zou_asset_data = asset_data.get("zou") + if not zou_asset_data: + raise AssertionError("Zou asset data not found in OpenPype!") + self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) - zoo_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") - if not zoo_task_data: - self.log.warning("Zoo task data not found in OpenPype!") - self.log.debug("Collected zoo task data: {}".format(zoo_task_data)) + zou_task_data = asset_data["tasks"][ + os.environ["AVALON_TASK"]].get("zou") + if not zou_task_data: + self.log.warning("Zou task data not found in OpenPype!") + self.log.debug("Collected zou task data: {}".format(zou_task_data)) - kitsu_project = gazu.project.get_project(zoo_asset_data["project_id"]) + kitsu_project = gazu.project.get_project(zou_asset_data["project_id"]) if not kitsu_project: - raise AssertionError("Project not not found in kitsu!") + raise AssertionError("Project not found in kitsu!") context.data["kitsu_project"] = kitsu_project self.log.debug("Collect kitsu project: {}".format(kitsu_project)) - kitsu_asset = gazu.asset.get_asset(zoo_asset_data["id"]) + kitsu_asset = gazu.asset.get_asset(zou_asset_data["id"]) if not kitsu_asset: - raise AssertionError("Asset not not found in kitsu!") + raise AssertionError("Asset not found in kitsu!") context.data["kitsu_asset"] = kitsu_asset self.log.debug("Collect kitsu asset: {}".format(kitsu_asset)) - if zoo_task_data: - kitsu_task = gazu.task.get_task(zoo_task_data["id"]) + if zou_task_data: + kitsu_task = gazu.task.get_task(zou_task_data["id"]) if not kitsu_task: - raise AssertionError("Task not not found in kitsu!") + raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task self.log.debug("Collect kitsu task: {}".format(kitsu_task)) - + else: kitsu_task_type = gazu.task.get_task_type_by_name( os.environ["AVALON_TASK"] @@ -59,6 +60,6 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): kitsu_task_type ) if not kitsu_task: - raise AssertionError("Task not not found in kitsu!") + raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task - self.log.debug("Collect kitsu task: {}".format(kitsu_task)) \ No newline at end of file + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py deleted file mode 100644 index 7c68785e9d..0000000000 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_file.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- \ No newline at end of file diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index afe388fd82..61e4d2454c 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -9,7 +9,6 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder label = "Kitsu Note and Status" # families = ["kitsu"] - # optional = True def process(self, context): @@ -31,8 +30,8 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): self.log.debug("Kitsu status: {}".format(kitsu_status)) kitsu_comment = gazu.task.add_comment( - context.data["kitsu_task"], - kitsu_status, + context.data["kitsu_task"], + kitsu_status, comment = publish_comment ) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 59b3bcf53e..c38f14e8a4 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -19,17 +19,17 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): for representation in instance.data.get("representations", []): - local_path = representation.get("published_path") + review_path = representation.get("published_path") if 'review' not in representation.get("tags", []): continue - - self.log.debug("Found review at: {}".format(local_path)) + + self.log.debug("Found review at: {}".format(review_path)) gazu.task.add_preview( task, comment, - local_path, + review_path, normalize_movie=True ) self.log.info("Review upload on comment") diff --git a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py b/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py deleted file mode 100644 index 5d6c76bc3f..0000000000 --- a/openpype/modules/kitsu/plugins/publish/kitsu_plugin.py +++ /dev/null @@ -1,49 +0,0 @@ -import os - -import gazu - -import pyblish.api - - -class CollectExampleAddon(pyblish.api.ContextPlugin): - order = pyblish.api.CollectorOrder + 0.4 - label = "Collect Kitsu" - - def process(self, context): - self.log.info("I'm in Kitsu's plugin!") - - -class IntegrateRig(pyblish.api.InstancePlugin): - """Copy files to an appropriate location where others may reach it""" - - order = pyblish.api.IntegratorOrder - families = ["model"] - - def process(self, instance): - - # Connect to server - gazu.client.set_host(os.environ["KITSU_SERVER"]) - - # Authenticate - gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) - - asset_data = instance.data["assetEntity"]["data"] - - # Get task - task_type = gazu.task.get_task_type_by_name(instance.data["task"]) - entity_task = gazu.task.get_task_by_entity( - asset_data["zou"], task_type - ) - - # Comment entity - gazu.task.add_comment( - entity_task, - entity_task["task_status_id"], - comment="Version {} has been published!\n".format( - instance.data["version"] - ) - # Add written comment in Pyblish - + "\n{}".format(instance.data["versionEntity"]["data"]["comment"]), - ) - - self.log.info("Version published to Kitsu successfully!") diff --git a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py index d7e1616f8d..c4a5b390e0 100644 --- a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py +++ b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -""".""" import gazu import pyblish.api diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py index 0597e8546a..c82130b33b 100644 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import pyblish.api import gazu @@ -9,7 +10,7 @@ class ValidateKitsuIntent(pyblish.api.ContextPlugin): label = "Kitsu Intent/Status" # families = ["kitsu"] optional = True - + def process(self, context): publish_status = context.data.get("intent", {}).get("value") @@ -19,7 +20,7 @@ class ValidateKitsuIntent(pyblish.api.ContextPlugin): kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) if not kitsu_status: raise AssertionError( - "Status `{}` not not found in kitsu!".format(kitsu_status) + "Status `{}` not found in kitsu!".format(kitsu_status) ) self.log.debug("Collect kitsu status: {}".format(kitsu_status)) From d9062b762ab82b85fb048084ef27ebf863a92366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 5 May 2022 11:49:16 +0200 Subject: [PATCH 61/82] Update openpype/modules/kitsu/utils/update_zou_with_op.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/modules/kitsu/utils/update_zou_with_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_zou_with_op.py b/openpype/modules/kitsu/utils/update_zou_with_op.py index d1fcde5601..526159d101 100644 --- a/openpype/modules/kitsu/utils/update_zou_with_op.py +++ b/openpype/modules/kitsu/utils/update_zou_with_op.py @@ -6,7 +6,7 @@ from typing import List import gazu from pymongo import UpdateOne -from avalon.api import AvalonMongoDB +from openpype.pipeline import AvalonMongoDB from openpype.api import get_project_settings from openpype.modules.kitsu.utils.credentials import validate_credentials From 26fcd19a5d8bed9849c924c62a949b25e62c6189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 12:29:30 +0200 Subject: [PATCH 62/82] use dbcon.Session instead of collection --- openpype/modules/kitsu/utils/sync_service.py | 83 +++++++++---------- .../modules/kitsu/utils/update_op_with_zou.py | 42 +++++----- .../modules/kitsu/utils/update_zou_with_op.py | 6 +- 3 files changed, 63 insertions(+), 68 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 01596e2667..ad18e1f391 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -119,8 +119,8 @@ class Listener: # Write into DB if update_project: - project_col = self.dbcon.database[project_name] - project_col.bulk_write([update_project]) + self.dbcon = self.dbcon.database[project_name] + self.dbcon.bulk_write([update_project]) def _delete_project(self, data): """Delete project.""" @@ -129,28 +129,27 @@ class Listener: project = gazu.project.get_project(data["project_id"]) # Delete project collection - project_col = self.dbcon.database[project["name"]] - project_col.drop() + # self.dbcon = self.dbcon.database[project["name"]] # == Asset == def _new_asset(self, data): """Create new asset into OP DB.""" # Get project entity - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Get gazu entity asset = gazu.asset.get_asset(data["asset_id"]) # Insert doc in DB - project_col.insert_one(create_op_asset(asset)) + self.dbcon.insert_one(create_op_asset(asset)) # Update self._update_asset(data) def _update_asset(self, data): """Update asset into OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity @@ -160,23 +159,23 @@ class Listener: # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in self.dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } zou_ids_and_asset_docs[asset["project_id"]] = project_doc # Update asset_doc_id, asset_update = update_op_assets( - project_col[asset], zou_ids_and_asset_docs + self.dbcon[asset], zou_ids_and_asset_docs )[0] - project_col.update_one({"_id": asset_doc_id}, asset_update) + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) def _delete_asset(self, data): """Delete asset of OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Delete - project_col.delete_one( + self.dbcon.delete_one( {"type": "asset", "data.zou.id": data["asset_id"]} ) @@ -184,20 +183,20 @@ class Listener: def _new_episode(self, data): """Create new episode into OP DB.""" # Get project entity - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Get gazu entity episode = gazu.shot.get_episode(data["episode_id"]) # Insert doc in DB - project_col.insert_one(create_op_asset(episode)) + self.dbcon.insert_one(create_op_asset(episode)) # Update self._update_episode(data) def _update_episode(self, data): """Update episode into OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity @@ -207,24 +206,24 @@ class Listener: # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in self.dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } zou_ids_and_asset_docs[episode["project_id"]] = project_doc # Update asset_doc_id, asset_update = update_op_assets( - project_col, [episode], zou_ids_and_asset_docs + self.dbcon, [episode], zou_ids_and_asset_docs )[0] - project_col.update_one({"_id": asset_doc_id}, asset_update) + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) def _delete_episode(self, data): """Delete shot of OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) print("delete episode") # TODO check bugfix # Delete - project_col.delete_one( + self.dbcon.delete_one( {"type": "asset", "data.zou.id": data["episode_id"]} ) @@ -232,20 +231,20 @@ class Listener: def _new_sequence(self, data): """Create new sequnce into OP DB.""" # Get project entity - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Get gazu entity sequence = gazu.shot.get_sequence(data["sequence_id"]) # Insert doc in DB - project_col.insert_one(create_op_asset(sequence)) + self.dbcon.insert_one(create_op_asset(sequence)) # Update self._update_sequence(data) def _update_sequence(self, data): """Update sequence into OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity @@ -255,24 +254,24 @@ class Listener: # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in self.dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } zou_ids_and_asset_docs[sequence["project_id"]] = project_doc # Update asset_doc_id, asset_update = update_op_assets( - project_col, [sequence], zou_ids_and_asset_docs + self.dbcon, [sequence], zou_ids_and_asset_docs )[0] - project_col.update_one({"_id": asset_doc_id}, asset_update) + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) def _delete_sequence(self, data): """Delete sequence of OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) print("delete sequence") # TODO check bugfix # Delete - project_col.delete_one( + self.dbcon.delete_one( {"type": "asset", "data.zou.id": data["sequence_id"]} ) @@ -280,20 +279,20 @@ class Listener: def _new_shot(self, data): """Create new shot into OP DB.""" # Get project entity - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Get gazu entity shot = gazu.shot.get_shot(data["shot_id"]) # Insert doc in DB - project_col.insert_one(create_op_asset(shot)) + self.dbcon.insert_one(create_op_asset(shot)) # Update self._update_shot(data) def _update_shot(self, data): """Update shot into OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) # Get gazu entity @@ -303,23 +302,23 @@ class Listener: # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in self.dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } zou_ids_and_asset_docs[shot["project_id"]] = project_doc # Update asset_doc_id, asset_update = update_op_assets( - project_col, [shot], zou_ids_and_asset_docs + self.dbcon, [shot], zou_ids_and_asset_docs )[0] - project_col.update_one({"_id": asset_doc_id}, asset_update) + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) def _delete_shot(self, data): """Delete shot of OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Delete - project_col.delete_one( + self.dbcon.delete_one( {"type": "asset", "data.zou.id": data["shot_id"]} ) @@ -327,13 +326,13 @@ class Listener: def _new_task(self, data): """Create new task into OP DB.""" # Get project entity - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Get gazu entity task = gazu.task.get_task(data["task_id"]) # Find asset doc - asset_doc = project_col.find_one( + asset_doc = self.dbcon.find_one( {"type": "asset", "data.zou.id": task["entity"]["id"]} ) @@ -341,7 +340,7 @@ class Listener: asset_tasks = asset_doc["data"].get("tasks") task_type_name = task["task_type"]["name"] asset_tasks[task_type_name] = {"type": task_type_name, "zou": task} - project_col.update_one( + self.dbcon.update_one( {"_id": asset_doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} ) @@ -352,10 +351,10 @@ class Listener: def _delete_task(self, data): """Delete task of OP DB.""" - project_col = set_op_project(self.dbcon, data["project_id"]) + set_op_project(self.dbcon, data["project_id"]) # Find asset doc - asset_docs = [doc for doc in project_col.find({"type": "asset"})] + asset_docs = [doc for doc in self.dbcon.find({"type": "asset"})] for doc in asset_docs: # Match task for name, task in doc["data"]["tasks"].items(): @@ -365,7 +364,7 @@ class Listener: asset_tasks.pop(name) # Delete task in DB - project_col.update_one( + self.dbcon.update_one( {"_id": doc["_id"]}, {"$set": {"data.tasks": asset_tasks}}, ) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 25c89800d4..6e82ffbd05 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -29,18 +29,17 @@ def create_op_asset(gazu_entity: dict) -> dict: } -def set_op_project(dbcon, project_id) -> Collection: +def set_op_project(dbcon: AvalonMongoDB, project_id: str): """Set project context. - :param dbcon: Connection to DB. - :param project_id: Project zou ID + Args: + dbcon (AvalonMongoDB): Connection to DB. + project_id (str): Project zou ID """ project = gazu.project.get_project(project_id) project_name = project["name"] dbcon.Session["AVALON_PROJECT"] = project_name - return dbcon.database[project_name] - def update_op_assets( project_col: Collection, @@ -258,28 +257,25 @@ def sync_all_project(login: str, password: str): dbcon.install() all_projects = gazu.project.all_open_projects() for project in all_projects: - sync_project_from_kitsu(project["name"], dbcon, project) + sync_project_from_kitsu(dbcon, project) def sync_project_from_kitsu( - project_name: str, dbcon: AvalonMongoDB, project: dict = None + dbcon: AvalonMongoDB, project: dict ): """Update OP project in DB with Zou data. Args: - project_name (str): Name of project to sync dbcon (AvalonMongoDB): MongoDB connection - project (dict, optional): Project dict got using gazu. - Defaults to None. + project (dict): Project dict got using gazu. """ bulk_writes = [] # Get project from zou if not project: - project = gazu.project.get_project_by_name(project_name) - project_code = project_name + project = gazu.project.get_project_by_name(project["name"]) - print(f"Synchronizing {project_name}...") + print(f"Synchronizing {project['name']}...") # Get all assets from zou all_assets = gazu.asset.all_assets_for_project(project) @@ -292,28 +288,28 @@ def sync_project_from_kitsu( bulk_writes.append(write_project_to_op(project, dbcon)) # Try to find project document - project_col = dbcon.database[project_code] - project_doc = project_col.find_one({"type": "project"}) + dbcon.Session["AVALON_PROJECT"] = project["name"] + project_doc = dbcon.find_one({"type": "project"}) # Query all assets of the local project zou_ids_and_asset_docs = { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou", {}).get("id") } zou_ids_and_asset_docs[project["id"]] = project_doc # Create entities root folders - project_module_settings = get_project_settings(project_name)["kitsu"] + project_module_settings = get_project_settings(project["name"])["kitsu"] for entity_type, root in project_module_settings["entities_root"].items(): parent_folders = root.split("/") direct_parent_doc = None for i, folder in enumerate(parent_folders, 1): - parent_doc = project_col.find_one( + parent_doc = dbcon.find_one( {"type": "asset", "name": folder, "data.root_of": entity_type} ) if not parent_doc: - direct_parent_doc = project_col.insert_one( + direct_parent_doc = dbcon.insert_one( { "name": folder, "type": "asset", @@ -338,13 +334,13 @@ def sync_project_from_kitsu( ) if to_insert: # Insert doc in DB - project_col.insert_many(to_insert) + dbcon.insert_many(to_insert) # Update existing docs zou_ids_and_asset_docs.update( { asset_doc["data"]["zou"]["id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in dbcon.find({"type": "asset"}) if asset_doc["data"].get("zou") } ) @@ -354,7 +350,7 @@ def sync_project_from_kitsu( [ UpdateOne({"_id": id}, update) for id, update in update_op_assets( - project_col, all_entities, zou_ids_and_asset_docs + dbcon, all_entities, zou_ids_and_asset_docs ) ] ) @@ -373,4 +369,4 @@ def sync_project_from_kitsu( # Write into DB if bulk_writes: - project_col.bulk_write(bulk_writes) + dbcon.bulk_write(bulk_writes) diff --git a/openpype/modules/kitsu/utils/update_zou_with_op.py b/openpype/modules/kitsu/utils/update_zou_with_op.py index 526159d101..81d421206f 100644 --- a/openpype/modules/kitsu/utils/update_zou_with_op.py +++ b/openpype/modules/kitsu/utils/update_zou_with_op.py @@ -93,10 +93,10 @@ def sync_zou_from_op_project( # Query all assets of the local project project_module_settings = get_project_settings(project_name)["kitsu"] - project_col = dbcon.database[project_name] + dbcon.Session["AVALON_PROJECT"] = project_name asset_docs = { asset_doc["_id"]: asset_doc - for asset_doc in project_col.find({"type": "asset"}) + for asset_doc in dbcon.find({"type": "asset"}) } # Create new assets @@ -259,4 +259,4 @@ def sync_zou_from_op_project( # Write into DB if bulk_writes: - project_col.bulk_write(bulk_writes) + dbcon.bulk_write(bulk_writes) From 765546e6f96cf8ba480c20e65ad3f19cc8a24267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 12:31:08 +0200 Subject: [PATCH 63/82] fix unused var --- openpype/modules/kitsu/utils/sync_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index ad18e1f391..8c7ce730a7 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -126,7 +126,7 @@ class Listener: """Delete project.""" # Get project entity print(data) # TODO check bugfix - project = gazu.project.get_project(data["project_id"]) + # project = gazu.project.get_project(data["project_id"]) # Delete project collection # self.dbcon = self.dbcon.database[project["name"]] From 27d9e9bd3e9a9ef03489290101535aeb79617e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 14:08:05 +0200 Subject: [PATCH 64/82] Fix delete project --- openpype/modules/kitsu/utils/sync_service.py | 9 +++++---- openpype/modules/kitsu/utils/update_op_with_zou.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 8c7ce730a7..8d1ffb199d 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -124,12 +124,13 @@ class Listener: def _delete_project(self, data): """Delete project.""" - # Get project entity - print(data) # TODO check bugfix - # project = gazu.project.get_project(data["project_id"]) + project_doc = self.dbcon.find_one( + {"type": "project", "data.zou_id": data["project_id"]} + ) # Delete project collection - # self.dbcon = self.dbcon.database[project["name"]] + self.dbcon.database[project_doc["name"]].drop() + # == Asset == diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 6e82ffbd05..03a10e76a6 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -218,6 +218,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: "fps": project["fps"], "resolutionWidth": project["resolution"].split("x")[0], "resolutionHeight": project["resolution"].split("x")[1], + "zou_id": project["id"], } ) From af77d5a888f371c3bab4c2e38483ecaa7e7c6023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 16:37:17 +0200 Subject: [PATCH 65/82] fix wrong name entities makes crash: they are skipped --- openpype/modules/kitsu/utils/sync_service.py | 2 +- .../modules/kitsu/utils/update_op_with_zou.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 8d1ffb199d..46d3422727 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -131,7 +131,6 @@ class Listener: # Delete project collection self.dbcon.database[project_doc["name"]].drop() - # == Asset == def _new_asset(self, data): @@ -150,6 +149,7 @@ class Listener: def _update_asset(self, data): """Update asset into OP DB.""" + # TODO check if asset doesn't exist, create it (case where name wasn't valid) set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 03a10e76a6..fbc23cf52e 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -1,5 +1,6 @@ """Functions to update OpenPype data using Kitsu DB (a.k.a Zou).""" from copy import deepcopy +import re from typing import Dict, List from pymongo import DeleteOne, UpdateOne @@ -16,6 +17,10 @@ from openpype.lib import create_project from openpype.modules.kitsu.utils.credentials import validate_credentials +# Accepted namin pattern for OP +naming_pattern = re.compile("^[a-zA-Z0-9_.]*$") + + def create_op_asset(gazu_entity: dict) -> dict: """Create OP asset dict from gazu entity. @@ -261,9 +266,7 @@ def sync_all_project(login: str, password: str): sync_project_from_kitsu(dbcon, project) -def sync_project_from_kitsu( - dbcon: AvalonMongoDB, project: dict -): +def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): """Update OP project in DB with Zou data. Args: @@ -283,7 +286,11 @@ def sync_project_from_kitsu( all_episodes = gazu.shot.all_episodes_for_project(project) all_seqs = gazu.shot.all_sequences_for_project(project) all_shots = gazu.shot.all_shots_for_project(project) - all_entities = all_assets + all_episodes + all_seqs + all_shots + all_entities = [ + item + for item in all_assets + all_episodes + all_seqs + all_shots + if naming_pattern.match(item["name"]) + ] # Sync project. Create if doesn't exist bulk_writes.append(write_project_to_op(project, dbcon)) From eb5605a3ade0e4f59b8ffb21a7914c446b75372d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 17:24:11 +0200 Subject: [PATCH 66/82] fix updated asset skipped because of wrong name --- openpype/modules/kitsu/utils/sync_service.py | 9 ++++--- .../modules/kitsu/utils/update_op_with_zou.py | 24 ++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py index 46d3422727..6c003942f8 100644 --- a/openpype/modules/kitsu/utils/sync_service.py +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -149,7 +149,6 @@ class Listener: def _update_asset(self, data): """Update asset into OP DB.""" - # TODO check if asset doesn't exist, create it (case where name wasn't valid) set_op_project(self.dbcon, data["project_id"]) project_doc = self.dbcon.find_one({"type": "project"}) @@ -167,7 +166,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - self.dbcon[asset], zou_ids_and_asset_docs + self.dbcon, project_doc, [asset], zou_ids_and_asset_docs )[0] self.dbcon.update_one({"_id": asset_doc_id}, asset_update) @@ -214,7 +213,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - self.dbcon, [episode], zou_ids_and_asset_docs + self.dbcon, project_doc, [episode], zou_ids_and_asset_docs )[0] self.dbcon.update_one({"_id": asset_doc_id}, asset_update) @@ -262,7 +261,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - self.dbcon, [sequence], zou_ids_and_asset_docs + self.dbcon, project_doc, [sequence], zou_ids_and_asset_docs )[0] self.dbcon.update_one({"_id": asset_doc_id}, asset_update) @@ -310,7 +309,7 @@ class Listener: # Update asset_doc_id, asset_update = update_op_assets( - self.dbcon, [shot], zou_ids_and_asset_docs + self.dbcon, project_doc, [shot], zou_ids_and_asset_docs )[0] self.dbcon.update_one({"_id": asset_doc_id}, asset_update) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index fbc23cf52e..fa0bee8365 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -38,7 +38,7 @@ def set_op_project(dbcon: AvalonMongoDB, project_id: str): """Set project context. Args: - dbcon (AvalonMongoDB): Connection to DB. + dbcon (AvalonMongoDB): Connection to DB project_id (str): Project zou ID """ project = gazu.project.get_project(project_id) @@ -47,7 +47,8 @@ def set_op_project(dbcon: AvalonMongoDB, project_id: str): def update_op_assets( - project_col: Collection, + dbcon: AvalonMongoDB, + project_doc: dict, entities_list: List[dict], asset_doc_ids: Dict[str, dict], ) -> List[Dict[str, dict]]: @@ -55,19 +56,27 @@ def update_op_assets( Set 'data' and 'parent' fields. Args: - project_col (Collection): Mongo project collection to sync + dbcon (AvalonMongoDB): Connection to DB entities_list (List[dict]): List of zou entities to update asset_doc_ids (Dict[str, dict]): Dicts of [{zou_id: asset_doc}, ...] Returns: List[Dict[str, dict]]: List of (doc_id, update_dict) tuples """ - project_name = project_col.name + project_name = project_doc["name"] assets_with_update = [] for item in entities_list: + # Check asset exists + item_doc = asset_doc_ids.get(item["id"]) + if not item_doc: # Create asset + op_asset = create_op_asset(item) + insert_result = dbcon.insert_one(op_asset) + item_doc = dbcon.find_one( + {"type": "asset", "_id": insert_result.inserted_id} + ) + # Update asset - item_doc = asset_doc_ids[item["id"]] item_data = deepcopy(item_doc["data"]) item_data.update(item.get("data") or {}) item_data["zou"] = item @@ -86,7 +95,6 @@ def update_op_assets( item_data["frameEnd"] = int(frame_out) # Fps, fallback to project's value when entity fps is deleted if not item_data.get("fps") and item_doc["data"].get("fps"): - project_doc = project_col.find_one({"type": "project"}) item_data["fps"] = project_doc["data"]["fps"] # Tasks @@ -139,7 +147,7 @@ def update_op_assets( ) if visual_parent_doc_id is None: # Find root folder doc - root_folder_doc = project_col.find_one( + root_folder_doc = dbcon.find_one( { "type": "asset", "name": entity_parent_folders[-1], @@ -358,7 +366,7 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): [ UpdateOne({"_id": id}, update) for id, update in update_op_assets( - dbcon, all_entities, zou_ids_and_asset_docs + dbcon, project_doc, all_entities, zou_ids_and_asset_docs ) ] ) From 70bb0fd7bc5c37447e6f7d7d9dcdce5f940de919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Wed, 11 May 2022 17:38:46 +0200 Subject: [PATCH 67/82] fix flake --- openpype/modules/kitsu/utils/update_op_with_zou.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index fa0bee8365..10349a999c 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -4,7 +4,6 @@ import re from typing import Dict, List from pymongo import DeleteOne, UpdateOne -from pymongo.collection import Collection import gazu from gazu.task import ( all_tasks_for_asset, From 987b5df1ddf44a0feabe27feca8930782a05d18b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 09:16:49 +0200 Subject: [PATCH 68/82] optim get_project_settings --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 10349a999c..0c72537c94 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -63,6 +63,7 @@ def update_op_assets( List[Dict[str, dict]]: List of (doc_id, update_dict) tuples """ project_name = project_doc["name"] + project_module_settings = get_project_settings(project_name)["kitsu"] assets_with_update = [] for item in entities_list: @@ -126,7 +127,6 @@ def update_op_assets( ) # TODO check consistency # Substitute Episode and Sequence by Shot - project_module_settings = get_project_settings(project_name)["kitsu"] substitute_item_type = ( "shots" if item_type in ["Episode", "Sequence"] From e0bd8777d1b9563d292a72ff592e461eb47e50f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 09:42:51 +0200 Subject: [PATCH 69/82] pop useless item_data --- openpype/modules/kitsu/utils/update_op_with_zou.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py index 0c72537c94..673a195747 100644 --- a/openpype/modules/kitsu/utils/update_op_with_zou.py +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -85,6 +85,7 @@ def update_op_assets( # Frame in, fallback on 0 frame_in = int(item_data.get("frame_in") or 0) item_data["frameStart"] = frame_in + item_data.pop("frame_in") # Frame out, fallback on frame_in + duration frames_duration = int(item.get("nb_frames") or 1) frame_out = ( @@ -93,6 +94,7 @@ def update_op_assets( else frame_in + frames_duration ) item_data["frameEnd"] = int(frame_out) + item_data.pop("frame_out") # Fps, fallback to project's value when entity fps is deleted if not item_data.get("fps") and item_doc["data"].get("fps"): item_data["fps"] = project_doc["data"]["fps"] From ae69db29cac488410fa00c547042126831d3be0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 09:59:02 +0200 Subject: [PATCH 70/82] black pyblish --- .../plugins/publish/collect_kitsu_credential.py | 4 ++-- .../plugins/publish/collect_kitsu_entities.py | 16 ++++------------ .../plugins/publish/integrate_kitsu_note.py | 6 ++---- .../plugins/publish/integrate_kitsu_review.py | 9 ++------- .../plugins/publish/validate_kitsu_intent.py | 6 ++---- 5 files changed, 12 insertions(+), 29 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py index 4a27117e03..b7f6f67a40 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py @@ -5,7 +5,7 @@ import gazu import pyblish.api -class CollectKitsuSession(pyblish.api.ContextPlugin): #rename log in +class CollectKitsuSession(pyblish.api.ContextPlugin): # rename log in """Collect Kitsu session using user credentials""" order = pyblish.api.CollectorOrder @@ -15,4 +15,4 @@ class CollectKitsuSession(pyblish.api.ContextPlugin): #rename log in def process(self, context): gazu.client.set_host(os.environ["KITSU_SERVER"]) - gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) \ No newline at end of file + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index 935b020641..66c35e54c4 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -19,8 +19,7 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): raise AssertionError("Zou asset data not found in OpenPype!") self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) - zou_task_data = asset_data["tasks"][ - os.environ["AVALON_TASK"]].get("zou") + zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") if not zou_task_data: self.log.warning("Zou task data not found in OpenPype!") self.log.debug("Collected zou task data: {}".format(zou_task_data)) @@ -45,20 +44,13 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): self.log.debug("Collect kitsu task: {}".format(kitsu_task)) else: - kitsu_task_type = gazu.task.get_task_type_by_name( - os.environ["AVALON_TASK"] - ) + kitsu_task_type = gazu.task.get_task_type_by_name(os.environ["AVALON_TASK"]) if not kitsu_task_type: raise AssertionError( - "Task type {} not found in Kitsu!".format( - os.environ["AVALON_TASK"] - ) + "Task type {} not found in Kitsu!".format(os.environ["AVALON_TASK"]) ) - kitsu_task = gazu.task.get_task_by_name( - kitsu_asset, - kitsu_task_type - ) + kitsu_task = gazu.task.get_task_by_name(kitsu_asset, kitsu_task_type) if not kitsu_task: raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 61e4d2454c..99d891d514 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -30,9 +30,7 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): self.log.debug("Kitsu status: {}".format(kitsu_status)) kitsu_comment = gazu.task.add_comment( - context.data["kitsu_task"], - kitsu_status, - comment = publish_comment + context.data["kitsu_task"], kitsu_status, comment=publish_comment ) - context.data["kitsu_comment"] = kitsu_comment \ No newline at end of file + context.data["kitsu_comment"] = kitsu_comment diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index c38f14e8a4..65179bc0bf 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -21,15 +21,10 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): review_path = representation.get("published_path") - if 'review' not in representation.get("tags", []): + if "review" not in representation.get("tags", []): continue self.log.debug("Found review at: {}".format(review_path)) - gazu.task.add_preview( - task, - comment, - review_path, - normalize_movie=True - ) + gazu.task.add_preview(task, comment, review_path, normalize_movie=True) self.log.info("Review upload on comment") diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py index c82130b33b..e0fad3b79f 100644 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -19,9 +19,7 @@ class ValidateKitsuIntent(pyblish.api.ContextPlugin): kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) if not kitsu_status: - raise AssertionError( - "Status `{}` not found in kitsu!".format(kitsu_status) - ) + raise AssertionError("Status `{}` not found in kitsu!".format(kitsu_status)) self.log.debug("Collect kitsu status: {}".format(kitsu_status)) - context.data["kitsu_status"] = kitsu_status \ No newline at end of file + context.data["kitsu_status"] = kitsu_status From 3583d85cd2a7d25abf184411c23a273b0f8671c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 09:59:35 +0200 Subject: [PATCH 71/82] black pyblish --- .../plugins/publish/collect_kitsu_entities.py | 16 ++++++++++++---- .../plugins/publish/integrate_kitsu_review.py | 4 +++- .../plugins/publish/validate_kitsu_intent.py | 4 +++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index 66c35e54c4..84c400bde9 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -19,7 +19,9 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): raise AssertionError("Zou asset data not found in OpenPype!") self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) - zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") + zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get( + "zou" + ) if not zou_task_data: self.log.warning("Zou task data not found in OpenPype!") self.log.debug("Collected zou task data: {}".format(zou_task_data)) @@ -44,13 +46,19 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): self.log.debug("Collect kitsu task: {}".format(kitsu_task)) else: - kitsu_task_type = gazu.task.get_task_type_by_name(os.environ["AVALON_TASK"]) + kitsu_task_type = gazu.task.get_task_type_by_name( + os.environ["AVALON_TASK"] + ) if not kitsu_task_type: raise AssertionError( - "Task type {} not found in Kitsu!".format(os.environ["AVALON_TASK"]) + "Task type {} not found in Kitsu!".format( + os.environ["AVALON_TASK"] + ) ) - kitsu_task = gazu.task.get_task_by_name(kitsu_asset, kitsu_task_type) + kitsu_task = gazu.task.get_task_by_name( + kitsu_asset, kitsu_task_type + ) if not kitsu_task: raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 65179bc0bf..08fa4ee010 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -26,5 +26,7 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): self.log.debug("Found review at: {}".format(review_path)) - gazu.task.add_preview(task, comment, review_path, normalize_movie=True) + gazu.task.add_preview( + task, comment, review_path, normalize_movie=True + ) self.log.info("Review upload on comment") diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py index e0fad3b79f..6b2635bf05 100644 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -19,7 +19,9 @@ class ValidateKitsuIntent(pyblish.api.ContextPlugin): kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) if not kitsu_status: - raise AssertionError("Status `{}` not found in kitsu!".format(kitsu_status)) + raise AssertionError( + "Status `{}` not found in kitsu!".format(kitsu_status) + ) self.log.debug("Collect kitsu status: {}".format(kitsu_status)) context.data["kitsu_status"] = kitsu_status From 687f7260ceef7c0c3b2966f387e0f678a0fa0f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 10:57:18 +0200 Subject: [PATCH 72/82] optim publish status intent --- .../modules/kitsu/plugins/publish/validate_kitsu_intent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py index 6b2635bf05..e2023b171e 100644 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py @@ -12,10 +12,11 @@ class ValidateKitsuIntent(pyblish.api.ContextPlugin): optional = True def process(self, context): - + # Check publish status exists publish_status = context.data.get("intent", {}).get("value") if not publish_status: self.log.info("Status is not set.") + return kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) if not kitsu_status: From 20f819c1dd2eab2e6401a80c3996c3a93ca0a089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 12:36:47 +0200 Subject: [PATCH 73/82] change intent wrongly used as status choice --- .../plugins/publish/collect_kitsu_entities.py | 16 +++-------- .../plugins/publish/integrate_kitsu_note.py | 20 ++++++++----- .../plugins/publish/integrate_kitsu_review.py | 21 ++++++++------ .../plugins/publish/validate_kitsu_intent.py | 28 ------------------- 4 files changed, 30 insertions(+), 55 deletions(-) delete mode 100644 openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index 84c400bde9..66c35e54c4 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -19,9 +19,7 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): raise AssertionError("Zou asset data not found in OpenPype!") self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) - zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get( - "zou" - ) + zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") if not zou_task_data: self.log.warning("Zou task data not found in OpenPype!") self.log.debug("Collected zou task data: {}".format(zou_task_data)) @@ -46,19 +44,13 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): self.log.debug("Collect kitsu task: {}".format(kitsu_task)) else: - kitsu_task_type = gazu.task.get_task_type_by_name( - os.environ["AVALON_TASK"] - ) + kitsu_task_type = gazu.task.get_task_type_by_name(os.environ["AVALON_TASK"]) if not kitsu_task_type: raise AssertionError( - "Task type {} not found in Kitsu!".format( - os.environ["AVALON_TASK"] - ) + "Task type {} not found in Kitsu!".format(os.environ["AVALON_TASK"]) ) - kitsu_task = gazu.task.get_task_by_name( - kitsu_asset, kitsu_task_type - ) + kitsu_task = gazu.task.get_task_by_name(kitsu_asset, kitsu_task_type) if not kitsu_task: raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 99d891d514..980589365d 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -11,24 +11,30 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): # families = ["kitsu"] def process(self, context): + # Check if work version for user + is_work_version = bool(context.data.get("intent", {}).get("value")) + if is_work_version: + self.log.info("Work version, nothing pushed to Kitsu.") + return + # Get comment text body publish_comment = context.data.get("comment") if not publish_comment: self.log.info("Comment is not set.") - publish_status = context.data.get("intent", {}).get("value") - if not publish_status: - self.log.info("Status is not set.") - self.log.debug("Comment is `{}`".format(publish_comment)) - self.log.debug("Status is `{}`".format(publish_status)) - kitsu_status = context.data.get("kitsu_status") + # Get Waiting for Approval status + kitsu_status = gazu.task.get_task_status_by_short_name("wfa") if not kitsu_status: - self.log.info("The status will not be changed") + self.log.info( + "Cannot find 'Waiting For Approval' status." + "The status will not be changed" + ) kitsu_status = context.data["kitsu_task"].get("task_status") self.log.debug("Kitsu status: {}".format(kitsu_status)) + # Add comment to kitsu task kitsu_comment = gazu.task.add_comment( context.data["kitsu_task"], kitsu_status, comment=publish_comment ) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 08fa4ee010..76cfe62988 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -15,18 +15,23 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): context = instance.context task = context.data["kitsu_task"] - comment = context.data["kitsu_comment"] + comment = context.data.get("kitsu_comment") - for representation in instance.data.get("representations", []): + # Check comment has been created + if not comment: + self.log.debug("Comment not created, review not pushed to preview.") + return + + # Add review representations as preview of comment + for representation in [ + r + for r in instance.data.get("representations", []) + if "review" in representation.get("tags", []) + ]: review_path = representation.get("published_path") - if "review" not in representation.get("tags", []): - continue - self.log.debug("Found review at: {}".format(review_path)) - gazu.task.add_preview( - task, comment, review_path, normalize_movie=True - ) + gazu.task.add_preview(task, comment, review_path, normalize_movie=True) self.log.info("Review upload on comment") diff --git a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py b/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py deleted file mode 100644 index e2023b171e..0000000000 --- a/openpype/modules/kitsu/plugins/publish/validate_kitsu_intent.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -import pyblish.api -import gazu - - -class ValidateKitsuIntent(pyblish.api.ContextPlugin): - """Validate Kitsu Status""" - - order = pyblish.api.ValidatorOrder - label = "Kitsu Intent/Status" - # families = ["kitsu"] - optional = True - - def process(self, context): - # Check publish status exists - publish_status = context.data.get("intent", {}).get("value") - if not publish_status: - self.log.info("Status is not set.") - return - - kitsu_status = gazu.task.get_task_status_by_short_name(publish_status) - if not kitsu_status: - raise AssertionError( - "Status `{}` not found in kitsu!".format(kitsu_status) - ) - self.log.debug("Collect kitsu status: {}".format(kitsu_status)) - - context.data["kitsu_status"] = kitsu_status From 330d4340cc082307bd2a5bda951d7f2e491279f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Thu, 12 May 2022 12:39:03 +0200 Subject: [PATCH 74/82] black --- .../plugins/publish/collect_kitsu_entities.py | 16 ++++++++++++---- .../plugins/publish/integrate_kitsu_review.py | 10 +++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py index 66c35e54c4..84c400bde9 100644 --- a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -19,7 +19,9 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): raise AssertionError("Zou asset data not found in OpenPype!") self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) - zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get("zou") + zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get( + "zou" + ) if not zou_task_data: self.log.warning("Zou task data not found in OpenPype!") self.log.debug("Collected zou task data: {}".format(zou_task_data)) @@ -44,13 +46,19 @@ class CollectKitsuEntities(pyblish.api.ContextPlugin): self.log.debug("Collect kitsu task: {}".format(kitsu_task)) else: - kitsu_task_type = gazu.task.get_task_type_by_name(os.environ["AVALON_TASK"]) + kitsu_task_type = gazu.task.get_task_type_by_name( + os.environ["AVALON_TASK"] + ) if not kitsu_task_type: raise AssertionError( - "Task type {} not found in Kitsu!".format(os.environ["AVALON_TASK"]) + "Task type {} not found in Kitsu!".format( + os.environ["AVALON_TASK"] + ) ) - kitsu_task = gazu.task.get_task_by_name(kitsu_asset, kitsu_task_type) + kitsu_task = gazu.task.get_task_by_name( + kitsu_asset, kitsu_task_type + ) if not kitsu_task: raise AssertionError("Task not found in kitsu!") context.data["kitsu_task"] = kitsu_task diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 76cfe62988..57e0286b00 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -19,19 +19,23 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): # Check comment has been created if not comment: - self.log.debug("Comment not created, review not pushed to preview.") + self.log.debug( + "Comment not created, review not pushed to preview." + ) return # Add review representations as preview of comment for representation in [ r for r in instance.data.get("representations", []) - if "review" in representation.get("tags", []) + if "review" in r.get("tags", []) ]: review_path = representation.get("published_path") self.log.debug("Found review at: {}".format(review_path)) - gazu.task.add_preview(task, comment, review_path, normalize_movie=True) + gazu.task.add_preview( + task, comment, review_path, normalize_movie=True + ) self.log.info("Review upload on comment") From 631ccf6318916d9c7ad98830dba3007c073e70db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Mon, 16 May 2022 10:24:39 +0200 Subject: [PATCH 75/82] Update openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../kitsu/plugins/publish/integrate_kitsu_review.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index 57e0286b00..a036f5f9cc 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -25,12 +25,9 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): return # Add review representations as preview of comment - for representation in [ - r - for r in instance.data.get("representations", []) - if "review" in r.get("tags", []) - ]: - + for representation in instance.data.get("representations", []): + if "review" not in r.get("tags", []): + continue review_path = representation.get("published_path") self.log.debug("Found review at: {}".format(review_path)) From 15aa5709ae18c3cca83e73fef9eaa557684a0a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Mon, 16 May 2022 10:25:41 +0200 Subject: [PATCH 76/82] cleaning --- .../modules/kitsu/plugins/publish/integrate_kitsu_review.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py index a036f5f9cc..bf80095225 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -26,8 +26,10 @@ class IntegrateKitsuReview(pyblish.api.InstancePlugin): # Add review representations as preview of comment for representation in instance.data.get("representations", []): - if "review" not in r.get("tags", []): + # Skip if not tagged as review + if "review" not in representation.get("tags", []): continue + review_path = representation.get("published_path") self.log.debug("Found review at: {}".format(review_path)) From e6286166fba934945756357ee08b5d6d89f95cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Mon, 16 May 2022 11:33:35 +0200 Subject: [PATCH 77/82] remove default intent effect on review integration --- .../modules/kitsu/plugins/publish/integrate_kitsu_note.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 980589365d..9e067a8ecb 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -11,11 +11,6 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): # families = ["kitsu"] def process(self, context): - # Check if work version for user - is_work_version = bool(context.data.get("intent", {}).get("value")) - if is_work_version: - self.log.info("Work version, nothing pushed to Kitsu.") - return # Get comment text body publish_comment = context.data.get("comment") From 05cb2e4bd938ee1c533bb39625f4d3bbb7b2dad4 Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Mon, 16 May 2022 13:49:39 +0200 Subject: [PATCH 78/82] waiting approval status can be set in project settings --- .../plugins/publish/integrate_kitsu_note.py | 9 ++++--- .../defaults/project_settings/kitsu.json | 5 ++++ .../projects_schema/schema_project_kitsu.json | 25 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 9e067a8ecb..876eb6bf29 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -9,6 +9,7 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder label = "Kitsu Note and Status" # families = ["kitsu"] + waiting_for_approval_status = "wfa" def process(self, context): @@ -20,11 +21,13 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): self.log.debug("Comment is `{}`".format(publish_comment)) # Get Waiting for Approval status - kitsu_status = gazu.task.get_task_status_by_short_name("wfa") + kitsu_status = gazu.task.get_task_status_by_short_name( + self.waiting_for_approval_status + ) if not kitsu_status: self.log.info( - "Cannot find 'Waiting For Approval' status." - "The status will not be changed" + "Cannot find {} status. The status will not be " + "changed!".format(self.waiting_for_approval_status) ) kitsu_status = context.data["kitsu_task"].get("task_status") self.log.debug("Kitsu status: {}".format(kitsu_status)) diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index a37146e1d2..2f1566d89a 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -7,5 +7,10 @@ "episode": "E##", "sequence": "SQ##", "shot": "SH##" + }, + "publish": { + "IntegrateKitsuNote": { + "waiting_for_approval_status": "wfa" + } } } \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index 8d71d0ecd6..cffd7ff578 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -43,6 +43,31 @@ "label": "Shot:" } ] + }, + { + "type": "dict", + "collapsible": true, + "key": "publish", + "label": "Publish plugins", + "children": [ + { + "type": "label", + "label": "Integrator" + }, + { + "type": "dict", + "collapsible": true, + "key": "IntegrateKitsuNote", + "label": "Integrate Kitsu Note", + "children": [ + { + "type": "text", + "key": "waiting_for_approval_status", + "label": "Waiting for Aproval Status:" + } + ] + } + ] } ] } From 0b0a9ca2815251ba423f543e376a38e5805c0aba Mon Sep 17 00:00:00 2001 From: "clement.hector" Date: Mon, 16 May 2022 15:46:47 +0200 Subject: [PATCH 79/82] by default use task status if not specified in config --- .../plugins/publish/integrate_kitsu_note.py | 34 ++++++++++++------- .../defaults/project_settings/kitsu.json | 3 +- .../projects_schema/schema_project_kitsu.json | 9 +++-- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 876eb6bf29..ae559e660e 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from distutils.log import debug import gazu import pyblish.api @@ -9,7 +10,8 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder label = "Kitsu Note and Status" # families = ["kitsu"] - waiting_for_approval_status = "wfa" + set_status_note = False + note_status_shortname = "wfa" def process(self, context): @@ -20,21 +22,29 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): self.log.debug("Comment is `{}`".format(publish_comment)) - # Get Waiting for Approval status - kitsu_status = gazu.task.get_task_status_by_short_name( - self.waiting_for_approval_status - ) - if not kitsu_status: - self.log.info( - "Cannot find {} status. The status will not be " - "changed!".format(self.waiting_for_approval_status) + # Get note status, by default uses the task status for the note + # if it is not specified in the configuration + note_status = context.data["kitsu_task"]["task_status_id"] + if self.set_status_note: + kitsu_status = gazu.task.get_task_status_by_short_name( + self.note_status_shortname ) - kitsu_status = context.data["kitsu_task"].get("task_status") - self.log.debug("Kitsu status: {}".format(kitsu_status)) + if not kitsu_status: + self.log.info( + "Cannot find {} status. The status will not be " + "changed!".format(self.note_status_shortname) + ) + else: + note_status = kitsu_status + self.log.info("Note Kitsu status: {}".format(note_status)) # Add comment to kitsu task + self.log.debug("Add new note in taks id {}".format( + context.data["kitsu_task"]['id'])) kitsu_comment = gazu.task.add_comment( - context.data["kitsu_task"], kitsu_status, comment=publish_comment + context.data["kitsu_task"], + note_status, + comment=publish_comment ) context.data["kitsu_comment"] = kitsu_comment diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index 2f1566d89a..ba02d8d259 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -10,7 +10,8 @@ }, "publish": { "IntegrateKitsuNote": { - "waiting_for_approval_status": "wfa" + "set_status_note": false, + "note_status_shortname": "wfa" } } } \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index cffd7ff578..014a1b7886 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -60,10 +60,15 @@ "key": "IntegrateKitsuNote", "label": "Integrate Kitsu Note", "children": [ + { + "type": "boolean", + "key": "set_status_note", + "label": "Set status on note" + }, { "type": "text", - "key": "waiting_for_approval_status", - "label": "Waiting for Aproval Status:" + "key": "note_status_shortname", + "label": "Note shortname" } ] } From 5059c0cedff88d33f7c0044f0020fa03d1cdca48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Hector?= Date: Tue, 17 May 2022 11:43:28 +0200 Subject: [PATCH 80/82] Update openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Félix David --- .../modules/kitsu/plugins/publish/integrate_kitsu_note.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index ae559e660e..78c5170856 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -29,14 +29,14 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): kitsu_status = gazu.task.get_task_status_by_short_name( self.note_status_shortname ) - if not kitsu_status: + if kitsu_status: + note_status = kitsu_status + self.log.info("Note Kitsu status: {}".format(note_status)) + else: self.log.info( "Cannot find {} status. The status will not be " "changed!".format(self.note_status_shortname) ) - else: - note_status = kitsu_status - self.log.info("Note Kitsu status: {}".format(note_status)) # Add comment to kitsu task self.log.debug("Add new note in taks id {}".format( From 667cff319d70ea699ded5d009872588f5d5ffee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 17 May 2022 11:49:54 +0200 Subject: [PATCH 81/82] black --- .../kitsu/plugins/publish/integrate_kitsu_note.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 78c5170856..3cd1f450ca 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -22,7 +22,7 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): self.log.debug("Comment is `{}`".format(publish_comment)) - # Get note status, by default uses the task status for the note + # Get note status, by default uses the task status for the note # if it is not specified in the configuration note_status = context.data["kitsu_task"]["task_status_id"] if self.set_status_note: @@ -39,12 +39,13 @@ class IntegrateKitsuNote(pyblish.api.ContextPlugin): ) # Add comment to kitsu task - self.log.debug("Add new note in taks id {}".format( - context.data["kitsu_task"]['id'])) + self.log.debug( + "Add new note in taks id {}".format( + context.data["kitsu_task"]["id"] + ) + ) kitsu_comment = gazu.task.add_comment( - context.data["kitsu_task"], - note_status, - comment=publish_comment + context.data["kitsu_task"], note_status, comment=publish_comment ) context.data["kitsu_comment"] = kitsu_comment From 6976546505590a59072095f675778c9e8a71fe03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20David?= Date: Tue, 17 May 2022 11:51:58 +0200 Subject: [PATCH 82/82] cleaning --- openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py index 3cd1f450ca..ea98e0b7cc 100644 --- a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from distutils.log import debug import gazu import pyblish.api