diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 96c1b84a54..bca64b19f8 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -905,6 +905,7 @@ class TrayModulesManager(ModulesManager): modules_menu_order = ( "user", "ftrack", + "kitsu", "muster", "launcher_tool", "avalon", diff --git a/openpype/modules/kitsu/__init__.py b/openpype/modules/kitsu/__init__.py new file mode 100644 index 0000000000..9220cb1762 --- /dev/null +++ b/openpype/modules/kitsu/__init__.py @@ -0,0 +1,9 @@ +""" Addon class definition and Settings definition must be imported here. + +If addon class or settings definition won't be here their definition won't +be found by OpenPype discovery. +""" + +from .kitsu_module import KitsuModule + +__all__ = ("KitsuModule",) diff --git a/openpype/modules/kitsu/kitsu_module.py b/openpype/modules/kitsu/kitsu_module.py new file mode 100644 index 0000000000..8e7ab6f78c --- /dev/null +++ b/openpype/modules/kitsu/kitsu_module.py @@ -0,0 +1,136 @@ +"""Kitsu module.""" + +import click +import os + +from openpype.modules import OpenPypeModule +from openpype_interfaces import IPluginPaths, ITrayAction + + +class KitsuModule(OpenPypeModule, IPluginPaths, ITrayAction): + """Kitsu module class.""" + + label = "Kitsu Connect" + name = "kitsu" + + def initialize(self, settings): + """Initialization of module.""" + module_settings = settings[self.name] + + # Enabled by settings + self.enabled = module_settings.get("enabled", False) + + # Add API URL schema + kitsu_url = module_settings["server"].strip() + if kitsu_url: + # Ensure web url + if not kitsu_url.startswith("http"): + kitsu_url = "https://" + kitsu_url + + # Check for "/api" url validity + if not kitsu_url.endswith("api"): + kitsu_url = "{}{}api".format( + kitsu_url, "" if kitsu_url.endswith("/") else "/" + ) + + self.server_url = kitsu_url + + # UI which must not be created at this time + self._dialog = None + + def tray_init(self): + """Tray init.""" + + self._create_dialog() + + def tray_start(self): + """Tray start.""" + from .utils.credentials import ( + load_credentials, + validate_credentials, + set_credentials_envs, + ) + + login, password = load_credentials() + + # Check credentials, ask them if needed + if validate_credentials(login, password): + set_credentials_envs(login, password) + else: + self.show_dialog() + + def get_global_environments(self): + """Kitsu's global environments.""" + return {"KITSU_SERVER": self.server_url} + + def _create_dialog(self): + # Don't recreate dialog if already exists + if self._dialog is not None: + return + + from .kitsu_widgets import KitsuPasswordDialog + + self._dialog = KitsuPasswordDialog() + + def show_dialog(self): + """Show dialog to log-in.""" + + # Make sure dialog is created + self._create_dialog() + + # Show dialog + self._dialog.open() + + def on_action_trigger(self): + """Implementation of abstract method for `ITrayAction`.""" + self.show_dialog() + + def get_plugin_paths(self): + """Implementation of abstract method for `IPluginPaths`.""" + current_dir = os.path.dirname(os.path.abspath(__file__)) + + return {"publish": [os.path.join(current_dir, "plugins", "publish")]} + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group(KitsuModule.name, help="Kitsu dynamic cli commands.") +def cli_main(): + pass + + +@cli_main.command() +@click.option("--login", envvar="KITSU_LOGIN", help="Kitsu login") +@click.option( + "--password", envvar="KITSU_PWD", help="Password for kitsu username" +) +def push_to_zou(login, password): + """Synchronize Zou database (Kitsu backend) with openpype database. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + from .utils.update_zou_with_op import sync_zou + + sync_zou(login, password) + + +@cli_main.command() +@click.option("-l", "--login", envvar="KITSU_LOGIN", help="Kitsu login") +@click.option( + "-p", "--password", envvar="KITSU_PWD", help="Password for kitsu username" +) +def sync_service(login, password): + """Synchronize openpype database from Zou sever database. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + from .utils.update_op_with_zou import sync_all_project + from .utils.sync_service import start_listeners + + sync_all_project(login, password) + start_listeners(login, password) diff --git a/openpype/modules/kitsu/kitsu_widgets.py b/openpype/modules/kitsu/kitsu_widgets.py new file mode 100644 index 0000000000..65baed9665 --- /dev/null +++ b/openpype/modules/kitsu/kitsu_widgets.py @@ -0,0 +1,188 @@ +from Qt import QtWidgets, QtCore, QtGui + +from openpype import style +from openpype.modules.kitsu.utils.credentials import ( + clear_credentials, + load_credentials, + save_credentials, + set_credentials_envs, + validate_credentials, +) +from openpype.resources import get_resource +from openpype.settings.lib import ( + get_system_settings, +) + +from openpype.widgets.password_dialog import PressHoverButton + + +class KitsuPasswordDialog(QtWidgets.QDialog): + """Kitsu login dialog.""" + + finished = QtCore.Signal(bool) + + def __init__(self, parent=None): + super(KitsuPasswordDialog, self).__init__(parent) + + self.setWindowTitle("Kitsu Credentials") + self.resize(300, 120) + + system_settings = get_system_settings() + user_login, user_pwd = load_credentials() + remembered = bool(user_login or user_pwd) + + self._final_result = None + self._connectable = bool( + system_settings["modules"].get("kitsu", {}).get("server") + ) + + # Server label + server_message = ( + system_settings["modules"]["kitsu"]["server"] + if self._connectable + else "no server url set in Studio Settings..." + ) + server_label = QtWidgets.QLabel( + f"Server: {server_message}", + self, + ) + + # Login input + login_widget = QtWidgets.QWidget(self) + + login_label = QtWidgets.QLabel("Login:", login_widget) + + login_input = QtWidgets.QLineEdit( + login_widget, + text=user_login if remembered else None, + ) + login_input.setPlaceholderText("Your Kitsu account login...") + + login_layout = QtWidgets.QHBoxLayout(login_widget) + login_layout.setContentsMargins(0, 0, 0, 0) + login_layout.addWidget(login_label) + login_layout.addWidget(login_input) + + # Password input + password_widget = QtWidgets.QWidget(self) + + password_label = QtWidgets.QLabel("Password:", password_widget) + + password_input = QtWidgets.QLineEdit( + password_widget, + text=user_pwd if remembered else None, + ) + password_input.setPlaceholderText("Your password...") + password_input.setEchoMode(QtWidgets.QLineEdit.Password) + + show_password_icon_path = get_resource("icons", "eye.png") + show_password_icon = QtGui.QIcon(show_password_icon_path) + show_password_btn = PressHoverButton(password_widget) + show_password_btn.setObjectName("PasswordBtn") + show_password_btn.setIcon(show_password_icon) + show_password_btn.setFocusPolicy(QtCore.Qt.ClickFocus) + + password_layout = QtWidgets.QHBoxLayout(password_widget) + password_layout.setContentsMargins(0, 0, 0, 0) + password_layout.addWidget(password_label) + password_layout.addWidget(password_input) + password_layout.addWidget(show_password_btn) + + # Message label + message_label = QtWidgets.QLabel("", self) + + # Buttons + buttons_widget = QtWidgets.QWidget(self) + + remember_checkbox = QtWidgets.QCheckBox("Remember", buttons_widget) + remember_checkbox.setObjectName("RememberCheckbox") + remember_checkbox.setChecked(remembered) + + ok_btn = QtWidgets.QPushButton("Ok", buttons_widget) + cancel_btn = QtWidgets.QPushButton("Cancel", buttons_widget) + + buttons_layout = QtWidgets.QHBoxLayout(buttons_widget) + buttons_layout.setContentsMargins(0, 0, 0, 0) + buttons_layout.addWidget(remember_checkbox) + buttons_layout.addStretch(1) + buttons_layout.addWidget(ok_btn) + buttons_layout.addWidget(cancel_btn) + + # Main layout + layout = QtWidgets.QVBoxLayout(self) + layout.addSpacing(5) + layout.addWidget(server_label, 0) + layout.addSpacing(5) + layout.addWidget(login_widget, 0) + layout.addWidget(password_widget, 0) + layout.addWidget(message_label, 0) + layout.addStretch(1) + layout.addWidget(buttons_widget, 0) + + ok_btn.clicked.connect(self._on_ok_click) + cancel_btn.clicked.connect(self._on_cancel_click) + show_password_btn.change_state.connect(self._on_show_password) + + self.login_input = login_input + self.password_input = password_input + self.remember_checkbox = remember_checkbox + self.message_label = message_label + + self.setStyleSheet(style.load_stylesheet()) + + def result(self): + return self._final_result + + def keyPressEvent(self, event): + if event.key() in (QtCore.Qt.Key_Return, QtCore.Qt.Key_Enter): + self._on_ok_click() + return event.accept() + super(KitsuPasswordDialog, self).keyPressEvent(event) + + def closeEvent(self, event): + super(KitsuPasswordDialog, self).closeEvent(event) + self.finished.emit(self.result()) + + def _on_ok_click(self): + # Check if is connectable + if not self._connectable: + self.message_label.setText( + "Please set server url in Studio Settings!" + ) + return + + # Collect values + login_value = self.login_input.text() + pwd_value = self.password_input.text() + remember = self.remember_checkbox.isChecked() + + # Authenticate + if validate_credentials(login_value, pwd_value): + set_credentials_envs(login_value, pwd_value) + else: + self.message_label.setText("Authentication failed...") + return + + # Remember password cases + if remember: + save_credentials(login_value, pwd_value) + else: + # Clear local settings + clear_credentials() + + # Clear input fields + self.login_input.clear() + self.password_input.clear() + + self._final_result = True + self.close() + + def _on_show_password(self, show_password): + if show_password: + echo_mode = QtWidgets.QLineEdit.Normal + else: + echo_mode = QtWidgets.QLineEdit.Password + self.password_input.setEchoMode(echo_mode) + + def _on_cancel_click(self): + self.close() diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py new file mode 100644 index 0000000000..b7f6f67a40 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_credential.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +import os + +import gazu +import pyblish.api + + +class CollectKitsuSession(pyblish.api.ContextPlugin): # rename log in + """Collect Kitsu session using user credentials""" + + order = pyblish.api.CollectorOrder + label = "Kitsu user session" + # families = ["kitsu"] + + def process(self, context): + + gazu.client.set_host(os.environ["KITSU_SERVER"]) + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) diff --git a/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py new file mode 100644 index 0000000000..84c400bde9 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/collect_kitsu_entities.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +import os + +import gazu +import pyblish.api + + +class CollectKitsuEntities(pyblish.api.ContextPlugin): + """Collect Kitsu entities according to the current context""" + + order = pyblish.api.CollectorOrder + 0.499 + label = "Kitsu entities" + + def process(self, context): + + asset_data = context.data["assetEntity"]["data"] + zou_asset_data = asset_data.get("zou") + if not zou_asset_data: + raise AssertionError("Zou asset data not found in OpenPype!") + self.log.debug("Collected zou asset data: {}".format(zou_asset_data)) + + zou_task_data = asset_data["tasks"][os.environ["AVALON_TASK"]].get( + "zou" + ) + if not zou_task_data: + self.log.warning("Zou task data not found in OpenPype!") + self.log.debug("Collected zou task data: {}".format(zou_task_data)) + + kitsu_project = gazu.project.get_project(zou_asset_data["project_id"]) + if not kitsu_project: + raise AssertionError("Project not found in kitsu!") + context.data["kitsu_project"] = kitsu_project + self.log.debug("Collect kitsu project: {}".format(kitsu_project)) + + kitsu_asset = gazu.asset.get_asset(zou_asset_data["id"]) + if not kitsu_asset: + raise AssertionError("Asset not found in kitsu!") + context.data["kitsu_asset"] = kitsu_asset + self.log.debug("Collect kitsu asset: {}".format(kitsu_asset)) + + if zou_task_data: + kitsu_task = gazu.task.get_task(zou_task_data["id"]) + if not kitsu_task: + raise AssertionError("Task not found in kitsu!") + context.data["kitsu_task"] = kitsu_task + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) + + else: + kitsu_task_type = gazu.task.get_task_type_by_name( + os.environ["AVALON_TASK"] + ) + if not kitsu_task_type: + raise AssertionError( + "Task type {} not found in Kitsu!".format( + os.environ["AVALON_TASK"] + ) + ) + + kitsu_task = gazu.task.get_task_by_name( + kitsu_asset, kitsu_task_type + ) + if not kitsu_task: + raise AssertionError("Task not found in kitsu!") + context.data["kitsu_task"] = kitsu_task + self.log.debug("Collect kitsu task: {}".format(kitsu_task)) diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py new file mode 100644 index 0000000000..ea98e0b7cc --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_note.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +import gazu +import pyblish.api + + +class IntegrateKitsuNote(pyblish.api.ContextPlugin): + """Integrate Kitsu Note""" + + order = pyblish.api.IntegratorOrder + label = "Kitsu Note and Status" + # families = ["kitsu"] + set_status_note = False + note_status_shortname = "wfa" + + def process(self, context): + + # Get comment text body + publish_comment = context.data.get("comment") + if not publish_comment: + self.log.info("Comment is not set.") + + self.log.debug("Comment is `{}`".format(publish_comment)) + + # Get note status, by default uses the task status for the note + # if it is not specified in the configuration + note_status = context.data["kitsu_task"]["task_status_id"] + if self.set_status_note: + kitsu_status = gazu.task.get_task_status_by_short_name( + self.note_status_shortname + ) + if kitsu_status: + note_status = kitsu_status + self.log.info("Note Kitsu status: {}".format(note_status)) + else: + self.log.info( + "Cannot find {} status. The status will not be " + "changed!".format(self.note_status_shortname) + ) + + # Add comment to kitsu task + self.log.debug( + "Add new note in taks id {}".format( + context.data["kitsu_task"]["id"] + ) + ) + kitsu_comment = gazu.task.add_comment( + context.data["kitsu_task"], note_status, comment=publish_comment + ) + + context.data["kitsu_comment"] = kitsu_comment diff --git a/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py new file mode 100644 index 0000000000..bf80095225 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/integrate_kitsu_review.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +import gazu +import pyblish.api + + +class IntegrateKitsuReview(pyblish.api.InstancePlugin): + """Integrate Kitsu Review""" + + order = pyblish.api.IntegratorOrder + 0.01 + label = "Kitsu Review" + # families = ["kitsu"] + optional = True + + def process(self, instance): + + context = instance.context + task = context.data["kitsu_task"] + comment = context.data.get("kitsu_comment") + + # Check comment has been created + if not comment: + self.log.debug( + "Comment not created, review not pushed to preview." + ) + return + + # Add review representations as preview of comment + for representation in instance.data.get("representations", []): + # Skip if not tagged as review + if "review" not in representation.get("tags", []): + continue + + review_path = representation.get("published_path") + + self.log.debug("Found review at: {}".format(review_path)) + + gazu.task.add_preview( + task, comment, review_path, normalize_movie=True + ) + self.log.info("Review upload on comment") diff --git a/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py new file mode 100644 index 0000000000..c4a5b390e0 --- /dev/null +++ b/openpype/modules/kitsu/plugins/publish/other_kitsu_log_out.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +import gazu +import pyblish.api + + +class KitsuLogOut(pyblish.api.ContextPlugin): + """ + Log out from Kitsu API + """ + + order = pyblish.api.IntegratorOrder + 10 + label = "Kitsu Log Out" + + def process(self, context): + gazu.log_out() diff --git a/openpype/modules/kitsu/utils/__init__.py b/openpype/modules/kitsu/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpype/modules/kitsu/utils/credentials.py b/openpype/modules/kitsu/utils/credentials.py new file mode 100644 index 0000000000..0529380d6d --- /dev/null +++ b/openpype/modules/kitsu/utils/credentials.py @@ -0,0 +1,104 @@ +"""Kitsu credentials functions.""" + +import os +from typing import Tuple +import gazu + +from openpype.lib.local_settings import OpenPypeSecureRegistry + + +def validate_credentials( + login: str, password: str, kitsu_url: str = None +) -> bool: + """Validate credentials by trying to connect to Kitsu host URL. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + kitsu_url (str, optional): Kitsu host URL. Defaults to None. + + Returns: + bool: Are credentials valid? + """ + if kitsu_url is None: + kitsu_url = os.environ.get("KITSU_SERVER") + + # Connect to server + validate_host(kitsu_url) + + # Authenticate + try: + gazu.log_in(login, password) + except gazu.exception.AuthFailedException: + return False + + return True + + +def validate_host(kitsu_url: str) -> bool: + """Validate credentials by trying to connect to Kitsu host URL. + + Args: + kitsu_url (str, optional): Kitsu host URL. + + Returns: + bool: Is host valid? + """ + # Connect to server + gazu.set_host(kitsu_url) + + # Test host + if gazu.client.host_is_valid(): + return True + else: + raise gazu.exception.HostException(f"Host '{kitsu_url}' is invalid.") + + +def clear_credentials(): + """Clear credentials in Secure Registry.""" + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + # Set local settings + user_registry.delete_item("login") + user_registry.delete_item("password") + + +def save_credentials(login: str, password: str): + """Save credentials in Secure Registry. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + # Set local settings + user_registry.set_item("login", login) + user_registry.set_item("password", password) + + +def load_credentials() -> Tuple[str, str]: + """Load registered credentials. + + Returns: + Tuple[str, str]: (Login, Password) + """ + # Get user registry + user_registry = OpenPypeSecureRegistry("kitsu_user") + + return user_registry.get_item("login", None), user_registry.get_item( + "password", None + ) + + +def set_credentials_envs(login: str, password: str): + """Set environment variables with Kitsu login and password. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + os.environ["KITSU_LOGIN"] = login + os.environ["KITSU_PWD"] = password diff --git a/openpype/modules/kitsu/utils/sync_service.py b/openpype/modules/kitsu/utils/sync_service.py new file mode 100644 index 0000000000..6c003942f8 --- /dev/null +++ b/openpype/modules/kitsu/utils/sync_service.py @@ -0,0 +1,384 @@ +import os + +import gazu + +from openpype.pipeline import AvalonMongoDB +from .credentials import validate_credentials +from .update_op_with_zou import ( + create_op_asset, + set_op_project, + write_project_to_op, + update_op_assets, +) + + +class Listener: + """Host Kitsu listener.""" + + def __init__(self, login, password): + """Create client and add listeners to events without starting it. + + Run `listener.start()` to actually start the service. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + AuthFailedException: Wrong user login and/or password + """ + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + gazu.set_event_host( + os.environ["KITSU_SERVER"].replace("api", "socket.io") + ) + self.event_client = gazu.events.init() + + gazu.events.add_listener( + self.event_client, "project:new", self._new_project + ) + gazu.events.add_listener( + self.event_client, "project:update", self._update_project + ) + gazu.events.add_listener( + self.event_client, "project:delete", self._delete_project + ) + + gazu.events.add_listener( + self.event_client, "asset:new", self._new_asset + ) + gazu.events.add_listener( + self.event_client, "asset:update", self._update_asset + ) + gazu.events.add_listener( + self.event_client, "asset:delete", self._delete_asset + ) + + gazu.events.add_listener( + self.event_client, "episode:new", self._new_episode + ) + gazu.events.add_listener( + self.event_client, "episode:update", self._update_episode + ) + gazu.events.add_listener( + self.event_client, "episode:delete", self._delete_episode + ) + + gazu.events.add_listener( + self.event_client, "sequence:new", self._new_sequence + ) + gazu.events.add_listener( + self.event_client, "sequence:update", self._update_sequence + ) + gazu.events.add_listener( + self.event_client, "sequence:delete", self._delete_sequence + ) + + gazu.events.add_listener(self.event_client, "shot:new", self._new_shot) + gazu.events.add_listener( + self.event_client, "shot:update", self._update_shot + ) + gazu.events.add_listener( + self.event_client, "shot:delete", self._delete_shot + ) + + gazu.events.add_listener(self.event_client, "task:new", self._new_task) + gazu.events.add_listener( + self.event_client, "task:update", self._update_task + ) + gazu.events.add_listener( + self.event_client, "task:delete", self._delete_task + ) + + def start(self): + gazu.events.run_client(self.event_client) + + # == Project == + def _new_project(self, data): + """Create new project into OP DB.""" + + # Use update process to avoid duplicating code + self._update_project(data) + + def _update_project(self, data): + """Update project into OP DB.""" + # Get project entity + project = gazu.project.get_project(data["project_id"]) + project_name = project["name"] + + update_project = write_project_to_op(project, self.dbcon) + + # Write into DB + if update_project: + self.dbcon = self.dbcon.database[project_name] + self.dbcon.bulk_write([update_project]) + + def _delete_project(self, data): + """Delete project.""" + project_doc = self.dbcon.find_one( + {"type": "project", "data.zou_id": data["project_id"]} + ) + + # Delete project collection + self.dbcon.database[project_doc["name"]].drop() + + # == Asset == + + def _new_asset(self, data): + """Create new asset into OP DB.""" + # Get project entity + set_op_project(self.dbcon, data["project_id"]) + + # Get gazu entity + asset = gazu.asset.get_asset(data["asset_id"]) + + # Insert doc in DB + self.dbcon.insert_one(create_op_asset(asset)) + + # Update + self._update_asset(data) + + def _update_asset(self, data): + """Update asset into OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) + + # Get gazu entity + asset = gazu.asset.get_asset(data["asset_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in self.dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[asset["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + self.dbcon, project_doc, [asset], zou_ids_and_asset_docs + )[0] + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) + + def _delete_asset(self, data): + """Delete asset of OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + + # Delete + self.dbcon.delete_one( + {"type": "asset", "data.zou.id": data["asset_id"]} + ) + + # == Episode == + def _new_episode(self, data): + """Create new episode into OP DB.""" + # Get project entity + set_op_project(self.dbcon, data["project_id"]) + + # Get gazu entity + episode = gazu.shot.get_episode(data["episode_id"]) + + # Insert doc in DB + self.dbcon.insert_one(create_op_asset(episode)) + + # Update + self._update_episode(data) + + def _update_episode(self, data): + """Update episode into OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) + + # Get gazu entity + episode = gazu.shot.get_episode(data["episode_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in self.dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[episode["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + self.dbcon, project_doc, [episode], zou_ids_and_asset_docs + )[0] + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) + + def _delete_episode(self, data): + """Delete shot of OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + print("delete episode") # TODO check bugfix + + # Delete + self.dbcon.delete_one( + {"type": "asset", "data.zou.id": data["episode_id"]} + ) + + # == Sequence == + def _new_sequence(self, data): + """Create new sequnce into OP DB.""" + # Get project entity + set_op_project(self.dbcon, data["project_id"]) + + # Get gazu entity + sequence = gazu.shot.get_sequence(data["sequence_id"]) + + # Insert doc in DB + self.dbcon.insert_one(create_op_asset(sequence)) + + # Update + self._update_sequence(data) + + def _update_sequence(self, data): + """Update sequence into OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) + + # Get gazu entity + sequence = gazu.shot.get_sequence(data["sequence_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in self.dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[sequence["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + self.dbcon, project_doc, [sequence], zou_ids_and_asset_docs + )[0] + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) + + def _delete_sequence(self, data): + """Delete sequence of OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + print("delete sequence") # TODO check bugfix + + # Delete + self.dbcon.delete_one( + {"type": "asset", "data.zou.id": data["sequence_id"]} + ) + + # == Shot == + def _new_shot(self, data): + """Create new shot into OP DB.""" + # Get project entity + set_op_project(self.dbcon, data["project_id"]) + + # Get gazu entity + shot = gazu.shot.get_shot(data["shot_id"]) + + # Insert doc in DB + self.dbcon.insert_one(create_op_asset(shot)) + + # Update + self._update_shot(data) + + def _update_shot(self, data): + """Update shot into OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + project_doc = self.dbcon.find_one({"type": "project"}) + + # Get gazu entity + shot = gazu.shot.get_shot(data["shot_id"]) + + # Find asset doc + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in self.dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[shot["project_id"]] = project_doc + + # Update + asset_doc_id, asset_update = update_op_assets( + self.dbcon, project_doc, [shot], zou_ids_and_asset_docs + )[0] + self.dbcon.update_one({"_id": asset_doc_id}, asset_update) + + def _delete_shot(self, data): + """Delete shot of OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + + # Delete + self.dbcon.delete_one( + {"type": "asset", "data.zou.id": data["shot_id"]} + ) + + # == Task == + def _new_task(self, data): + """Create new task into OP DB.""" + # Get project entity + set_op_project(self.dbcon, data["project_id"]) + + # Get gazu entity + task = gazu.task.get_task(data["task_id"]) + + # Find asset doc + asset_doc = self.dbcon.find_one( + {"type": "asset", "data.zou.id": task["entity"]["id"]} + ) + + # Update asset tasks with new one + asset_tasks = asset_doc["data"].get("tasks") + task_type_name = task["task_type"]["name"] + asset_tasks[task_type_name] = {"type": task_type_name, "zou": task} + self.dbcon.update_one( + {"_id": asset_doc["_id"]}, {"$set": {"data.tasks": asset_tasks}} + ) + + def _update_task(self, data): + """Update task into OP DB.""" + # TODO is it necessary? + pass + + def _delete_task(self, data): + """Delete task of OP DB.""" + set_op_project(self.dbcon, data["project_id"]) + + # Find asset doc + asset_docs = [doc for doc in self.dbcon.find({"type": "asset"})] + for doc in asset_docs: + # Match task + for name, task in doc["data"]["tasks"].items(): + if task.get("zou") and data["task_id"] == task["zou"]["id"]: + # Pop task + asset_tasks = doc["data"].get("tasks", {}) + asset_tasks.pop(name) + + # Delete task in DB + self.dbcon.update_one( + {"_id": doc["_id"]}, + {"$set": {"data.tasks": asset_tasks}}, + ) + return + + +def start_listeners(login: str, password: str): + """Start listeners to keep OpenPype up-to-date with Kitsu. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + """ + + # Connect to server + listener = Listener(login, password) + listener.start() diff --git a/openpype/modules/kitsu/utils/update_op_with_zou.py b/openpype/modules/kitsu/utils/update_op_with_zou.py new file mode 100644 index 0000000000..673a195747 --- /dev/null +++ b/openpype/modules/kitsu/utils/update_op_with_zou.py @@ -0,0 +1,389 @@ +"""Functions to update OpenPype data using Kitsu DB (a.k.a Zou).""" +from copy import deepcopy +import re +from typing import Dict, List + +from pymongo import DeleteOne, UpdateOne +import gazu +from gazu.task import ( + all_tasks_for_asset, + all_tasks_for_shot, +) + +from openpype.pipeline import AvalonMongoDB +from openpype.api import get_project_settings +from openpype.lib import create_project +from openpype.modules.kitsu.utils.credentials import validate_credentials + + +# Accepted namin pattern for OP +naming_pattern = re.compile("^[a-zA-Z0-9_.]*$") + + +def create_op_asset(gazu_entity: dict) -> dict: + """Create OP asset dict from gazu entity. + + :param gazu_entity: + """ + return { + "name": gazu_entity["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": {"zou": gazu_entity, "tasks": {}}, + } + + +def set_op_project(dbcon: AvalonMongoDB, project_id: str): + """Set project context. + + Args: + dbcon (AvalonMongoDB): Connection to DB + project_id (str): Project zou ID + """ + project = gazu.project.get_project(project_id) + project_name = project["name"] + dbcon.Session["AVALON_PROJECT"] = project_name + + +def update_op_assets( + dbcon: AvalonMongoDB, + project_doc: dict, + entities_list: List[dict], + asset_doc_ids: Dict[str, dict], +) -> List[Dict[str, dict]]: + """Update OpenPype assets. + Set 'data' and 'parent' fields. + + Args: + dbcon (AvalonMongoDB): Connection to DB + entities_list (List[dict]): List of zou entities to update + asset_doc_ids (Dict[str, dict]): Dicts of [{zou_id: asset_doc}, ...] + + Returns: + List[Dict[str, dict]]: List of (doc_id, update_dict) tuples + """ + project_name = project_doc["name"] + project_module_settings = get_project_settings(project_name)["kitsu"] + + assets_with_update = [] + for item in entities_list: + # Check asset exists + item_doc = asset_doc_ids.get(item["id"]) + if not item_doc: # Create asset + op_asset = create_op_asset(item) + insert_result = dbcon.insert_one(op_asset) + item_doc = dbcon.find_one( + {"type": "asset", "_id": insert_result.inserted_id} + ) + + # Update asset + item_data = deepcopy(item_doc["data"]) + item_data.update(item.get("data") or {}) + item_data["zou"] = item + + # == Asset settings == + # Frame in, fallback on 0 + frame_in = int(item_data.get("frame_in") or 0) + item_data["frameStart"] = frame_in + item_data.pop("frame_in") + # Frame out, fallback on frame_in + duration + frames_duration = int(item.get("nb_frames") or 1) + frame_out = ( + item_data["frame_out"] + if item_data.get("frame_out") + else frame_in + frames_duration + ) + item_data["frameEnd"] = int(frame_out) + item_data.pop("frame_out") + # Fps, fallback to project's value when entity fps is deleted + if not item_data.get("fps") and item_doc["data"].get("fps"): + item_data["fps"] = project_doc["data"]["fps"] + + # Tasks + tasks_list = [] + item_type = item["type"] + if item_type == "Asset": + tasks_list = all_tasks_for_asset(item) + elif item_type == "Shot": + tasks_list = all_tasks_for_shot(item) + # TODO frame in and out + item_data["tasks"] = { + t["task_type_name"]: {"type": t["task_type_name"]} + for t in tasks_list + } + + # Get zou parent id for correct hierarchy + # Use parent substitutes if existing + substitute_parent_item = ( + item_data["parent_substitutes"][0] + if item_data.get("parent_substitutes") + else None + ) + if substitute_parent_item: + parent_zou_id = substitute_parent_item["parent_id"] + else: + parent_zou_id = ( + item.get("parent_id") + or item.get("episode_id") + or item.get("source_id") + ) # TODO check consistency + + # Substitute Episode and Sequence by Shot + substitute_item_type = ( + "shots" + if item_type in ["Episode", "Sequence"] + else f"{item_type.lower()}s" + ) + entity_parent_folders = [ + f + for f in project_module_settings["entities_root"] + .get(substitute_item_type) + .split("/") + if f + ] + + # Root parent folder if exist + visual_parent_doc_id = ( + asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None + ) + if visual_parent_doc_id is None: + # Find root folder doc + root_folder_doc = dbcon.find_one( + { + "type": "asset", + "name": entity_parent_folders[-1], + "data.root_of": substitute_item_type, + }, + ["_id"], + ) + if root_folder_doc: + visual_parent_doc_id = root_folder_doc["_id"] + + # Visual parent for hierarchy + item_data["visualParent"] = visual_parent_doc_id + + # Add parents for hierarchy + item_data["parents"] = [] + while parent_zou_id is not None: + parent_doc = asset_doc_ids[parent_zou_id] + item_data["parents"].insert(0, parent_doc["name"]) + + # Get parent entity + parent_entity = parent_doc["data"]["zou"] + parent_zou_id = parent_entity["parent_id"] + + # Set root folders parents + item_data["parents"] = entity_parent_folders + item_data["parents"] + + # Update 'data' different in zou DB + updated_data = { + k: v for k, v in item_data.items() if item_doc["data"].get(k) != v + } + if updated_data or not item_doc.get("parent"): + assets_with_update.append( + ( + item_doc["_id"], + { + "$set": { + "name": item["name"], + "data": item_data, + "parent": asset_doc_ids[item["project_id"]]["_id"], + } + }, + ) + ) + + return assets_with_update + + +def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne: + """Write gazu project to OP database. + Create project if doesn't exist. + + Args: + project (dict): Gazu project + dbcon (AvalonMongoDB): DB to create project in + + Returns: + UpdateOne: Update instance for the project + """ + project_name = project["name"] + project_doc = dbcon.database[project_name].find_one({"type": "project"}) + if not project_doc: + print(f"Creating project '{project_name}'") + project_doc = create_project(project_name, project_name, dbcon=dbcon) + + # Project data and tasks + project_data = project["data"] or {} + + # Build project code and update Kitsu + project_code = project.get("code") + if not project_code: + project_code = project["name"].replace(" ", "_").lower() + project["code"] = project_code + + # Update Zou + gazu.project.update_project(project) + + # Update data + project_data.update( + { + "code": project_code, + "fps": project["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + "zou_id": project["id"], + } + ) + + return UpdateOne( + {"_id": project_doc["_id"]}, + { + "$set": { + "config.tasks": { + t["name"]: {"short_name": t.get("short_name", t["name"])} + for t in gazu.task.all_task_types_for_project(project) + }, + "data": project_data, + } + }, + ) + + +def sync_all_project(login: str, password: str): + """Update all OP projects in DB with Zou data. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + gazu.exception.AuthFailedException: Wrong user login and/or password + """ + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + all_projects = gazu.project.all_open_projects() + for project in all_projects: + sync_project_from_kitsu(dbcon, project) + + +def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict): + """Update OP project in DB with Zou data. + + Args: + dbcon (AvalonMongoDB): MongoDB connection + project (dict): Project dict got using gazu. + """ + bulk_writes = [] + + # Get project from zou + if not project: + project = gazu.project.get_project_by_name(project["name"]) + + print(f"Synchronizing {project['name']}...") + + # Get all assets from zou + all_assets = gazu.asset.all_assets_for_project(project) + all_episodes = gazu.shot.all_episodes_for_project(project) + all_seqs = gazu.shot.all_sequences_for_project(project) + all_shots = gazu.shot.all_shots_for_project(project) + all_entities = [ + item + for item in all_assets + all_episodes + all_seqs + all_shots + if naming_pattern.match(item["name"]) + ] + + # Sync project. Create if doesn't exist + bulk_writes.append(write_project_to_op(project, dbcon)) + + # Try to find project document + dbcon.Session["AVALON_PROJECT"] = project["name"] + project_doc = dbcon.find_one({"type": "project"}) + + # Query all assets of the local project + zou_ids_and_asset_docs = { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou", {}).get("id") + } + zou_ids_and_asset_docs[project["id"]] = project_doc + + # Create entities root folders + project_module_settings = get_project_settings(project["name"])["kitsu"] + for entity_type, root in project_module_settings["entities_root"].items(): + parent_folders = root.split("/") + direct_parent_doc = None + for i, folder in enumerate(parent_folders, 1): + parent_doc = dbcon.find_one( + {"type": "asset", "name": folder, "data.root_of": entity_type} + ) + if not parent_doc: + direct_parent_doc = dbcon.insert_one( + { + "name": folder, + "type": "asset", + "schema": "openpype:asset-3.0", + "data": { + "root_of": entity_type, + "parents": parent_folders[:i], + "visualParent": direct_parent_doc, + "tasks": {}, + }, + } + ) + + # Create + to_insert = [] + to_insert.extend( + [ + create_op_asset(item) + for item in all_entities + if item["id"] not in zou_ids_and_asset_docs.keys() + ] + ) + if to_insert: + # Insert doc in DB + dbcon.insert_many(to_insert) + + # Update existing docs + zou_ids_and_asset_docs.update( + { + asset_doc["data"]["zou"]["id"]: asset_doc + for asset_doc in dbcon.find({"type": "asset"}) + if asset_doc["data"].get("zou") + } + ) + + # Update + bulk_writes.extend( + [ + UpdateOne({"_id": id}, update) + for id, update in update_op_assets( + dbcon, project_doc, all_entities, zou_ids_and_asset_docs + ) + ] + ) + + # Delete + diff_assets = set(zou_ids_and_asset_docs.keys()) - { + e["id"] for e in all_entities + [project] + } + if diff_assets: + bulk_writes.extend( + [ + DeleteOne(zou_ids_and_asset_docs[asset_id]) + for asset_id in diff_assets + ] + ) + + # Write into DB + if bulk_writes: + dbcon.bulk_write(bulk_writes) diff --git a/openpype/modules/kitsu/utils/update_zou_with_op.py b/openpype/modules/kitsu/utils/update_zou_with_op.py new file mode 100644 index 0000000000..81d421206f --- /dev/null +++ b/openpype/modules/kitsu/utils/update_zou_with_op.py @@ -0,0 +1,262 @@ +"""Functions to update Kitsu DB (a.k.a Zou) using OpenPype Data.""" + +import re +from typing import List + +import gazu +from pymongo import UpdateOne + +from openpype.pipeline import AvalonMongoDB +from openpype.api import get_project_settings +from openpype.modules.kitsu.utils.credentials import validate_credentials + + +def sync_zou(login: str, password: str): + """Synchronize Zou database (Kitsu backend) with openpype database. + This is an utility function to help updating zou data with OP's, it may not + handle correctly all cases, a human intervention might + be required after all. + Will work better if OP DB has been previously synchronized from zou/kitsu. + + Args: + login (str): Kitsu user login + password (str): Kitsu user password + + Raises: + gazu.exception.AuthFailedException: Wrong user login and/or password + """ + + # Authenticate + if not validate_credentials(login, password): + raise gazu.exception.AuthFailedException( + f"Kitsu authentication failed for login: '{login}'..." + ) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + + op_projects = [p for p in dbcon.projects()] + for project_doc in op_projects: + sync_zou_from_op_project(project_doc["name"], dbcon, project_doc) + + +def sync_zou_from_op_project( + project_name: str, dbcon: AvalonMongoDB, project_doc: dict = None +) -> List[UpdateOne]: + """Update OP project in DB with Zou data. + + Args: + project_name (str): Name of project to sync + dbcon (AvalonMongoDB): MongoDB connection + project_doc (str, optional): Project doc to sync + """ + # Get project doc if not provided + if not project_doc: + project_doc = dbcon.database[project_name].find_one( + {"type": "project"} + ) + + # Get all entities from zou + print(f"Synchronizing {project_name}...") + zou_project = gazu.project.get_project_by_name(project_name) + + # Create project + if zou_project is None: + raise RuntimeError( + f"Project '{project_name}' doesn't exist in Zou database, " + "please create it in Kitsu and add OpenPype user to it before " + "running synchronization." + ) + + # Update project settings and data + if project_doc["data"]: + zou_project.update( + { + "code": project_doc["data"]["code"], + "fps": project_doc["data"]["fps"], + "resolution": f"{project_doc['data']['resolutionWidth']}" + f"x{project_doc['data']['resolutionHeight']}", + } + ) + gazu.project.update_project_data(zou_project, data=project_doc["data"]) + gazu.project.update_project(zou_project) + + asset_types = gazu.asset.all_asset_types() + all_assets = gazu.asset.all_assets_for_project(zou_project) + all_episodes = gazu.shot.all_episodes_for_project(zou_project) + all_seqs = gazu.shot.all_sequences_for_project(zou_project) + all_shots = gazu.shot.all_shots_for_project(zou_project) + all_entities_ids = { + e["id"] for e in all_episodes + all_seqs + all_shots + all_assets + } + + # Query all assets of the local project + project_module_settings = get_project_settings(project_name)["kitsu"] + dbcon.Session["AVALON_PROJECT"] = project_name + asset_docs = { + asset_doc["_id"]: asset_doc + for asset_doc in dbcon.find({"type": "asset"}) + } + + # Create new assets + new_assets_docs = [ + doc + for doc in asset_docs.values() + if doc["data"].get("zou", {}).get("id") not in all_entities_ids + ] + naming_pattern = project_module_settings["entities_naming_pattern"] + regex_ep = re.compile( + r"(.*{}.*)|(.*{}.*)|(.*{}.*)".format( + naming_pattern["shot"].replace("#", ""), + naming_pattern["sequence"].replace("#", ""), + naming_pattern["episode"].replace("#", ""), + ), + re.IGNORECASE, + ) + bulk_writes = [] + for doc in new_assets_docs: + visual_parent_id = doc["data"]["visualParent"] + parent_substitutes = [] + + # Match asset type by it's name + match = regex_ep.match(doc["name"]) + if not match: # Asset + new_entity = gazu.asset.new_asset( + zou_project, asset_types[0], doc["name"] + ) + # Match case in shot