Updated Kitsu Sync module to fully work with all events

Manually tried all events, added logging for events and cleaned up the code some.
This commit is contained in:
Jacob Danell 2023-02-10 13:32:03 +01:00 committed by Milan Kolar
parent bb61d43c27
commit 47caa76030
2 changed files with 248 additions and 87 deletions

View file

@ -1,3 +1,15 @@
"""
Bugs:
* Error when adding task type to anything that isn't Shot or Assets
* Assets don't get added under an episode if TV show
* Assets added under Main Pack throws error. Can't get the name of Main Pack?
Features ToDo:
* Select in settings what types you wish to sync
* Print what's updated on entity-update
* Add listener for Edits
"""
import os
import threading
@ -5,6 +17,7 @@ import gazu
from openpype.client import get_project, get_assets, get_asset_by_name
from openpype.pipeline import AvalonMongoDB
from openpype.lib import Logger
from .credentials import validate_credentials
from .update_op_with_zou import (
create_op_asset,
@ -14,6 +27,8 @@ from .update_op_with_zou import (
update_op_assets,
)
log = Logger.get_logger(__name__)
class Listener:
"""Host Kitsu listener."""
@ -33,7 +48,7 @@ class Listener:
self.dbcon = AvalonMongoDB()
self.dbcon.install()
gazu.client.set_host(os.environ["KITSU_SERVER"])
gazu.client.set_host(os.environ['KITSU_SERVER'])
# Authenticate
if not validate_credentials(login, password):
@ -42,7 +57,7 @@ class Listener:
)
gazu.set_event_host(
os.environ["KITSU_SERVER"].replace("api", "socket.io")
os.environ['KITSU_SERVER'].replace("api", "socket.io")
)
self.event_client = gazu.events.init()
@ -103,6 +118,8 @@ class Listener:
)
def start(self):
"""Start listening for events."""
log.info("Listening to Kitsu events...")
gazu.events.run_client(self.event_client)
# == Project ==
@ -112,36 +129,49 @@ class Listener:
# Use update process to avoid duplicating code
self._update_project(data)
# Print message
## Happens in write_project_to_op()
def _update_project(self, data):
"""Update project into OP DB."""
# Get project entity
project = gazu.project.get_project(data["project_id"])
project_name = project["name"]
project = gazu.project.get_project(data['project_id'])
update_project = write_project_to_op(project, self.dbcon)
# Write into DB
if update_project:
self.dbcon.Session["AVALON_PROJECT"] = project_name
self.dbcon.Session['AVALON_PROJECT'] = get_kitsu_project_name(
data['project_id'])
self.dbcon.bulk_write([update_project])
def _delete_project(self, data):
"""Delete project."""
project_name = get_kitsu_project_name(data["project_id"])
collections = self.dbcon.database.list_collection_names()
project_name = None
for collection in collections:
post = self.dbcon.database[collection].find_one(
{"data.zou_id": data['project_id']})
if post:
project_name = post['name']
break
# Delete project collection
self.dbcon.database[project_name].drop()
if project_name:
# Delete project collection
self.dbcon.database[project_name].drop()
# Print message
log.info(f"Project deleted: {project_name}")
# == Asset ==
def _new_asset(self, data):
"""Create new asset into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
# Get gazu entity
asset = gazu.asset.get_asset(data["asset_id"])
# Get asset entity
asset = gazu.asset.get_asset(data['asset_id'])
# Insert doc in DB
self.dbcon.insert_one(create_op_asset(asset))
@ -149,27 +179,43 @@ class Listener:
# Update
self._update_asset(data)
# Print message
episode = None
ep_id = asset['episode_id']
if ep_id and ep_id != "":
episode = gazu.asset.get_episode(ep_id)
msg = "Asset created: "
msg = msg + f"{asset['project_name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{asset['asset_type_name']}_"
msg = msg + f"{asset['name']}"
log.info(msg)
def _update_asset(self, data):
"""Update asset into OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
asset = gazu.asset.get_asset(data["asset_id"])
asset = gazu.asset.get_asset(data['asset_id'])
# Find asset doc
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
asset_doc['data']['zou']['id']: asset_doc
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
if asset_doc['data'].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[asset["project_id"]] = project_doc
zou_ids_and_asset_docs[asset['project_id']] = project_doc
gazu_project = gazu.project.get_project(asset['project_id'])
# Update
update_op_result = update_op_assets(
self.dbcon, project_doc, [asset], zou_ids_and_asset_docs
self.dbcon, gazu_project, project_doc,
[asset], zou_ids_and_asset_docs
)
if update_op_result:
asset_doc_id, asset_update = update_op_result[0]
@ -177,21 +223,37 @@ class Listener:
def _delete_asset(self, data):
"""Delete asset of OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data["asset_id"]}
)
asset = self.dbcon.find_one({"data.zou.id": data['asset_id']})
if asset:
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data['asset_id']}
)
# Print message
episode = None
ep_id = asset['data']['zou']['episode_id']
if ep_id and ep_id != "":
episode = gazu.asset.get_episode(ep_id)
msg = "Asset deleted: "
msg = msg + f"{asset['data']['zou']['project_name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{asset['data']['zou']['asset_type_name']}_"
msg = msg + f"'{asset['name']}"
log.info(msg)
# == Episode ==
def _new_episode(self, data):
"""Create new episode into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
# Get gazu entity
episode = gazu.shot.get_episode(data["episode_id"])
episode = gazu.shot.get_episode(data['episode_id'])
# Insert doc in DB
self.dbcon.insert_one(create_op_asset(episode))
@ -199,27 +261,34 @@ class Listener:
# Update
self._update_episode(data)
# Print message
msg = "Episode created: "
msg = msg + f"{episode['project_name']} - "
msg = msg + f"{episode['name']}"
def _update_episode(self, data):
"""Update episode into OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
episode = gazu.shot.get_episode(data["episode_id"])
episode = gazu.shot.get_episode(data['episode_id'])
# Find asset doc
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
asset_doc['data']['zou']['id']: asset_doc
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
if asset_doc['data'].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[episode["project_id"]] = project_doc
zou_ids_and_asset_docs[episode['project_id']] = project_doc
gazu_project = gazu.project.get_project(episode['project_id'])
# Update
update_op_result = update_op_assets(
self.dbcon, project_doc, [episode], zou_ids_and_asset_docs
self.dbcon, gazu_project, project_doc, [
episode], zou_ids_and_asset_docs
)
if update_op_result:
asset_doc_id, asset_update = update_op_result[0]
@ -227,22 +296,31 @@ class Listener:
def _delete_episode(self, data):
"""Delete shot of OP DB."""
set_op_project(self.dbcon, data["project_id"])
print("delete episode") # TODO check bugfix
set_op_project(self.dbcon, data['project_id'])
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data["episode_id"]}
)
episode = self.dbcon.find_one({"data.zou.id": data['episode_id']})
if episode:
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data['episode_id']}
)
# Print message
project = gazu.project.get_project(
episode['data']['zou']['project_id'])
msg = "Episode deleted: "
msg = msg + f"{project['name']} - "
msg = msg + f"{episode['name']}"
# == Sequence ==
def _new_sequence(self, data):
"""Create new sequnce into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
# Get gazu entity
sequence = gazu.shot.get_sequence(data["sequence_id"])
sequence = gazu.shot.get_sequence(data['sequence_id'])
# Insert doc in DB
self.dbcon.insert_one(create_op_asset(sequence))
@ -250,27 +328,43 @@ class Listener:
# Update
self._update_sequence(data)
# Print message
episode = None
ep_id = sequence['episode_id']
if ep_id and ep_id != "":
episode = gazu.asset.get_episode(ep_id)
msg = "Sequence created: "
msg = msg + f"{sequence['project_name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{sequence['name']}"
log.info(msg)
def _update_sequence(self, data):
"""Update sequence into OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
sequence = gazu.shot.get_sequence(data["sequence_id"])
sequence = gazu.shot.get_sequence(data['sequence_id'])
# Find asset doc
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
asset_doc['data']['zou']['id']: asset_doc
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
if asset_doc['data'].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[sequence["project_id"]] = project_doc
zou_ids_and_asset_docs[sequence['project_id']] = project_doc
gazu_project = gazu.project.get_project(sequence['project_id'])
# Update
update_op_result = update_op_assets(
self.dbcon, project_doc, [sequence], zou_ids_and_asset_docs
self.dbcon, gazu_project, project_doc,
[sequence], zou_ids_and_asset_docs
)
if update_op_result:
asset_doc_id, asset_update = update_op_result[0]
@ -278,22 +372,30 @@ class Listener:
def _delete_sequence(self, data):
"""Delete sequence of OP DB."""
set_op_project(self.dbcon, data["project_id"])
print("delete sequence") # TODO check bugfix
set_op_project(self.dbcon, data['project_id'])
sequence = self.dbcon.find_one({"data.zou.id": data['sequence_id']})
if sequence:
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data['sequence_id']}
)
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data["sequence_id"]}
)
# Print message
gazu_project = gazu.project.get_project(
sequence['data']['zou']['project_id'])
msg = f"Sequence deleted: "
msg = msg + f"{gazu_project['name']} - "
msg = msg + f"{sequence['name']}"
log.info(msg)
# == Shot ==
def _new_shot(self, data):
"""Create new shot into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
# Get gazu entity
shot = gazu.shot.get_shot(data["shot_id"])
shot = gazu.shot.get_shot(data['shot_id'])
# Insert doc in DB
self.dbcon.insert_one(create_op_asset(shot))
@ -301,89 +403,151 @@ class Listener:
# Update
self._update_shot(data)
# Print message
episode = None
if shot['episode_id'] and shot['episode_id'] != "":
episode = gazu.asset.get_episode(shot['episode_id'])
msg = "Shot created: "
msg = msg + f"{shot['project_name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{shot['sequence_name']}_"
msg = msg + f"{shot['name']}"
log.info(msg)
def _update_shot(self, data):
"""Update shot into OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
shot = gazu.shot.get_shot(data["shot_id"])
shot = gazu.shot.get_shot(data['shot_id'])
# Find asset doc
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
asset_doc['data']['zou']['id']: asset_doc
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
if asset_doc['data'].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[shot["project_id"]] = project_doc
zou_ids_and_asset_docs[shot['project_id']] = project_doc
gazu_project = gazu.project.get_project(shot['project_id'])
# Update
update_op_result = update_op_assets(
self.dbcon, project_doc, [shot], zou_ids_and_asset_docs
self.dbcon, gazu_project, project_doc,
[shot], zou_ids_and_asset_docs
)
if update_op_result:
asset_doc_id, asset_update = update_op_result[0]
self.dbcon.update_one({"_id": asset_doc_id}, asset_update)
def _delete_shot(self, data):
"""Delete shot of OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
shot = self.dbcon.find_one({"data.zou.id": data['shot_id']})
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data["shot_id"]}
)
if shot:
# Delete
self.dbcon.delete_one(
{"type": "asset", "data.zou.id": data['shot_id']}
)
# Print message
gazu_project = gazu.project.get_project(
shot['data']['zou']['project_id'])
msg = "Shot deleted: "
msg = msg + f"{gazu_project['name']} - "
msg = msg + f"{shot['name']}"
log.info(msg)
# == Task ==
def _new_task(self, data):
"""Create new task into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
# Get gazu entity
task = gazu.task.get_task(data["task_id"])
task = gazu.task.get_task(data['task_id'])
# Find asset doc
parent_name = task["entity"]["name"]
episode = None
ep_id = task['episode_id']
if ep_id and ep_id != "":
episode = gazu.asset.get_episode(ep_id)
parent_name = ""
if episode is not None:
parent_name = episode['name'] + "_"
parent_name = parent_name + \
task['sequence']['name'] + "_" + task['entity']['name']
asset_doc = get_asset_by_name(project_name, parent_name)
# Update asset tasks with new one
asset_tasks = asset_doc["data"].get("tasks")
task_type_name = task["task_type"]["name"]
asset_tasks = asset_doc['data'].get("tasks")
task_type_name = task['task_type']['name']
asset_tasks[task_type_name] = {"type": task_type_name, "zou": task}
self.dbcon.update_one(
{"_id": asset_doc["_id"]}, {"$set": {"data.tasks": asset_tasks}}
{"_id": asset_doc['_id']}, {"$set": {"data.tasks": asset_tasks}}
)
# Print message
msg = "Task created: "
msg = msg + f"{task['project']['name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{task['sequence']['name']}_"
msg = msg + f"{task['entity']['name']} - "
msg = msg + f"{task['task_type']['name']}"
log.info(msg)
def _update_task(self, data):
"""Update task into OP DB."""
# TODO is it necessary?
pass
def _delete_task(self, data):
"""Delete task of OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data['project_id'])
project_name = self.dbcon.active_project()
# Find asset doc
asset_docs = list(get_assets(project_name))
for doc in asset_docs:
# Match task
for name, task in doc["data"]["tasks"].items():
if task.get("zou") and data["task_id"] == task["zou"]["id"]:
for name, task in doc['data']['tasks'].items():
if task.get("zou") and data['task_id'] == task['zou']['id']:
# Pop task
asset_tasks = doc["data"].get("tasks", {})
asset_tasks = doc['data'].get("tasks", {})
asset_tasks.pop(name)
# Delete task in DB
self.dbcon.update_one(
{"_id": doc["_id"]},
{"_id": doc['_id']},
{"$set": {"data.tasks": asset_tasks}},
)
# Print message
shot = gazu.shot.get_shot(task['zou']['entity_id'])
episode = None
ep_id = shot['episode_id']
if ep_id and ep_id != "":
episode = gazu.asset.get_episode(ep_id)
msg = "Task deleted: "
msg = msg + f"{shot['project_name']} - "
if episode is not None:
msg = msg + f"{episode['name']}_"
msg = msg + f"{shot['sequence_name']}_"
msg = msg + f"{shot['name']} - "
msg = msg + f"{task['type']}"
log.info(msg)
return
@ -396,7 +560,7 @@ def start_listeners(login: str, password: str):
"""
# Refresh token every week
def refresh_token_every_week():
print("Refreshing token...")
log.info("Refreshing token...")
gazu.refresh_token()
threading.Timer(7 * 3600 * 24, refresh_token_every_week).start()

View file

@ -5,10 +5,6 @@ from typing import Dict, List
from pymongo import DeleteOne, UpdateOne
import gazu
from gazu.task import (
all_tasks_for_asset,
all_tasks_for_shot,
)
from openpype.client import (
get_project,
@ -18,7 +14,6 @@ from openpype.client import (
create_project,
)
from openpype.pipeline import AvalonMongoDB
from openpype.settings import get_project_settings
from openpype.modules.kitsu.utils.credentials import validate_credentials
from openpype.lib import Logger
@ -85,8 +80,10 @@ def update_op_assets(
Returns:
List[Dict[str, dict]]: List of (doc_id, update_dict) tuples
"""
if not project_doc:
return
project_name = project_doc["name"]
project_module_settings = get_project_settings(project_name)["kitsu"]
assets_with_update = []
for item in entities_list:
@ -170,9 +167,9 @@ def update_op_assets(
tasks_list = []
item_type = item["type"]
if item_type == "Asset":
tasks_list = all_tasks_for_asset(item)
tasks_list = gazu.task.all_tasks_for_asset(item)
elif item_type == "Shot":
tasks_list = all_tasks_for_shot(item)
tasks_list = gazu.task.all_tasks_for_shot(item)
item_data["tasks"] = {
t["task_type_name"]: {"type": t["task_type_name"], "zou": t}
for t in tasks_list
@ -207,7 +204,7 @@ def update_op_assets(
# Root parent folder if exist
visual_parent_doc_id = (
asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None
asset_doc_ids[parent_zou_id].get("_id") if parent_zou_id else None
)
if visual_parent_doc_id is None:
# Find root folder doc ("Assets" or "Shots")
@ -282,7 +279,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne:
project_name = project["name"]
project_doc = get_project(project_name)
if not project_doc:
log.info(f"Creating project '{project_name}'")
log.info(f"Project created: {project_name}")
project_doc = create_project(project_name, project_name)
# Project data and tasks