diff --git a/openpype/modules/default_modules/kitsu/kitsu_module.py b/openpype/modules/default_modules/kitsu/kitsu_module.py index e52d18b84b..c4f627d5ad 100644 --- a/openpype/modules/default_modules/kitsu/kitsu_module.py +++ b/openpype/modules/default_modules/kitsu/kitsu_module.py @@ -6,12 +6,14 @@ in global space here until are required or used. - imports of Python 3 packages - we still support Python 2 hosts where addon definition should available """ -import os -from typing import Dict, List, Set import click +import os +import re +from typing import Dict, List from avalon.api import AvalonMongoDB import gazu +from openpype.api import get_project_settings from openpype.lib import create_project from openpype.modules import JsonFilesSettingsDef, OpenPypeModule, ModulesManager from pymongo import DeleteOne, UpdateOne @@ -145,8 +147,8 @@ def cli_main(): @cli_main.command() -def sync_openpype(): - """Synchronize openpype database from Zou sever database.""" +def sync_zou(): + """Synchronize Zou server database (Kitsu backend) with openpype database.""" # Connect to server gazu.client.set_host(os.environ["KITSU_SERVER"]) @@ -157,8 +159,108 @@ def sync_openpype(): # Iterate projects dbcon = AvalonMongoDB() dbcon.install() - all_projects = gazu.project.all_projects() + + op_projects = [p for p in dbcon.projects()] bulk_writes = [] + for op_project in op_projects: + # Create project locally + # Try to find project document + project_name = op_project["name"] + project_code = op_project["data"]["code"] + dbcon.Session["AVALON_PROJECT"] = project_name + + # Get all entities from zou + zou_project = gazu.project.get_project_by_name(project_name) + + # Create project + if zou_project is None: + raise RuntimeError( + f"Project '{project_name}' doesn't exist in Zou database, please create it in Kitsu and add logged user to it before running synchronization." + ) + + # Update project settings and data + zou_project.update( + { + "code": op_project["data"]["code"], + "fps": op_project["data"]["fps"], + "resolution": f"{op_project['data']['resolutionWidth']}x{op_project['data']['resolutionHeight']}", + } + ) + gazu.project.update_project(zou_project) + gazu.project.update_project_data(zou_project, data=op_project["data"]) + + all_assets = gazu.asset.all_assets_for_project(zou_project) + all_episodes = gazu.shot.all_episodes_for_project(zou_project) + all_seqs = gazu.shot.all_sequences_for_project(zou_project) + all_shots = gazu.shot.all_shots_for_project(zou_project) + print(zou_project["name"]) + all_entities_ids = { + e["id"] for e in all_episodes + all_seqs + all_shots + all_assets + } + + project_module_settings = get_project_settings(project_name)["kitsu"] + + # Create new assets + # Query all assets of the local project + project_col = dbcon.database[project_name] + asset_docs = [asset_doc for asset_doc in project_col.find({"type": "asset"})] + + new_assets_docs = [ + doc + for doc in asset_docs + if doc["data"].get("zou_id") not in all_entities_ids + ] + naming_pattern = project_module_settings["entities_naming_pattern"] + regex_ep = re.compile( + r"({})|({})|({})".format( + naming_pattern["episode"].replace("#", "\d"), + naming_pattern["sequence"].replace("#", "\d"), + naming_pattern["shot"].replace("#", "\d"), + ), + re.IGNORECASE, + ) + for doc in new_assets_docs: + match = regex_ep.match(doc["name"]) + if not match: + # TODO asset + continue + + print(doc) + if match.group(1): # Episode + new_episode = gazu.shot.new_episode(zou_project, doc["name"]) + + # Update doc with zou id + bulk_writes.append( + UpdateOne( + {"_id": doc["_id"]}, + {"$set": {"data.zou_id": new_episode["id"]}}, + ) + ) + elif match.group(2): # Sequence + # TODO match zou episode + new_sequence = gazu.shot.new_sequence(zou_project, doc["name"]) + + # Update doc with zou id + bulk_writes.append( + UpdateOne( + {"_id": doc["_id"]}, + {"$set": {"data.zou_id": new_sequence["id"]}}, + ) + ) + elif match.group(3): # Shot + pass + + # Delete + # if gazu. + + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) + + dbcon.uninstall() + + return + for project in all_projects: # Create project locally # Try to find project document @@ -245,9 +347,117 @@ def sync_openpype(): [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] ) - # Write into DB - if bulk_writes: - project_col.bulk_write(bulk_writes) + +@cli_main.command() +def sync_openpype(): + """Synchronize openpype database from Zou sever database.""" + + # Connect to server + gazu.client.set_host(os.environ["KITSU_SERVER"]) + + # Authenticate + gazu.log_in(os.environ["KITSU_LOGIN"], os.environ["KITSU_PWD"]) + + # Iterate projects + dbcon = AvalonMongoDB() + dbcon.install() + all_projects = gazu.project.all_projects() + bulk_writes = [] + for project in all_projects: + # Create project locally + # Try to find project document + project_name = project["name"] + project_code = project_name + dbcon.Session["AVALON_PROJECT"] = project_name + project_doc = dbcon.find_one({"type": "project"}) + + # Get all assets from zou + all_assets = gazu.asset.all_assets_for_project(project) + all_episodes = gazu.shot.all_episodes_for_project(project) + all_seqs = gazu.shot.all_sequences_for_project(project) + all_shots = gazu.shot.all_shots_for_project(project) + + # Create project if is not available + # - creation is required to be able set project anatomy and attributes + to_insert = [] + if not project_doc: + print(f"Creating project '{project_name}'") + project_doc = create_project(project_name, project_code, dbcon=dbcon) + + # Project data and tasks + bulk_writes.append( + UpdateOne( + {"_id": project_doc["_id"]}, + { + "$set": { + "config.tasks": { + t["name"]: {"short_name": t.get("short_name", t["name"])} + for t in gazu.task.all_task_types_for_project(project) + }, + "data": project["data"].update( + { + "code": project["code"], + "fps": project_code["fps"], + "resolutionWidth": project["resolution"].split("x")[0], + "resolutionHeight": project["resolution"].split("x")[1], + } + ), + } + }, + ) + ) + + # Query all assets of the local project + project_col = dbcon.database[project_code] + asset_doc_ids = { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + asset_doc_ids[project["id"]] = project_doc + + # Create + to_insert.extend( + [ + { + "name": item["name"], + "type": "asset", + "schema": "openpype:asset-3.0", + "data": {"zou_id": item["id"], "tasks": {}}, + } + for item in all_episodes + all_assets + all_seqs + all_shots + if item["id"] not in asset_doc_ids.keys() + ] + ) + if to_insert: + # Insert in doc + project_col.insert_many(to_insert) + + # Update existing docs + asset_doc_ids.update( + { + asset_doc["data"]["zou_id"]: asset_doc + for asset_doc in project_col.find({"type": "asset"}) + if asset_doc["data"].get("zou_id") + } + ) + + # Update + all_entities = all_assets + all_episodes + all_seqs + all_shots + bulk_writes.extend(update_op_assets(all_entities, asset_doc_ids)) + + # Delete + diff_assets = set(asset_doc_ids.keys()) - { + e["id"] for e in all_entities + [project] + } + if diff_assets: + bulk_writes.extend( + [DeleteOne(asset_doc_ids[asset_id]) for asset_id in diff_assets] + ) + + # Write into DB + if bulk_writes: + project_col.bulk_write(bulk_writes) dbcon.uninstall() diff --git a/openpype/settings/defaults/project_settings/kitsu.json b/openpype/settings/defaults/project_settings/kitsu.json index b4d2ccc611..435814a9d1 100644 --- a/openpype/settings/defaults/project_settings/kitsu.json +++ b/openpype/settings/defaults/project_settings/kitsu.json @@ -1,3 +1,7 @@ { - "number": 0 + "entities_naming_pattern": { + "episode": "E##", + "sequence": "SQ##", + "shot": "SH##" + } } \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json index 93976cc03b..a504959001 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_kitsu.json @@ -6,12 +6,26 @@ "is_file": true, "children": [ { - "type": "number", - "key": "number", - "label": "This is your lucky number:", - "minimum": 7, - "maximum": 7, - "decimals": 0 + "type": "dict", + "key": "entities_naming_pattern", + "label": "Entities naming pattern", + "children": [ + { + "type": "text", + "key": "episode", + "label": "Episode:" + }, + { + "type": "text", + "key": "sequence", + "label": "Sequence:" + }, + { + "type": "text", + "key": "shot", + "label": "Shot:" + } + ] } ] }