Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into feature/OP-3593_Move-load-functions-into-pipeline

Browse source
This commit is contained in:
Jakub Trllo 2022-07-21 11:33:16 +02:00
commit 7cbe636b7b
4 changed files with 153 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

4
openpype/modules/base.py
View file

@ -49,6 +49,7 @@ class _ModuleClass(object):
Object of this class can be stored to `sys.modules` and used for storing
dynamically imported modules.
"""
def __init__(self, name):
# Call setattr on super class
super(_ModuleClass, self).__setattr__("name", name)
@ -116,12 +117,13 @@ class _InterfacesClass(_ModuleClass):
- this is because interfaces must be available even if are missing
implementation
"""
def __getattr__(self, attr_name):
if attr_name not in self.__attributes__:
if attr_name in ("__path__", "__file__"):
return None
raise ImportError((
raise AttributeError((
"cannot import name '{}' from 'openpype_interfaces'"
).format(attr_name))

47
openpype/modules/kitsu/utils/sync_service.py
View file

@ -2,11 +2,17 @@ import os
import gazu
from openpype.client import (
get_project,
get_assets,
get_asset_by_name
)
from openpype.pipeline import AvalonMongoDB
from .credentials import validate_credentials
from .update_op_with_zou import (
create_op_asset,
set_op_project,
get_kitsu_project_name,
write_project_to_op,
update_op_assets,
)
@ -119,17 +125,16 @@ class Listener:
# Write into DB
if update_project:
self.dbcon = self.dbcon.database[project_name]
self.dbcon.Session["AVALON_PROJECT"] = project_name
self.dbcon.bulk_write([update_project])
def _delete_project(self, data):
"""Delete project."""
project_doc = self.dbcon.find_one(
{"type": "project", "data.zou_id": data["project_id"]}
)
project_name = get_kitsu_project_name(data["project_id"])
# Delete project collection
self.dbcon.database[project_doc["name"]].drop()
self.dbcon.database[project_name].drop()
# == Asset ==
@ -150,7 +155,8 @@ class Listener:
def _update_asset(self, data):
"""Update asset into OP DB."""
set_op_project(self.dbcon, data["project_id"])
project_doc = self.dbcon.find_one({"type": "project"})
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
asset = gazu.asset.get_asset(data["asset_id"])
@ -159,7 +165,7 @@ class Listener:
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in self.dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[asset["project_id"]] = project_doc
@ -199,7 +205,8 @@ class Listener:
def _update_episode(self, data):
"""Update episode into OP DB."""
set_op_project(self.dbcon, data["project_id"])
project_doc = self.dbcon.find_one({"type": "project"})
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
episode = gazu.shot.get_episode(data["episode_id"])
@ -208,7 +215,7 @@ class Listener:
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in self.dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[episode["project_id"]] = project_doc
@ -249,7 +256,8 @@ class Listener:
def _update_sequence(self, data):
"""Update sequence into OP DB."""
set_op_project(self.dbcon, data["project_id"])
project_doc = self.dbcon.find_one({"type": "project"})
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
sequence = gazu.shot.get_sequence(data["sequence_id"])
@ -258,7 +266,7 @@ class Listener:
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in self.dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[sequence["project_id"]] = project_doc
@ -299,7 +307,8 @@ class Listener:
def _update_shot(self, data):
"""Update shot into OP DB."""
set_op_project(self.dbcon, data["project_id"])
project_doc = self.dbcon.find_one({"type": "project"})
project_name = self.dbcon.active_project()
project_doc = get_project(project_name)
# Get gazu entity
shot = gazu.shot.get_shot(data["shot_id"])
@ -308,7 +317,7 @@ class Listener:
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in self.dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[shot["project_id"]] = project_doc
@ -335,14 +344,15 @@ class Listener:
"""Create new task into OP DB."""
# Get project entity
set_op_project(self.dbcon, data["project_id"])
project_name = self.dbcon.active_project()
# Get gazu entity
task = gazu.task.get_task(data["task_id"])
# Find asset doc
asset_doc = self.dbcon.find_one(
{"type": "asset", "data.zou.id": task["entity"]["id"]}
)
parent_name = task["entity"]["name"]
asset_doc = get_asset_by_name(project_name, parent_name)
# Update asset tasks with new one
asset_tasks = asset_doc["data"].get("tasks")
@ -359,10 +369,11 @@ class Listener:
def _delete_task(self, data):
"""Delete task of OP DB."""
set_op_project(self.dbcon, data["project_id"])
set_op_project(self.dbcon, data["project_id"])
project_name = self.dbcon.active_project()
# Find asset doc
asset_docs = [doc for doc in self.dbcon.find({"type": "asset"})]
asset_docs = list(get_assets(project_name))
for doc in asset_docs:
# Match task
for name, task in doc["data"]["tasks"].items():

161
openpype/modules/kitsu/utils/update_op_with_zou.py
View file

@ -10,6 +10,12 @@ from gazu.task import (
all_tasks_for_shot,
)
from openpype.client import (
get_project,
get_assets,
get_asset_by_id,
get_asset_by_name,
)
from openpype.pipeline import AvalonMongoDB
from openpype.api import get_project_settings
from openpype.lib import create_project
@ -33,6 +39,20 @@ def create_op_asset(gazu_entity: dict) -> dict:
}
def get_kitsu_project_name(project_id: str) -> str:
"""Get project name based on project id in kitsu.
Args:
project_id (str): UUID of project in Kitsu.
Returns:
str: Name of Kitsu project.
"""
project = gazu.project.get_project(project_id)
return project["name"]
def set_op_project(dbcon: AvalonMongoDB, project_id: str):
"""Set project context.
@ -40,9 +60,8 @@ def set_op_project(dbcon: AvalonMongoDB, project_id: str):
dbcon (AvalonMongoDB): Connection to DB
project_id (str): Project zou ID
"""
project = gazu.project.get_project(project_id)
project_name = project["name"]
dbcon.Session["AVALON_PROJECT"] = project_name
dbcon.Session["AVALON_PROJECT"] = get_kitsu_project_name(project_id)
def update_op_assets(
@ -72,9 +91,7 @@ def update_op_assets(
if not item_doc: # Create asset
op_asset = create_op_asset(item)
insert_result = dbcon.insert_one(op_asset)
item_doc = dbcon.find_one(
{"type": "asset", "_id": insert_result.inserted_id}
)
item_doc = get_asset_by_id(project_name, insert_result.inserted_id)
# Update asset
item_data = deepcopy(item_doc["data"])
@ -137,17 +154,23 @@ def update_op_assets(
parent_zou_id = substitute_parent_item["parent_id"]
else:
parent_zou_id = (
item.get("parent_id")
# For Asset, put under asset type directory
item.get("entity_type_id")
if item_type == "Asset"
else None
# Else, fallback on usual hierarchy
or item.get("parent_id")
or item.get("episode_id")
or item.get("source_id")
) # TODO check consistency
)
# Substitute Episode and Sequence by Shot
substitute_item_type = (
"shots"
if item_type in ["Episode", "Sequence"]
else f"{item_type.lower()}s"
)
# Substitute item type for general classification (assets or shots)
if item_type in ["Asset", "AssetType"]:
substitute_item_type = "assets"
elif item_type in ["Episode", "Sequence"]:
substitute_item_type = "shots"
else:
substitute_item_type = f"{item_type.lower()}s"
entity_parent_folders = [
f
for f in project_module_settings["entities_root"]
@ -161,15 +184,33 @@ def update_op_assets(
asset_doc_ids[parent_zou_id]["_id"] if parent_zou_id else None
)
if visual_parent_doc_id is None:
# Find root folder doc
root_folder_doc = dbcon.find_one(
{
"type": "asset",
"name": entity_parent_folders[-1],
"data.root_of": substitute_item_type,
},
["_id"],
# Find root folder docs
root_folder_docs = get_assets(
project_name,
asset_names=[entity_parent_folders[-1]],
fields=["_id", "data.root_of"],
)
# NOTE: Not sure why it's checking for entity type?
# OP3 does not support multiple assets with same names so type
# filtering is irelevant.
# This way mimics previous implementation:
# ```
# root_folder_doc = dbcon.find_one(
# {
# "type": "asset",
# "name": entity_parent_folders[-1],
# "data.root_of": substitute_item_type,
# },
# ["_id"],
# )
# ```
root_folder_doc = None
for folder_doc in root_folder_docs:
root_of = folder_doc.get("data", {}).get("root_of")
if root_of == substitute_item_type:
root_folder_doc = folder_doc
break
if root_folder_doc:
visual_parent_doc_id = root_folder_doc["_id"]
@ -184,7 +225,14 @@ def update_op_assets(
# Get parent entity
parent_entity = parent_doc["data"]["zou"]
parent_zou_id = parent_entity["parent_id"]
parent_zou_id = parent_entity.get("parent_id")
if item_type in ["Shot", "Sequence"]:
# Name with parents hierarchy "({episode}_){sequence}_{shot}"
# to avoid duplicate name issue
item_name = "_".join(item_data["parents"] + [item_doc["name"]])
else:
item_name = item_doc["name"]
# Set root folders parents
item_data["parents"] = entity_parent_folders + item_data["parents"]
@ -199,9 +247,9 @@ def update_op_assets(
item_doc["_id"],
{
"$set": {
"name": item["name"],
"name": item_name,
"data": item_data,
"parent": asset_doc_ids[item["project_id"]]["_id"],
"parent": project_doc["_id"],
}
},
)
@ -222,7 +270,7 @@ def write_project_to_op(project: dict, dbcon: AvalonMongoDB) -> UpdateOne:
UpdateOne: Update instance for the project
"""
project_name = project["name"]
project_doc = dbcon.database[project_name].find_one({"type": "project"})
project_doc = get_project(project_name)
if not project_doc:
print(f"Creating project '{project_name}'")
project_doc = create_project(project_name, project_name, dbcon=dbcon)
@ -292,6 +340,11 @@ def sync_all_projects(login: str, password: str):
def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict):
"""Update OP project in DB with Zou data.
`root_of` is meant to sort entities by type for a better readability in
the data tree. It puts all shot like (Shot and Episode and Sequence) and
asset entities under two different root folders or hierarchy, defined in
settings.
Args:
dbcon (AvalonMongoDB): MongoDB connection
project (dict): Project dict got using gazu.
@ -306,12 +359,17 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict):
# Get all assets from zou
all_assets = gazu.asset.all_assets_for_project(project)
all_asset_types = gazu.asset.all_asset_types_for_project(project)
all_episodes = gazu.shot.all_episodes_for_project(project)
all_seqs = gazu.shot.all_sequences_for_project(project)
all_shots = gazu.shot.all_shots_for_project(project)
all_entities = [
item
for item in all_assets + all_episodes + all_seqs + all_shots
for item in all_assets
+ all_asset_types
+ all_episodes
+ all_seqs
+ all_shots
if naming_pattern.match(item["name"])
]
@ -319,26 +377,44 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict):
bulk_writes.append(write_project_to_op(project, dbcon))
# Try to find project document
dbcon.Session["AVALON_PROJECT"] = project["name"]
project_doc = dbcon.find_one({"type": "project"})
project_name = project["name"]
dbcon.Session["AVALON_PROJECT"] = project_name
project_doc = get_project(project_name)
# Query all assets of the local project
zou_ids_and_asset_docs = {
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou", {}).get("id")
}
zou_ids_and_asset_docs[project["id"]] = project_doc
# Create entities root folders
project_module_settings = get_project_settings(project["name"])["kitsu"]
project_module_settings = get_project_settings(project_name)["kitsu"]
for entity_type, root in project_module_settings["entities_root"].items():
parent_folders = root.split("/")
direct_parent_doc = None
for i, folder in enumerate(parent_folders, 1):
parent_doc = dbcon.find_one(
{"type": "asset", "name": folder, "data.root_of": entity_type}
parent_doc = get_asset_by_name(
project_name, folder, fields=["_id", "data.root_of"]
)
# NOTE: Not sure why it's checking for entity type?
# OP3 does not support multiple assets with same names so type
# filtering is irelevant.
# Also all of the entities could find be queried at once using
# 'get_assets'.
# This way mimics previous implementation:
# ```
# parent_doc = dbcon.find_one(
# {"type": "asset", "name": folder, "data.root_of": entity_type}
# )
# ```
if (
parent_doc
and parent_doc.get("data", {}).get("root_of") != entity_type
):
parent_doc = None
if not parent_doc:
direct_parent_doc = dbcon.insert_one(
{
@ -348,21 +424,20 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict):
"data": {
"root_of": entity_type,
"parents": parent_folders[:i],
"visualParent": direct_parent_doc,
"visualParent": direct_parent_doc.inserted_id
if direct_parent_doc
else None,
"tasks": {},
},
}
)
# Create
to_insert = []
to_insert.extend(
[
create_op_asset(item)
for item in all_entities
if item["id"] not in zou_ids_and_asset_docs.keys()
]
)
to_insert = [
create_op_asset(item)
for item in all_entities
if item["id"] not in zou_ids_and_asset_docs.keys()
]
if to_insert:
# Insert doc in DB
dbcon.insert_many(to_insert)
@ -371,7 +446,7 @@ def sync_project_from_kitsu(dbcon: AvalonMongoDB, project: dict):
zou_ids_and_asset_docs.update(
{
asset_doc["data"]["zou"]["id"]: asset_doc
for asset_doc in dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
if asset_doc["data"].get("zou")
}
)

7
openpype/modules/kitsu/utils/update_zou_with_op.py
View file

@ -6,6 +6,7 @@ from typing import List
import gazu
from pymongo import UpdateOne
from openpype.client import get_project, get_assets
from openpype.pipeline import AvalonMongoDB
from openpype.api import get_project_settings
from openpype.modules.kitsu.utils.credentials import validate_credentials
@ -53,9 +54,7 @@ def sync_zou_from_op_project(
"""
# Get project doc if not provided
if not project_doc:
project_doc = dbcon.database[project_name].find_one(
{"type": "project"}
)
project_doc = get_project(project_name)
# Get all entities from zou
print(f"Synchronizing {project_name}...")
@ -96,7 +95,7 @@ def sync_zou_from_op_project(
dbcon.Session["AVALON_PROJECT"] = project_name
asset_docs = {
asset_doc["_id"]: asset_doc
for asset_doc in dbcon.find({"type": "asset"})
for asset_doc in get_assets(project_name)
}
# Create new assets