mirror of
https://github.com/ynput/ayon-core.git
synced 2026-01-01 16:34:53 +01:00
Merge pull request #3838 from pypeclub/feature/OP-3828_Workfile-template-build-enhancements
General: Workfile template build enhancements
This commit is contained in:
Jakub Trllo
GitHub
commit
a7d82ecbc0
20 changed files with 2482 additions and 1793 deletions
|
|
@ -1,528 +0,0 @@
|
|||
import os
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
||||
import six
|
||||
import logging
|
||||
from functools import reduce
|
||||
|
||||
from openpype.client import (
|
||||
get_asset_by_name,
|
||||
get_linked_assets,
|
||||
)
|
||||
from openpype.settings import get_project_settings
|
||||
from openpype.lib import (
|
||||
StringTemplate,
|
||||
Logger,
|
||||
filter_profiles,
|
||||
)
|
||||
from openpype.pipeline import legacy_io, Anatomy
|
||||
from openpype.pipeline.load import (
|
||||
get_loaders_by_name,
|
||||
get_representation_context,
|
||||
load_with_repre_context,
|
||||
)
|
||||
|
||||
from .build_template_exceptions import (
|
||||
TemplateAlreadyImported,
|
||||
TemplateLoadingFailed,
|
||||
TemplateProfileNotFound,
|
||||
TemplateNotFound
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def update_representations(entities, entity):
|
||||
if entity['context']['subset'] not in entities:
|
||||
entities[entity['context']['subset']] = entity
|
||||
else:
|
||||
current = entities[entity['context']['subset']]
|
||||
incomming = entity
|
||||
entities[entity['context']['subset']] = max(
|
||||
current, incomming,
|
||||
key=lambda entity: entity["context"].get("version", -1))
|
||||
|
||||
return entities
|
||||
|
||||
|
||||
def parse_loader_args(loader_args):
|
||||
if not loader_args:
|
||||
return dict()
|
||||
try:
|
||||
parsed_args = eval(loader_args)
|
||||
if not isinstance(parsed_args, dict):
|
||||
return dict()
|
||||
else:
|
||||
return parsed_args
|
||||
except Exception as err:
|
||||
print(
|
||||
"Error while parsing loader arguments '{}'.\n{}: {}\n\n"
|
||||
"Continuing with default arguments. . .".format(
|
||||
loader_args,
|
||||
err.__class__.__name__,
|
||||
err))
|
||||
return dict()
|
||||
|
||||
|
||||
@six.add_metaclass(ABCMeta)
|
||||
class AbstractTemplateLoader:
|
||||
"""
|
||||
Abstraction of Template Loader.
|
||||
Properties:
|
||||
template_path : property to get current template path
|
||||
Methods:
|
||||
import_template : Abstract Method. Used to load template,
|
||||
depending on current host
|
||||
get_template_nodes : Abstract Method. Used to query nodes acting
|
||||
as placeholders. Depending on current host
|
||||
"""
|
||||
|
||||
_log = None
|
||||
|
||||
def __init__(self, placeholder_class):
|
||||
# TODO template loader should expect host as and argument
|
||||
# - host have all responsibility for most of code (also provide
|
||||
# placeholder class)
|
||||
# - also have responsibility for current context
|
||||
# - this won't work in DCCs where multiple workfiles with
|
||||
# different contexts can be opened at single time
|
||||
# - template loader should have ability to change context
|
||||
project_name = legacy_io.active_project()
|
||||
asset_name = legacy_io.Session["AVALON_ASSET"]
|
||||
|
||||
self.loaders_by_name = get_loaders_by_name()
|
||||
self.current_asset = asset_name
|
||||
self.project_name = project_name
|
||||
self.host_name = legacy_io.Session["AVALON_APP"]
|
||||
self.task_name = legacy_io.Session["AVALON_TASK"]
|
||||
self.placeholder_class = placeholder_class
|
||||
self.current_asset_doc = get_asset_by_name(project_name, asset_name)
|
||||
self.task_type = (
|
||||
self.current_asset_doc
|
||||
.get("data", {})
|
||||
.get("tasks", {})
|
||||
.get(self.task_name, {})
|
||||
.get("type")
|
||||
)
|
||||
|
||||
self.log.info(
|
||||
"BUILDING ASSET FROM TEMPLATE :\n"
|
||||
"Starting templated build for {asset} in {project}\n\n"
|
||||
"Asset : {asset}\n"
|
||||
"Task : {task_name} ({task_type})\n"
|
||||
"Host : {host}\n"
|
||||
"Project : {project}\n".format(
|
||||
asset=self.current_asset,
|
||||
host=self.host_name,
|
||||
project=self.project_name,
|
||||
task_name=self.task_name,
|
||||
task_type=self.task_type
|
||||
))
|
||||
# Skip if there is no loader
|
||||
if not self.loaders_by_name:
|
||||
self.log.warning(
|
||||
"There is no registered loaders. No assets will be loaded")
|
||||
return
|
||||
|
||||
@property
|
||||
def log(self):
|
||||
if self._log is None:
|
||||
self._log = Logger.get_logger(self.__class__.__name__)
|
||||
return self._log
|
||||
|
||||
def template_already_imported(self, err_msg):
|
||||
"""In case template was already loaded.
|
||||
Raise the error as a default action.
|
||||
Override this method in your template loader implementation
|
||||
to manage this case."""
|
||||
self.log.error("{}: {}".format(
|
||||
err_msg.__class__.__name__,
|
||||
err_msg))
|
||||
raise TemplateAlreadyImported(err_msg)
|
||||
|
||||
def template_loading_failed(self, err_msg):
|
||||
"""In case template loading failed
|
||||
Raise the error as a default action.
|
||||
Override this method in your template loader implementation
|
||||
to manage this case.
|
||||
"""
|
||||
self.log.error("{}: {}".format(
|
||||
err_msg.__class__.__name__,
|
||||
err_msg))
|
||||
raise TemplateLoadingFailed(err_msg)
|
||||
|
||||
@property
|
||||
def template_path(self):
|
||||
"""
|
||||
Property returning template path. Avoiding setter.
|
||||
Getting template path from open pype settings based on current avalon
|
||||
session and solving the path variables if needed.
|
||||
Returns:
|
||||
str: Solved template path
|
||||
Raises:
|
||||
TemplateProfileNotFound: No profile found from settings for
|
||||
current avalon session
|
||||
KeyError: Could not solve path because a key does not exists
|
||||
in avalon context
|
||||
TemplateNotFound: Solved path does not exists on current filesystem
|
||||
"""
|
||||
project_name = self.project_name
|
||||
host_name = self.host_name
|
||||
task_name = self.task_name
|
||||
task_type = self.task_type
|
||||
|
||||
anatomy = Anatomy(project_name)
|
||||
project_settings = get_project_settings(project_name)
|
||||
|
||||
build_info = project_settings[host_name]["templated_workfile_build"]
|
||||
profile = filter_profiles(
|
||||
build_info["profiles"],
|
||||
{
|
||||
"task_types": task_type,
|
||||
"task_names": task_name
|
||||
}
|
||||
)
|
||||
|
||||
if not profile:
|
||||
raise TemplateProfileNotFound(
|
||||
"No matching profile found for task '{}' of type '{}' "
|
||||
"with host '{}'".format(task_name, task_type, host_name)
|
||||
)
|
||||
|
||||
path = profile["path"]
|
||||
if not path:
|
||||
raise TemplateLoadingFailed(
|
||||
"Template path is not set.\n"
|
||||
"Path need to be set in {}\\Template Workfile Build "
|
||||
"Settings\\Profiles".format(host_name.title()))
|
||||
|
||||
# Try fill path with environments and anatomy roots
|
||||
fill_data = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
}
|
||||
fill_data["root"] = anatomy.roots
|
||||
result = StringTemplate.format_template(path, fill_data)
|
||||
if result.solved:
|
||||
path = result.normalized()
|
||||
|
||||
if path and os.path.exists(path):
|
||||
self.log.info("Found template at: '{}'".format(path))
|
||||
return path
|
||||
|
||||
solved_path = None
|
||||
while True:
|
||||
try:
|
||||
solved_path = anatomy.path_remapper(path)
|
||||
except KeyError as missing_key:
|
||||
raise KeyError(
|
||||
"Could not solve key '{}' in template path '{}'".format(
|
||||
missing_key, path))
|
||||
|
||||
if solved_path is None:
|
||||
solved_path = path
|
||||
if solved_path == path:
|
||||
break
|
||||
path = solved_path
|
||||
|
||||
solved_path = os.path.normpath(solved_path)
|
||||
if not os.path.exists(solved_path):
|
||||
raise TemplateNotFound(
|
||||
"Template found in openPype settings for task '{}' with host "
|
||||
"'{}' does not exists. (Not found : {})".format(
|
||||
task_name, host_name, solved_path))
|
||||
|
||||
self.log.info("Found template at: '{}'".format(solved_path))
|
||||
|
||||
return solved_path
|
||||
|
||||
def populate_template(self, ignored_ids=None):
|
||||
"""
|
||||
Use template placeholders to load assets and parent them in hierarchy
|
||||
Arguments :
|
||||
ignored_ids :
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
|
||||
loaders_by_name = self.loaders_by_name
|
||||
current_asset_doc = self.current_asset_doc
|
||||
linked_assets = get_linked_assets(current_asset_doc)
|
||||
|
||||
ignored_ids = ignored_ids or []
|
||||
placeholders = self.get_placeholders()
|
||||
self.log.debug("Placeholders found in template: {}".format(
|
||||
[placeholder.name for placeholder in placeholders]
|
||||
))
|
||||
for placeholder in placeholders:
|
||||
self.log.debug("Start to processing placeholder {}".format(
|
||||
placeholder.name
|
||||
))
|
||||
placeholder_representations = self.get_placeholder_representations(
|
||||
placeholder,
|
||||
current_asset_doc,
|
||||
linked_assets
|
||||
)
|
||||
|
||||
if not placeholder_representations:
|
||||
self.log.info(
|
||||
"There's no representation for this placeholder: "
|
||||
"{}".format(placeholder.name)
|
||||
)
|
||||
continue
|
||||
|
||||
for representation in placeholder_representations:
|
||||
self.preload(placeholder, loaders_by_name, representation)
|
||||
|
||||
if self.load_data_is_incorrect(
|
||||
placeholder,
|
||||
representation,
|
||||
ignored_ids):
|
||||
continue
|
||||
|
||||
self.log.info(
|
||||
"Loading {}_{} with loader {}\n"
|
||||
"Loader arguments used : {}".format(
|
||||
representation['context']['asset'],
|
||||
representation['context']['subset'],
|
||||
placeholder.loader_name,
|
||||
placeholder.loader_args))
|
||||
|
||||
try:
|
||||
container = self.load(
|
||||
placeholder, loaders_by_name, representation)
|
||||
except Exception:
|
||||
self.load_failed(placeholder, representation)
|
||||
else:
|
||||
self.load_succeed(placeholder, container)
|
||||
finally:
|
||||
self.postload(placeholder)
|
||||
|
||||
def get_placeholder_representations(
|
||||
self, placeholder, current_asset_doc, linked_asset_docs
|
||||
):
|
||||
placeholder_representations = placeholder.get_representations(
|
||||
current_asset_doc,
|
||||
linked_asset_docs
|
||||
)
|
||||
for repre_doc in reduce(
|
||||
update_representations,
|
||||
placeholder_representations,
|
||||
dict()
|
||||
).values():
|
||||
yield repre_doc
|
||||
|
||||
def load_data_is_incorrect(
|
||||
self, placeholder, last_representation, ignored_ids):
|
||||
if not last_representation:
|
||||
self.log.warning(placeholder.err_message())
|
||||
return True
|
||||
if (str(last_representation['_id']) in ignored_ids):
|
||||
print("Ignoring : ", last_representation['_id'])
|
||||
return True
|
||||
return False
|
||||
|
||||
def preload(self, placeholder, loaders_by_name, last_representation):
|
||||
pass
|
||||
|
||||
def load(self, placeholder, loaders_by_name, last_representation):
|
||||
repre = get_representation_context(last_representation)
|
||||
return load_with_repre_context(
|
||||
loaders_by_name[placeholder.loader_name],
|
||||
repre,
|
||||
options=parse_loader_args(placeholder.loader_args))
|
||||
|
||||
def load_succeed(self, placeholder, container):
|
||||
placeholder.parent_in_hierarchy(container)
|
||||
|
||||
def load_failed(self, placeholder, last_representation):
|
||||
self.log.warning(
|
||||
"Got error trying to load {}:{} with {}".format(
|
||||
last_representation['context']['asset'],
|
||||
last_representation['context']['subset'],
|
||||
placeholder.loader_name
|
||||
),
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
def postload(self, placeholder):
|
||||
placeholder.clean()
|
||||
|
||||
def update_missing_containers(self):
|
||||
loaded_containers_ids = self.get_loaded_containers_by_id()
|
||||
self.populate_template(ignored_ids=loaded_containers_ids)
|
||||
|
||||
def get_placeholders(self):
|
||||
placeholders = map(self.placeholder_class, self.get_template_nodes())
|
||||
valid_placeholders = filter(
|
||||
lambda i: i.is_valid,
|
||||
placeholders
|
||||
)
|
||||
sorted_placeholders = list(sorted(
|
||||
valid_placeholders,
|
||||
key=lambda i: i.order
|
||||
))
|
||||
return sorted_placeholders
|
||||
|
||||
@abstractmethod
|
||||
def get_loaded_containers_by_id(self):
|
||||
"""
|
||||
Collect already loaded containers for updating scene
|
||||
Return:
|
||||
dict (string, node): A dictionnary id as key
|
||||
and containers as value
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def import_template(self, template_path):
|
||||
"""
|
||||
Import template in current host
|
||||
Args:
|
||||
template_path (str): fullpath to current task and
|
||||
host's template file
|
||||
Return:
|
||||
None
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_template_nodes(self):
|
||||
"""
|
||||
Returning a list of nodes acting as host placeholders for
|
||||
templating. The data representation is by user.
|
||||
AbstractLoadTemplate (and LoadTemplate) won't directly manipulate nodes
|
||||
Args :
|
||||
None
|
||||
Returns:
|
||||
list(AnyNode): Solved template path
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@six.add_metaclass(ABCMeta)
|
||||
class AbstractPlaceholder:
|
||||
"""Abstraction of placeholders logic.
|
||||
|
||||
Properties:
|
||||
required_keys: A list of mandatory keys to decribe placeholder
|
||||
and assets to load.
|
||||
optional_keys: A list of optional keys to decribe
|
||||
placeholder and assets to load
|
||||
loader_name: Name of linked loader to use while loading assets
|
||||
|
||||
Args:
|
||||
identifier (str): Placeholder identifier. Should be possible to be
|
||||
used as identifier in "a scene" (e.g. unique node name).
|
||||
"""
|
||||
|
||||
required_keys = {
|
||||
"builder_type",
|
||||
"family",
|
||||
"representation",
|
||||
"order",
|
||||
"loader",
|
||||
"loader_args"
|
||||
}
|
||||
optional_keys = {}
|
||||
|
||||
def __init__(self, identifier):
|
||||
self._log = None
|
||||
self._name = identifier
|
||||
self.get_data(identifier)
|
||||
|
||||
@property
|
||||
def log(self):
|
||||
if self._log is None:
|
||||
self._log = Logger.get_logger(repr(self))
|
||||
return self._log
|
||||
|
||||
def __repr__(self):
|
||||
return "< {} {} >".format(self.__class__.__name__, self.name)
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def loader_args(self):
|
||||
return self.data["loader_args"]
|
||||
|
||||
@property
|
||||
def builder_type(self):
|
||||
return self.data["builder_type"]
|
||||
|
||||
@property
|
||||
def order(self):
|
||||
return self.data["order"]
|
||||
|
||||
@property
|
||||
def loader_name(self):
|
||||
"""Return placeholder loader name.
|
||||
|
||||
Returns:
|
||||
str: Loader name that will be used to load placeholder
|
||||
representations.
|
||||
"""
|
||||
|
||||
return self.data["loader"]
|
||||
|
||||
@property
|
||||
def is_valid(self):
|
||||
"""Test validity of placeholder.
|
||||
|
||||
i.e.: every required key exists in placeholder data
|
||||
|
||||
Returns:
|
||||
bool: True if every key is in data
|
||||
"""
|
||||
|
||||
if set(self.required_keys).issubset(self.data.keys()):
|
||||
self.log.debug("Valid placeholder : {}".format(self.name))
|
||||
return True
|
||||
self.log.info("Placeholder is not valid : {}".format(self.name))
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def parent_in_hierarchy(self, container):
|
||||
"""Place loaded container in correct hierarchy given by placeholder
|
||||
|
||||
Args:
|
||||
container (Dict[str, Any]): Loaded container created by loader.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def clean(self):
|
||||
"""Clean placeholder from hierarchy after loading assets."""
|
||||
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_representations(self, current_asset_doc, linked_asset_docs):
|
||||
"""Query representations based on placeholder data.
|
||||
|
||||
Args:
|
||||
current_asset_doc (Dict[str, Any]): Document of current
|
||||
context asset.
|
||||
linked_asset_docs (List[Dict[str, Any]]): Documents of assets
|
||||
linked to current context asset.
|
||||
|
||||
Returns:
|
||||
Iterable[Dict[str, Any]]: Representations that are matching
|
||||
placeholder filters.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_data(self, identifier):
|
||||
"""Collect information about placeholder by identifier.
|
||||
|
||||
Args:
|
||||
identifier (str): A unique placeholder identifier defined by
|
||||
implementation.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
import os
|
||||
from importlib import import_module
|
||||
from openpype.lib import classes_from_module
|
||||
from openpype.host import HostBase
|
||||
from openpype.pipeline import registered_host
|
||||
|
||||
from .abstract_template_loader import (
|
||||
AbstractPlaceholder,
|
||||
AbstractTemplateLoader)
|
||||
|
||||
from .build_template_exceptions import (
|
||||
TemplateLoadingFailed,
|
||||
TemplateAlreadyImported,
|
||||
MissingHostTemplateModule,
|
||||
MissingTemplatePlaceholderClass,
|
||||
MissingTemplateLoaderClass
|
||||
)
|
||||
|
||||
_module_path_format = 'openpype.hosts.{host}.api.template_loader'
|
||||
|
||||
|
||||
def build_workfile_template(*args):
|
||||
template_loader = build_template_loader()
|
||||
try:
|
||||
template_loader.import_template(template_loader.template_path)
|
||||
except TemplateAlreadyImported as err:
|
||||
template_loader.template_already_imported(err)
|
||||
except TemplateLoadingFailed as err:
|
||||
template_loader.template_loading_failed(err)
|
||||
else:
|
||||
template_loader.populate_template()
|
||||
|
||||
|
||||
def update_workfile_template(*args):
|
||||
template_loader = build_template_loader()
|
||||
template_loader.update_missing_containers()
|
||||
|
||||
|
||||
def build_template_loader():
|
||||
# TODO refactor to use advantage of 'HostBase' and don't import dynamically
|
||||
# - hosts should have methods that gives option to return builders
|
||||
host = registered_host()
|
||||
if isinstance(host, HostBase):
|
||||
host_name = host.name
|
||||
else:
|
||||
host_name = os.environ.get("AVALON_APP")
|
||||
if not host_name:
|
||||
host_name = host.__name__.split(".")[-2]
|
||||
|
||||
module_path = _module_path_format.format(host=host_name)
|
||||
module = import_module(module_path)
|
||||
if not module:
|
||||
raise MissingHostTemplateModule(
|
||||
"No template loader found for host {}".format(host_name))
|
||||
|
||||
template_loader_class = classes_from_module(
|
||||
AbstractTemplateLoader,
|
||||
module
|
||||
)
|
||||
template_placeholder_class = classes_from_module(
|
||||
AbstractPlaceholder,
|
||||
module
|
||||
)
|
||||
|
||||
if not template_loader_class:
|
||||
raise MissingTemplateLoaderClass()
|
||||
template_loader_class = template_loader_class[0]
|
||||
|
||||
if not template_placeholder_class:
|
||||
raise MissingTemplatePlaceholderClass()
|
||||
template_placeholder_class = template_placeholder_class[0]
|
||||
return template_loader_class(template_placeholder_class)
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
class MissingHostTemplateModule(Exception):
|
||||
"""Error raised when expected module does not exists"""
|
||||
pass
|
||||
|
||||
|
||||
class MissingTemplatePlaceholderClass(Exception):
|
||||
"""Error raised when module doesn't implement a placeholder class"""
|
||||
pass
|
||||
|
||||
|
||||
class MissingTemplateLoaderClass(Exception):
|
||||
"""Error raised when module doesn't implement a template loader class"""
|
||||
pass
|
||||
|
||||
|
||||
class TemplateNotFound(Exception):
|
||||
"""Exception raised when template does not exist."""
|
||||
pass
|
||||
|
||||
|
||||
class TemplateProfileNotFound(Exception):
|
||||
"""Exception raised when current profile
|
||||
doesn't match any template profile"""
|
||||
pass
|
||||
|
||||
|
||||
class TemplateAlreadyImported(Exception):
|
||||
"""Error raised when Template was already imported by host for
|
||||
this session"""
|
||||
pass
|
||||
|
||||
|
||||
class TemplateLoadingFailed(Exception):
|
||||
"""Error raised whend Template loader was unable to load the template"""
|
||||
pass
|
||||
|
|
@ -1,3 +1,14 @@
|
|||
"""Workfile build based on settings.
|
||||
|
||||
Workfile builder will do stuff based on project settings. Advantage is that
|
||||
it need only access to settings. Disadvantage is that it is hard to focus
|
||||
build per context and being explicit about loaded content.
|
||||
|
||||
For more explicit workfile build is recommended 'AbstractTemplateBuilder'
|
||||
from '~/openpype/pipeline/workfile/workfile_template_builder'. Which gives
|
||||
more abilities to define how build happens but require more code to achive it.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import collections
|
||||
|
|
|
|||
1451
openpype/pipeline/workfile/workfile_template_builder.py
Normal file
1451
openpype/pipeline/workfile/workfile_template_builder.py
Normal file
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue