Merge branch 'develop' into feature/hinted-line-edit

This commit is contained in:
Jakub Trllo 2024-08-21 16:30:23 +02:00 committed by GitHub
commit 2e24a6fcff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 329 additions and 138 deletions

View file

@ -20,6 +20,7 @@ from ayon_core.lib import (
Logger,
is_dev_mode_enabled,
get_launcher_storage_dir,
is_headless_mode_enabled,
)
from ayon_core.settings import get_studio_settings
@ -80,36 +81,41 @@ class ProcessPreparationError(Exception):
class ProcessContext:
"""Context of child process.
"""Hold context of process that is going to be started.
Notes:
This class is used to pass context to child process. It can be used
to use different behavior of addon based on information in
the context.
The context can be enhanced in future versions.
Right now the context is simple, having information about addon that wants
to trigger preparation and possibly project name for which it should
happen.
Preparation for process can be required for ayon-core or any other addon.
It can be, change of environment variables, or request login to
a project management.
At the moment of creation is 'ProcessContext' only data holder, but that
might change in future if there will be need.
Args:
addon_name (Optional[str]): Addon name which triggered process.
addon_version (Optional[str]): Addon version which triggered process.
addon_name (str): Addon name which triggered process.
addon_version (str): Addon version which triggered process.
project_name (Optional[str]): Project name. Can be filled in case
process is triggered for specific project. Some addons can have
different behavior based on project.
headless (Optional[bool]): Is process running in headless mode.
different behavior based on project. Value is NOT autofilled.
headless (Optional[bool]): Is process running in headless mode. Value
is filled with value based on state set in AYON launcher.
"""
def __init__(
self,
addon_name: Optional[str] = None,
addon_version: Optional[str] = None,
addon_name: str,
addon_version: str,
project_name: Optional[str] = None,
headless: Optional[bool] = None,
**kwargs,
):
if headless is None:
# TODO use lib function to get headless mode
headless = os.getenv("AYON_HEADLESS_MODE") == "1"
self.addon_name: Optional[str] = addon_name
self.addon_version: Optional[str] = addon_version
headless = is_headless_mode_enabled()
self.addon_name: str = addon_name
self.addon_version: str = addon_version
self.project_name: Optional[str] = project_name
self.headless: bool = headless

View file

@ -72,13 +72,17 @@ def ensure_addons_are_process_context_ready(
process_context: ProcessContext,
addons_manager: Optional[AddonsManager] = None,
exit_on_failure: bool = True,
) -> Optional[Exception]:
) -> bool:
"""Ensure all enabled addons are ready to be used in the given context.
Call this method only in AYON launcher process and as first thing
to avoid possible clashes with preparation. For example 'QApplication'
should not be created.
Todos:
Run all preparations and allow to "ignore" failed preparations.
Right now single addon can block using certain actions.
Args:
process_context (ProcessContext): The context in which the
addons should be prepared.
@ -88,14 +92,12 @@ def ensure_addons_are_process_context_ready(
if an error occurs. Defaults to True.
Returns:
Optional[Exception]: The exception that occurred during the
preparation, if any.
bool: True if all addons are ready, False otherwise.
"""
if addons_manager is None:
addons_manager = AddonsManager()
exception = None
message = None
failed = False
use_detail = False
@ -112,13 +114,11 @@ def ensure_addons_are_process_context_ready(
addon.ensure_is_process_ready(process_context)
addon_failed = False
except ProcessPreparationError as exc:
exception = exc
message = str(exc)
print(f"Addon preparation failed: '{addon.name}'")
print(message)
except BaseException as exc:
exception = exc
except BaseException:
use_detail = True
message = "An unexpected error occurred."
formatted_traceback = "".join(traceback.format_exception(
@ -140,7 +140,7 @@ def ensure_addons_are_process_context_ready(
if not failed:
if not process_context.headless:
_start_tray()
return None
return True
detail = None
if use_detail:
@ -150,16 +150,21 @@ def ensure_addons_are_process_context_ready(
detail = output_str
_handle_error(process_context, message, detail)
if not exit_on_failure:
return exception
sys.exit(1)
if exit_on_failure:
sys.exit(1)
return False
def ensure_addons_are_process_ready(
addon_name: str,
addon_version: str,
project_name: Optional[str] = None,
headless: Optional[bool] = None,
*,
addons_manager: Optional[AddonsManager] = None,
exit_on_failure: bool = True,
**kwargs,
) -> Optional[Exception]:
) -> bool:
"""Ensure all enabled addons are ready to be used in the given context.
Call this method only in AYON launcher process and as first thing
@ -167,6 +172,13 @@ def ensure_addons_are_process_ready(
should not be created.
Args:
addon_name (str): Addon name which triggered process.
addon_version (str): Addon version which triggered process.
project_name (Optional[str]): Project name. Can be filled in case
process is triggered for specific project. Some addons can have
different behavior based on project. Value is NOT autofilled.
headless (Optional[bool]): Is process running in headless mode. Value
is filled with value based on state set in AYON launcher.
addons_manager (Optional[AddonsManager]): The addons
manager to use. If not provided, a new one will be created.
exit_on_failure (bool, optional): If True, the process will exit
@ -174,11 +186,16 @@ def ensure_addons_are_process_ready(
kwargs: The keyword arguments to pass to the ProcessContext.
Returns:
Optional[Exception]: The exception that occurred during the
preparation, if any.
bool: True if all addons are ready, False otherwise.
"""
context: ProcessContext = ProcessContext(**kwargs)
context: ProcessContext = ProcessContext(
addon_name,
addon_version,
project_name,
headless,
**kwargs
)
return ensure_addons_are_process_context_ready(
context, addons_manager, exit_on_failure
)

View file

@ -132,6 +132,7 @@ from .ayon_info import (
is_in_ayon_launcher_process,
is_running_from_build,
is_using_ayon_console,
is_headless_mode_enabled,
is_staging_enabled,
is_dev_mode_enabled,
is_in_tests,
@ -245,6 +246,7 @@ __all__ = [
"is_in_ayon_launcher_process",
"is_running_from_build",
"is_using_ayon_console",
"is_headless_mode_enabled",
"is_staging_enabled",
"is_dev_mode_enabled",
"is_in_tests",

View file

@ -78,6 +78,10 @@ def is_using_ayon_console():
return "ayon_console" in executable_filename
def is_headless_mode_enabled():
return os.getenv("AYON_HEADLESS_MODE") == "1"
def is_staging_enabled():
return os.getenv("AYON_USE_STAGING") == "1"

View file

@ -1,7 +1,6 @@
import ayon_api
from ayon_core.lib import StringTemplate, filter_profiles, prepare_template_data
from ayon_core.settings import get_project_settings
from ayon_core.lib import filter_profiles, prepare_template_data
from .constants import DEFAULT_PRODUCT_TEMPLATE
@ -183,7 +182,10 @@ def get_product_name(
fill_pairs[key] = value
try:
return template.format(**prepare_template_data(fill_pairs))
return StringTemplate.format_strict_template(
template=template,
data=prepare_template_data(fill_pairs)
)
except KeyError as exp:
raise TemplateFillError(
"Value for {} key is missing in template '{}'."

View file

@ -1,5 +1,5 @@
import os
import copy
import os
import re
import warnings
from copy import deepcopy
@ -7,14 +7,11 @@ from copy import deepcopy
import attr
import ayon_api
import clique
from ayon_core.pipeline import (
get_current_project_name,
get_representation_path,
)
from ayon_core.lib import Logger
from ayon_core.pipeline.publish import KnownPublishError
from ayon_core.pipeline import get_current_project_name, get_representation_path
from ayon_core.pipeline.create import get_product_name
from ayon_core.pipeline.farm.patterning import match_aov_pattern
from ayon_core.pipeline.publish import KnownPublishError
@attr.s
@ -250,6 +247,9 @@ def create_skeleton_instance(
"colorspace": data.get("colorspace")
}
if data.get("renderlayer"):
instance_skeleton_data["renderlayer"] = data["renderlayer"]
# skip locking version if we are creating v01
instance_version = data.get("version") # take this if exists
if instance_version != 1:
@ -464,7 +464,9 @@ def create_instances_for_aov(instance, skeleton, aov_filter,
Args:
instance (pyblish.api.Instance): Original instance.
skeleton (dict): Skeleton instance data.
aov_filter (dict): AOV filter.
skip_integration_repre_list (list): skip
do_not_add_review (bool): Explicitly disable reviews
Returns:
list of pyblish.api.Instance: Instances created from
@ -515,6 +517,131 @@ def create_instances_for_aov(instance, skeleton, aov_filter,
)
def _get_legacy_product_name_and_group(
product_type,
source_product_name,
task_name,
dynamic_data):
"""Get product name with legacy logic.
This function holds legacy behaviour of creating product name
that is deprecated. This wasn't using product name templates
at all, only hardcoded values. It shouldn't be used anymore,
but transition to templates need careful checking of the project
and studio settings.
Deprecated:
since 0.4.4
Args:
product_type (str): Product type.
source_product_name (str): Source product name.
task_name (str): Task name.
dynamic_data (dict): Dynamic data (camera, aov, ...)
Returns:
tuple: product name and group name
"""
warnings.warn("Using legacy product name for renders",
DeprecationWarning)
if not source_product_name.startswith(product_type):
resulting_group_name = '{}{}{}{}{}'.format(
product_type,
task_name[0].upper(), task_name[1:],
source_product_name[0].upper(), source_product_name[1:])
else:
resulting_group_name = source_product_name
# create product name `<product type><Task><Product name>`
if not source_product_name.startswith(product_type):
resulting_group_name = '{}{}{}{}{}'.format(
product_type,
task_name[0].upper(), task_name[1:],
source_product_name[0].upper(), source_product_name[1:])
else:
resulting_group_name = source_product_name
resulting_product_name = resulting_group_name
camera = dynamic_data.get("camera")
aov = dynamic_data.get("aov")
if camera:
if not aov:
resulting_product_name = '{}_{}'.format(
resulting_group_name, camera)
elif not aov.startswith(camera):
resulting_product_name = '{}_{}_{}'.format(
resulting_group_name, camera, aov)
else:
resulting_product_name = "{}_{}".format(
resulting_group_name, aov)
else:
if aov:
resulting_product_name = '{}_{}'.format(
resulting_group_name, aov)
return resulting_product_name, resulting_group_name
def get_product_name_and_group_from_template(
project_name,
task_entity,
product_type,
variant,
host_name,
dynamic_data=None):
"""Get product name and group name from template.
This will get product name and group name from template based on
data provided. It is doing similar work as
`func::_get_legacy_product_name_and_group` but using templates.
To get group name, template is called without any dynamic data, so
(depending on the template itself) it should be product name without
aov.
Todo:
Maybe we should introduce templates for the groups themselves.
Args:
task_entity (dict): Task entity.
project_name (str): Project name.
host_name (str): Host name.
product_type (str): Product type.
variant (str): Variant.
dynamic_data (dict): Dynamic data (aov, renderlayer, camera, ...).
Returns:
tuple: product name and group name.
"""
# remove 'aov' from data used to format group. See todo comment above
# for possible solution.
_dynamic_data = deepcopy(dynamic_data) or {}
_dynamic_data.pop("aov", None)
resulting_group_name = get_product_name(
project_name=project_name,
task_name=task_entity["name"],
task_type=task_entity["taskType"],
host_name=host_name,
product_type=product_type,
dynamic_data=_dynamic_data,
variant=variant,
)
resulting_product_name = get_product_name(
project_name=project_name,
task_name=task_entity["name"],
task_type=task_entity["taskType"],
host_name=host_name,
product_type=product_type,
dynamic_data=dynamic_data,
variant=variant,
)
return resulting_product_name, resulting_group_name
def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
skip_integration_repre_list, do_not_add_review):
"""Create instance for each AOV found.
@ -526,10 +653,10 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
instance (pyblish.api.Instance): Original instance.
skeleton (dict): Skeleton data for instance (those needed) later
by collector.
additional_data (dict): ..
additional_data (dict): ...
skip_integration_repre_list (list): list of extensions that shouldn't
be published
do_not_addbe _review (bool): explicitly disable review
do_not_add_review (bool): explicitly disable review
Returns:
@ -539,68 +666,70 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
ValueError:
"""
# TODO: this needs to be taking the task from context or instance
task = os.environ["AYON_TASK_NAME"]
anatomy = instance.context.data["anatomy"]
s_product_name = skeleton["productName"]
source_product_name = skeleton["productName"]
cameras = instance.data.get("cameras", [])
exp_files = instance.data["expectedFiles"]
expected_files = instance.data["expectedFiles"]
log = Logger.get_logger("farm_publishing")
instances = []
# go through AOVs in expected files
for aov, files in exp_files[0].items():
cols, rem = clique.assemble(files)
# we shouldn't have any reminders. And if we do, it should
# be just one item for single frame renders.
if not cols and rem:
if len(rem) != 1:
raise ValueError("Found multiple non related files "
"to render, don't know what to do "
"with them.")
col = rem[0]
ext = os.path.splitext(col)[1].lstrip(".")
else:
# but we really expect only one collection.
# Nothing else make sense.
if len(cols) != 1:
raise ValueError("Only one image sequence type is expected.") # noqa: E501
ext = cols[0].tail.lstrip(".")
col = list(cols[0])
for aov, files in expected_files[0].items():
collected_files = _collect_expected_files_for_aov(files)
# create product name `<product type><Task><Product name>`
# TODO refactor/remove me
product_type = skeleton["productType"]
if not s_product_name.startswith(product_type):
group_name = '{}{}{}{}{}'.format(
product_type,
task[0].upper(), task[1:],
s_product_name[0].upper(), s_product_name[1:])
else:
group_name = s_product_name
expected_filepath = collected_files
if isinstance(collected_files, (list, tuple)):
expected_filepath = collected_files[0]
# if there are multiple cameras, we need to add camera name
expected_filepath = col[0] if isinstance(col, (list, tuple)) else col
cams = [cam for cam in cameras if cam in expected_filepath]
if cams:
for cam in cams:
if not aov:
product_name = '{}_{}'.format(group_name, cam)
elif not aov.startswith(cam):
product_name = '{}_{}_{}'.format(group_name, cam, aov)
else:
product_name = "{}_{}".format(group_name, aov)
else:
if aov:
product_name = '{}_{}'.format(group_name, aov)
else:
product_name = '{}'.format(group_name)
dynamic_data = {
"aov": aov,
"renderlayer": instance.data.get("renderlayer"),
}
# find if camera is used in the file path
# TODO: this must be changed to be more robust. Any coincidence
# of camera name in the file path will be considered as
# camera name. This is not correct.
camera = [cam for cam in cameras if cam in expected_filepath]
# Is there just one camera matching?
# TODO: this is not true, we can have multiple cameras in the scene
# and we should be able to detect them all. Currently, we are
# keeping the old behavior, taking the first one found.
if camera:
dynamic_data["camera"] = camera[0]
project_settings = instance.context.data.get("project_settings")
use_legacy_product_name = True
try:
use_legacy_product_name = project_settings["core"]["tools"]["creator"]["use_legacy_product_names_for_renders"] # noqa: E501
except KeyError:
warnings.warn(
("use_legacy_for_renders not found in project settings. "
"Using legacy product name for renders. Please update "
"your ayon-core version."), DeprecationWarning)
use_legacy_product_name = True
if use_legacy_product_name:
product_name, group_name = _get_legacy_product_name_and_group(
product_type=skeleton["productType"],
source_product_name=source_product_name,
task_name=instance.data["task"],
dynamic_data=dynamic_data)
if isinstance(col, (list, tuple)):
staging = os.path.dirname(col[0])
else:
staging = os.path.dirname(col)
product_name, group_name = get_product_name_and_group_from_template(
task_entity=instance.data["taskEntity"],
project_name=instance.context.data["projectName"],
host_name=instance.context.data["hostName"],
product_type=skeleton["productType"],
variant=instance.data.get("variant", source_product_name),
dynamic_data=dynamic_data
)
staging = os.path.dirname(expected_filepath)
try:
staging = remap_source(staging, anatomy)
@ -611,10 +740,8 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
app = os.environ.get("AYON_HOST_NAME", "")
if isinstance(col, list):
render_file_name = os.path.basename(col[0])
else:
render_file_name = os.path.basename(col)
render_file_name = os.path.basename(expected_filepath)
aov_patterns = aov_filter
preview = match_aov_pattern(app, aov_patterns, render_file_name)
@ -622,9 +749,10 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
new_instance = deepcopy(skeleton)
new_instance["productName"] = product_name
new_instance["productGroup"] = group_name
new_instance["aov"] = aov
# toggle preview on if multipart is on
# Because we cant query the multipartExr data member of each AOV we'll
# Because we can't query the multipartExr data member of each AOV we'll
# need to have hardcoded rule of excluding any renders with
# "cryptomatte" in the file name from being a multipart EXR. This issue
# happens with Redshift that forces Cryptomatte renders to be separate
@ -650,10 +778,7 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
new_instance["review"] = True
# create representation
if isinstance(col, (list, tuple)):
files = [os.path.basename(f) for f in col]
else:
files = os.path.basename(col)
ext = os.path.splitext(render_file_name)[-1].lstrip(".")
# Copy render product "colorspace" data to representation.
colorspace = ""
@ -708,6 +833,35 @@ def _create_instances_for_aov(instance, skeleton, aov_filter, additional_data,
return instances
def _collect_expected_files_for_aov(files):
"""Collect expected files.
Args:
files (list): List of files.
Returns:
list or str: Collection of files or single file.
Raises:
ValueError: If there are multiple collections.
"""
cols, rem = clique.assemble(files)
# we shouldn't have any reminders. And if we do, it should
# be just one item for single frame renders.
if not cols and rem:
if len(rem) != 1:
raise ValueError("Found multiple non related files "
"to render, don't know what to do "
"with them.")
return rem[0]
# but we really expect only one collection.
# Nothing else make sense.
if len(cols) != 1:
raise ValueError("Only one image sequence type is expected.") # noqa: E501
return list(cols[0])
def get_resources(project_name, version_entity, extension=None):
"""Get the files from the specific version.

View file

@ -1,24 +0,0 @@
import pyblish.api
from ayon_core.pipeline import Anatomy
from typing import Tuple, List
class TimeData:
start: int
end: int
fps: float | int
step: int
handle_start: int
handle_end: int
def __init__(self, start: int, end: int, fps: float | int, step: int, handle_start: int, handle_end: int):
...
...
def remap_source(source: str, anatomy: Anatomy): ...
def extend_frames(folder_path: str, product_name: str, start: int, end: int) -> Tuple[int, int]: ...
def get_time_data_from_instance_or_context(instance: pyblish.api.Instance) -> TimeData: ...
def get_transferable_representations(instance: pyblish.api.Instance) -> list: ...
def create_skeleton_instance(instance: pyblish.api.Instance, families_transfer: list = ..., instance_transfer: dict = ...) -> dict: ...
def create_instances_for_aov(instance: pyblish.api.Instance, skeleton: dict, aov_filter: dict) -> List[pyblish.api.Instance]: ...
def attach_instances_to_product(attach_to: list, instances: list) -> list: ...

View file

@ -138,7 +138,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
folder_path_by_id = {}
for instance in context:
folder_entity = instance.data.get("folderEntity")
# Skip if instnace does not have filled folder entity
# Skip if instance does not have filled folder entity
if not folder_entity:
continue
folder_id = folder_entity["id"]
@ -385,8 +385,19 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
json.dumps(anatomy_data, indent=4)
))
# make render layer available in anatomy data
render_layer = instance.data.get("renderlayer")
if render_layer:
anatomy_data["renderlayer"] = render_layer
# make aov name available in anatomy data
aov = instance.data.get("aov")
if aov:
anatomy_data["aov"] = aov
def _fill_folder_data(self, instance, project_entity, anatomy_data):
# QUESTION should we make sure that all folder data are poped if
# QUESTION: should we make sure that all folder data are popped if
# folder data cannot be found?
# - 'folder', 'hierarchy', 'parent', 'folder'
folder_entity = instance.data.get("folderEntity")
@ -426,7 +437,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
})
def _fill_task_data(self, instance, task_types_by_name, anatomy_data):
# QUESTION should we make sure that all task data are poped if task
# QUESTION: should we make sure that all task data are popped if task
# data cannot be resolved?
# - 'task'

View file

@ -11,7 +11,11 @@ Multiples instances from your scene are set to publish into the same folder > pr
### How to repair?
Remove the offending instances or rename to have a unique name.
Remove the offending instances or rename to have a unique name. Also, please
check your product name templates to ensure that resolved names are
sufficiently unique. You can find that settings:
ayon+settings://core/tools/creator/product_name_profiles
</description>
</error>
</root>
</root>

View file

@ -744,6 +744,11 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
if not is_udim:
repre_context["frame"] = first_index_padded
# store renderlayer in context if it exists
# to be later used for example by delivery templates
if instance.data.get("renderlayer"):
repre_context["renderlayer"] = instance.data["renderlayer"]
# Update the destination indexes and padding
dst_collection = clique.assemble(dst_filepaths)[0][0]
dst_collection.padding = destination_padding

View file

@ -562,12 +562,12 @@ class ExtractBurninDef(BaseSettingsModel):
_isGroup = True
_layout = "expanded"
name: str = SettingsField("")
TOP_LEFT: str = SettingsField("", topic="Top Left")
TOP_CENTERED: str = SettingsField("", topic="Top Centered")
TOP_RIGHT: str = SettingsField("", topic="Top Right")
BOTTOM_LEFT: str = SettingsField("", topic="Bottom Left")
BOTTOM_CENTERED: str = SettingsField("", topic="Bottom Centered")
BOTTOM_RIGHT: str = SettingsField("", topic="Bottom Right")
TOP_LEFT: str = SettingsField("", title="Top Left")
TOP_CENTERED: str = SettingsField("", title="Top Centered")
TOP_RIGHT: str = SettingsField("", title="Top Right")
BOTTOM_LEFT: str = SettingsField("", title="Bottom Left")
BOTTOM_CENTERED: str = SettingsField("", title="Bottom Centered")
BOTTOM_RIGHT: str = SettingsField("", title="Bottom Right")
filter: ExtractBurninDefFilter = SettingsField(
default_factory=ExtractBurninDefFilter,
title="Additional filtering"

View file

@ -22,6 +22,7 @@ class ProductTypeSmartSelectModel(BaseSettingsModel):
class ProductNameProfile(BaseSettingsModel):
_layout = "expanded"
product_types: list[str] = SettingsField(
default_factory=list, title="Product types"
)
@ -65,6 +66,15 @@ class CreatorToolModel(BaseSettingsModel):
title="Create Smart Select"
)
)
# TODO: change to False in next releases
use_legacy_product_names_for_renders: bool = SettingsField(
True,
title="Use legacy product names for renders",
description="Use product naming templates for renders. "
"This is for backwards compatibility enabled by default."
"When enabled, it will ignore any templates for renders "
"that are set in the product name profiles.")
product_name_profiles: list[ProductNameProfile] = SettingsField(
default_factory=list,
title="Product name profiles"