diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 0762eb2f20..87d904fc84 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -35,6 +35,8 @@ body: label: Version description: What version are you running? Look to OpenPype Tray options: + - 3.17.1-nightly.2 + - 3.17.1-nightly.1 - 3.17.0 - 3.16.7 - 3.16.7-nightly.2 @@ -133,8 +135,6 @@ body: - 3.14.10-nightly.5 - 3.14.10-nightly.4 - 3.14.10-nightly.3 - - 3.14.10-nightly.2 - - 3.14.10-nightly.1 validations: required: true - type: dropdown diff --git a/openpype/hosts/maya/plugins/create/create_unreal_yeticache.py b/openpype/hosts/maya/plugins/create/create_unreal_yeticache.py new file mode 100644 index 0000000000..c9f9cd9ba8 --- /dev/null +++ b/openpype/hosts/maya/plugins/create/create_unreal_yeticache.py @@ -0,0 +1,39 @@ +from openpype.hosts.maya.api import ( + lib, + plugin +) +from openpype.lib import NumberDef + + +class CreateYetiCache(plugin.MayaCreator): + """Output for procedural plugin nodes of Yeti """ + + identifier = "io.openpype.creators.maya.unrealyeticache" + label = "Unreal - Yeti Cache" + family = "yeticacheUE" + icon = "pagelines" + + def get_instance_attr_defs(self): + + defs = [ + NumberDef("preroll", + label="Preroll", + minimum=0, + default=0, + decimals=0) + ] + + # Add animation data without step and handles + defs.extend(lib.collect_animation_defs()) + remove = {"step", "handleStart", "handleEnd"} + defs = [attr_def for attr_def in defs if attr_def.key not in remove] + + # Add samples after frame range + defs.append( + NumberDef("samples", + label="Samples", + default=3, + decimals=0) + ) + + return defs diff --git a/openpype/hosts/maya/plugins/publish/collect_yeti_cache.py b/openpype/hosts/maya/plugins/publish/collect_yeti_cache.py index 4dcda29050..7a1516997a 100644 --- a/openpype/hosts/maya/plugins/publish/collect_yeti_cache.py +++ b/openpype/hosts/maya/plugins/publish/collect_yeti_cache.py @@ -39,7 +39,7 @@ class CollectYetiCache(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.45 label = "Collect Yeti Cache" - families = ["yetiRig", "yeticache"] + families = ["yetiRig", "yeticache", "yeticacheUE"] hosts = ["maya"] def process(self, instance): diff --git a/openpype/hosts/maya/plugins/publish/extract_unreal_yeticache.py b/openpype/hosts/maya/plugins/publish/extract_unreal_yeticache.py new file mode 100644 index 0000000000..963285093e --- /dev/null +++ b/openpype/hosts/maya/plugins/publish/extract_unreal_yeticache.py @@ -0,0 +1,61 @@ +import os + +from maya import cmds + +from openpype.pipeline import publish + + +class ExtractYetiCache(publish.Extractor): + """Producing Yeti cache files using scene time range. + + This will extract Yeti cache file sequence and fur settings. + """ + + label = "Extract Yeti Cache" + hosts = ["maya"] + families = ["yeticacheUE"] + + def process(self, instance): + + yeti_nodes = cmds.ls(instance, type="pgYetiMaya") + if not yeti_nodes: + raise RuntimeError("No pgYetiMaya nodes found in the instance") + + # Define extract output file path + dirname = self.staging_dir(instance) + + # Collect information for writing cache + start_frame = instance.data["frameStartHandle"] + end_frame = instance.data["frameEndHandle"] + preroll = instance.data["preroll"] + if preroll > 0: + start_frame -= preroll + + kwargs = {} + samples = instance.data.get("samples", 0) + if samples == 0: + kwargs.update({"sampleTimes": "0.0 1.0"}) + else: + kwargs.update({"samples": samples}) + + self.log.debug(f"Writing out cache {start_frame} - {end_frame}") + filename = f"{instance.name}.abc" + path = os.path.join(dirname, filename) + cmds.pgYetiCommand(yeti_nodes, + writeAlembic=path, + range=(start_frame, end_frame), + asUnrealAbc=True, + **kwargs) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'abc', + 'ext': 'abc', + 'files': filename, + 'stagingDir': dirname + } + instance.data["representations"].append(representation) + + self.log.debug(f"Extracted {instance} to {dirname}") diff --git a/openpype/hosts/nuke/api/lib.py b/openpype/hosts/nuke/api/lib.py index 41e6a27cef..fb2b5d0f45 100644 --- a/openpype/hosts/nuke/api/lib.py +++ b/openpype/hosts/nuke/api/lib.py @@ -2316,27 +2316,53 @@ Reopening Nuke should synchronize these paths and resolve any discrepancies. ''' Adds correct colorspace to write node dict ''' - for node in nuke.allNodes(filter="Group"): + for node in nuke.allNodes(filter="Group", group=self._root_node): + log.info("Setting colorspace to `{}`".format(node.name())) # get data from avalon knob avalon_knob_data = read_avalon_data(node) + node_data = get_node_data(node, INSTANCE_DATA_KNOB) - if avalon_knob_data.get("id") != "pyblish.avalon.instance": + if ( + # backward compatibility + # TODO: remove this once old avalon data api will be removed + avalon_knob_data + and avalon_knob_data.get("id") != "pyblish.avalon.instance" + ): + continue + elif ( + node_data + and node_data.get("id") != "pyblish.avalon.instance" + ): continue - if "creator" not in avalon_knob_data: + if ( + # backward compatibility + # TODO: remove this once old avalon data api will be removed + avalon_knob_data + and "creator" not in avalon_knob_data + ): + continue + elif ( + node_data + and "creator_identifier" not in node_data + ): continue - # establish families - families = [avalon_knob_data["family"]] - if avalon_knob_data.get("families"): - families.append(avalon_knob_data.get("families")) + nuke_imageio_writes = None + if avalon_knob_data: + # establish families + families = [avalon_knob_data["family"]] + if avalon_knob_data.get("families"): + families.append(avalon_knob_data.get("families")) - nuke_imageio_writes = get_imageio_node_setting( - node_class=avalon_knob_data["families"], - plugin_name=avalon_knob_data["creator"], - subset=avalon_knob_data["subset"] - ) + nuke_imageio_writes = get_imageio_node_setting( + node_class=avalon_knob_data["families"], + plugin_name=avalon_knob_data["creator"], + subset=avalon_knob_data["subset"] + ) + elif node_data: + nuke_imageio_writes = get_write_node_template_attr(node) log.debug("nuke_imageio_writes: `{}`".format(nuke_imageio_writes)) diff --git a/openpype/hosts/nuke/api/plugin.py b/openpype/hosts/nuke/api/plugin.py index a0e1525cd0..6d5d7eddf1 100644 --- a/openpype/hosts/nuke/api/plugin.py +++ b/openpype/hosts/nuke/api/plugin.py @@ -580,18 +580,25 @@ class ExporterReview(object): def get_file_info(self): if self.collection: # get path - self.fname = os.path.basename(self.collection.format( - "{head}{padding}{tail}")) + self.fname = os.path.basename( + self.collection.format("{head}{padding}{tail}") + ) self.fhead = self.collection.format("{head}") # get first and last frame self.first_frame = min(self.collection.indexes) self.last_frame = max(self.collection.indexes) + + # make sure slate frame is not included + frame_start_handle = self.instance.data["frameStartHandle"] + if frame_start_handle > self.first_frame: + self.first_frame = frame_start_handle + else: self.fname = os.path.basename(self.path_in) self.fhead = os.path.splitext(self.fname)[0] + "." - self.first_frame = self.instance.data.get("frameStartHandle", None) - self.last_frame = self.instance.data.get("frameEndHandle", None) + self.first_frame = self.instance.data["frameStartHandle"] + self.last_frame = self.instance.data["frameEndHandle"] if "#" in self.fhead: self.fhead = self.fhead.replace("#", "")[:-1] @@ -869,6 +876,11 @@ class ExporterReviewMov(ExporterReview): r_node["origlast"].setValue(self.last_frame) r_node["colorspace"].setValue(self.write_colorspace) + # do not rely on defaults, set explicitly + # to be sure it is set correctly + r_node["frame_mode"].setValue("expression") + r_node["frame"].setValue("") + if read_raw: r_node["raw"].setValue(1) diff --git a/openpype/hosts/photoshop/lib.py b/openpype/hosts/photoshop/lib.py index ae7a33b7b6..9f603a70d2 100644 --- a/openpype/hosts/photoshop/lib.py +++ b/openpype/hosts/photoshop/lib.py @@ -1,5 +1,8 @@ +import re + import openpype.hosts.photoshop.api as api from openpype.client import get_asset_by_name +from openpype.lib import prepare_template_data from openpype.pipeline import ( AutoCreator, CreatedInstance @@ -78,3 +81,17 @@ class PSAutoCreator(AutoCreator): existing_instance["asset"] = asset_name existing_instance["task"] = task_name existing_instance["subset"] = subset_name + + +def clean_subset_name(subset_name): + """Clean all variants leftover {layer} from subset name.""" + dynamic_data = prepare_template_data({"layer": "{layer}"}) + for value in dynamic_data.values(): + if value in subset_name: + subset_name = (subset_name.replace(value, "") + .replace("__", "_") + .replace("..", ".")) + # clean trailing separator as Main_ + pattern = r'[\W_]+$' + replacement = '' + return re.sub(pattern, replacement, subset_name) diff --git a/openpype/hosts/photoshop/plugins/create/create_flatten_image.py b/openpype/hosts/photoshop/plugins/create/create_flatten_image.py index e4229788bd..afde77fdb4 100644 --- a/openpype/hosts/photoshop/plugins/create/create_flatten_image.py +++ b/openpype/hosts/photoshop/plugins/create/create_flatten_image.py @@ -2,7 +2,7 @@ from openpype.pipeline import CreatedInstance from openpype.lib import BoolDef import openpype.hosts.photoshop.api as api -from openpype.hosts.photoshop.lib import PSAutoCreator +from openpype.hosts.photoshop.lib import PSAutoCreator, clean_subset_name from openpype.pipeline.create import get_subset_name from openpype.lib import prepare_template_data from openpype.client import get_asset_by_name @@ -129,14 +129,4 @@ class AutoImageCreator(PSAutoCreator): self.family, variant, task_name, asset_doc, project_name, host_name, dynamic_data=dynamic_data ) - return self._clean_subset_name(subset_name) - - def _clean_subset_name(self, subset_name): - """Clean all variants leftover {layer} from subset name.""" - dynamic_data = prepare_template_data({"layer": "{layer}"}) - for value in dynamic_data.values(): - if value in subset_name: - return (subset_name.replace(value, "") - .replace("__", "_") - .replace("..", ".")) - return subset_name + return clean_subset_name(subset_name) diff --git a/openpype/hosts/photoshop/plugins/create/create_image.py b/openpype/hosts/photoshop/plugins/create/create_image.py index af20d456e0..4f2e90886a 100644 --- a/openpype/hosts/photoshop/plugins/create/create_image.py +++ b/openpype/hosts/photoshop/plugins/create/create_image.py @@ -10,6 +10,7 @@ from openpype.pipeline import ( from openpype.lib import prepare_template_data from openpype.pipeline.create import SUBSET_NAME_ALLOWED_SYMBOLS from openpype.hosts.photoshop.api.pipeline import cache_and_get_instances +from openpype.hosts.photoshop.lib import clean_subset_name class ImageCreator(Creator): @@ -88,6 +89,7 @@ class ImageCreator(Creator): layer_fill = prepare_template_data({"layer": layer_name}) subset_name = subset_name.format(**layer_fill) + subset_name = clean_subset_name(subset_name) if group.long_name: for directory in group.long_name[::-1]: @@ -184,7 +186,6 @@ class ImageCreator(Creator): self.mark_for_review = plugin_settings["mark_for_review"] self.enabled = plugin_settings["enabled"] - def get_detail_description(self): return """Creator for Image instances diff --git a/openpype/hosts/unreal/plugins/load/load_yeticache.py b/openpype/hosts/unreal/plugins/load/load_yeticache.py new file mode 100644 index 0000000000..22f5029bac --- /dev/null +++ b/openpype/hosts/unreal/plugins/load/load_yeticache.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +"""Loader for Yeti Cache.""" +import os +import json + +from openpype.pipeline import ( + get_representation_path, + AYON_CONTAINER_ID +) +from openpype.hosts.unreal.api import plugin +from openpype.hosts.unreal.api import pipeline as unreal_pipeline +import unreal # noqa + + +class YetiLoader(plugin.Loader): + """Load Yeti Cache""" + + families = ["yeticacheUE"] + label = "Import Yeti" + representations = ["abc"] + icon = "pagelines" + color = "orange" + + @staticmethod + def get_task(filename, asset_dir, asset_name, replace): + task = unreal.AssetImportTask() + options = unreal.AbcImportSettings() + + task.set_editor_property('filename', filename) + task.set_editor_property('destination_path', asset_dir) + task.set_editor_property('destination_name', asset_name) + task.set_editor_property('replace_existing', replace) + task.set_editor_property('automated', True) + task.set_editor_property('save', True) + + task.options = options + + return task + + @staticmethod + def is_groom_module_active(): + """ + Check if Groom plugin is active. + + This is a workaround, because the Unreal python API don't have + any method to check if plugin is active. + """ + prj_file = unreal.Paths.get_project_file_path() + + with open(prj_file, "r") as fp: + data = json.load(fp) + + plugins = data.get("Plugins") + + if not plugins: + return False + + plugin_names = [p.get("Name") for p in plugins] + + return "HairStrands" in plugin_names + + def load(self, context, name, namespace, options): + """Load and containerise representation into Content Browser. + + This is two step process. First, import FBX to temporary path and + then call `containerise()` on it - this moves all content to new + directory and then it will create AssetContainer there and imprint it + with metadata. This will mark this path as container. + + Args: + context (dict): application context + name (str): subset name + namespace (str): in Unreal this is basically path to container. + This is not passed here, so namespace is set + by `containerise()` because only then we know + real path. + data (dict): Those would be data to be imprinted. This is not used + now, data are imprinted by `containerise()`. + + Returns: + list(str): list of container content + + """ + # Check if Groom plugin is active + if not self.is_groom_module_active(): + raise RuntimeError("Groom plugin is not activated.") + + # Create directory for asset and Ayon container + root = "/Game/Ayon/Assets" + asset = context.get('asset').get('name') + suffix = "_CON" + asset_name = f"{asset}_{name}" if asset else f"{name}" + + tools = unreal.AssetToolsHelpers().get_asset_tools() + asset_dir, container_name = tools.create_unique_asset_name( + f"{root}/{asset}/{name}", suffix="") + + unique_number = 1 + while unreal.EditorAssetLibrary.does_directory_exist( + f"{asset_dir}_{unique_number:02}" + ): + unique_number += 1 + + asset_dir = f"{asset_dir}_{unique_number:02}" + container_name = f"{container_name}_{unique_number:02}{suffix}" + + if not unreal.EditorAssetLibrary.does_directory_exist(asset_dir): + unreal.EditorAssetLibrary.make_directory(asset_dir) + + path = self.filepath_from_context(context) + task = self.get_task(path, asset_dir, asset_name, False) + + unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) # noqa: E501 + + # Create Asset Container + unreal_pipeline.create_container( + container=container_name, path=asset_dir) + + data = { + "schema": "ayon:container-2.0", + "id": AYON_CONTAINER_ID, + "asset": asset, + "namespace": asset_dir, + "container_name": container_name, + "asset_name": asset_name, + "loader": str(self.__class__.__name__), + "representation": context["representation"]["_id"], + "parent": context["representation"]["parent"], + "family": context["representation"]["context"]["family"] + } + unreal_pipeline.imprint(f"{asset_dir}/{container_name}", data) + + asset_content = unreal.EditorAssetLibrary.list_assets( + asset_dir, recursive=True, include_folder=True + ) + + for a in asset_content: + unreal.EditorAssetLibrary.save_asset(a) + + return asset_content + + def update(self, container, representation): + name = container["asset_name"] + source_path = get_representation_path(representation) + destination_path = container["namespace"] + + task = self.get_task(source_path, destination_path, name, True) + + # do import fbx and replace existing data + unreal.AssetToolsHelpers.get_asset_tools().import_asset_tasks([task]) + + container_path = f'{container["namespace"]}/{container["objectName"]}' + # update metadata + unreal_pipeline.imprint( + container_path, + { + "representation": str(representation["_id"]), + "parent": str(representation["parent"]) + }) + + asset_content = unreal.EditorAssetLibrary.list_assets( + destination_path, recursive=True, include_folder=True + ) + + for a in asset_content: + unreal.EditorAssetLibrary.save_asset(a) + + def remove(self, container): + path = container["namespace"] + parent_path = os.path.dirname(path) + + unreal.EditorAssetLibrary.delete_directory(path) + + asset_content = unreal.EditorAssetLibrary.list_assets( + parent_path, recursive=False + ) + + if len(asset_content) == 0: + unreal.EditorAssetLibrary.delete_directory(parent_path) diff --git a/openpype/pipeline/colorspace.py b/openpype/pipeline/colorspace.py index 44e5cb6c47..2800050496 100644 --- a/openpype/pipeline/colorspace.py +++ b/openpype/pipeline/colorspace.py @@ -2,9 +2,12 @@ from copy import deepcopy import re import os import json -import platform import contextlib +import functools +import platform import tempfile +import warnings + from openpype import PACKAGE_DIR from openpype.settings import get_project_settings from openpype.lib import ( @@ -20,12 +23,60 @@ log = Logger.get_logger(__name__) class CachedData: - remapping = {} + remapping = None + has_compatible_ocio_package = None + config_version_data = {} + ocio_config_colorspaces = {} allowed_exts = { ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) } +class DeprecatedWarning(DeprecationWarning): + pass + + +def deprecated(new_destination): + """Mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + + func = None + if callable(new_destination): + func = new_destination + new_destination = None + + def _decorator(decorated_func): + if new_destination is None: + warning_message = ( + " Please check content of deprecated function to figure out" + " possible replacement." + ) + else: + warning_message = " Please replace your usage with '{}'.".format( + new_destination + ) + + @functools.wraps(decorated_func) + def wrapper(*args, **kwargs): + warnings.simplefilter("always", DeprecatedWarning) + warnings.warn( + ( + "Call to deprecated function '{}'" + "\nFunction was moved or removed.{}" + ).format(decorated_func.__name__, warning_message), + category=DeprecatedWarning, + stacklevel=4 + ) + return decorated_func(*args, **kwargs) + return wrapper + + if func is None: + return _decorator + return _decorator(func) + + @contextlib.contextmanager def _make_temp_json_file(): """Wrapping function for json temp file @@ -69,124 +120,264 @@ def get_ocio_config_script_path(): ) -def get_imageio_colorspace_from_filepath( - path, host_name, project_name, +def get_colorspace_name_from_filepath( + filepath, host_name, project_name, config_data=None, file_rules=None, project_settings=None, validate=True ): """Get colorspace name from filepath - ImageIO Settings file rules are tested for matching rule. - Args: - path (str): path string, file rule pattern is tested on it + filepath (str): path string, file rule pattern is tested on it host_name (str): host name project_name (str): project name - config_data (dict, optional): config path and template in dict. + config_data (Optional[dict]): config path and template in dict. Defaults to None. - file_rules (dict, optional): file rule data from settings. + file_rules (Optional[dict]): file rule data from settings. Defaults to None. - project_settings (dict, optional): project settings. Defaults to None. - validate (bool, optional): should resulting colorspace be validated - with config file? Defaults to True. + project_settings (Optional[dict]): project settings. Defaults to None. + validate (Optional[bool]): should resulting colorspace be validated + with config file? Defaults to True. Returns: str: name of colorspace """ - if not any([config_data, file_rules]): - project_settings = project_settings or get_project_settings( - project_name - ) - config_data = get_imageio_config( - project_name, host_name, project_settings) + project_settings, config_data, file_rules = _get_context_settings( + host_name, project_name, + config_data=config_data, file_rules=file_rules, + project_settings=project_settings + ) - # in case host color management is not enabled - if not config_data: - return None + if not config_data: + # in case global or host color management is not enabled + return None - file_rules = get_imageio_file_rules( - project_name, host_name, project_settings) + # use ImageIO file rules + colorspace_name = get_imageio_file_rules_colorspace_from_filepath( + filepath, host_name, project_name, + config_data=config_data, file_rules=file_rules, + project_settings=project_settings + ) - # match file rule from path - colorspace_name = None - for _frule_name, file_rule in file_rules.items(): - pattern = file_rule["pattern"] - extension = file_rule["ext"] - ext_match = re.match( - r".*(?=.{})".format(extension), path - ) - file_match = re.search( - pattern, path - ) + # try to get colorspace from OCIO v2 file rules + if ( + not colorspace_name + and compatibility_check_config_version(config_data["path"], major=2) + ): + colorspace_name = get_config_file_rules_colorspace_from_filepath( + config_data["path"], filepath) - if ext_match and file_match: - colorspace_name = file_rule["colorspace"] + # use parse colorspace from filepath as fallback + colorspace_name = colorspace_name or parse_colorspace_from_filepath( + filepath, config_path=config_data["path"] + ) if not colorspace_name: log.info("No imageio file rule matched input path: '{}'".format( - path + filepath )) return None # validate matching colorspace with config - if validate and config_data: + if validate: validate_imageio_colorspace_in_config( config_data["path"], colorspace_name) return colorspace_name -def parse_colorspace_from_filepath( - path, host_name, project_name, - config_data=None, +# TODO: remove this in future - backward compatibility +@deprecated("get_imageio_file_rules_colorspace_from_filepath") +def get_imageio_colorspace_from_filepath(*args, **kwargs): + return get_imageio_file_rules_colorspace_from_filepath(*args, **kwargs) + +# TODO: remove this in future - backward compatibility +@deprecated("get_imageio_file_rules_colorspace_from_filepath") +def get_colorspace_from_filepath(*args, **kwargs): + return get_imageio_file_rules_colorspace_from_filepath(*args, **kwargs) + + +def _get_context_settings( + host_name, project_name, + config_data=None, file_rules=None, project_settings=None +): + project_settings = project_settings or get_project_settings( + project_name + ) + + config_data = config_data or get_imageio_config( + project_name, host_name, project_settings) + + # in case host color management is not enabled + if not config_data: + return (None, None, None) + + file_rules = file_rules or get_imageio_file_rules( + project_name, host_name, project_settings) + + return project_settings, config_data, file_rules + + +def get_imageio_file_rules_colorspace_from_filepath( + filepath, host_name, project_name, + config_data=None, file_rules=None, + project_settings=None +): + """Get colorspace name from filepath + + ImageIO Settings file rules are tested for matching rule. + + Args: + filepath (str): path string, file rule pattern is tested on it + host_name (str): host name + project_name (str): project name + config_data (Optional[dict]): config path and template in dict. + Defaults to None. + file_rules (Optional[dict]): file rule data from settings. + Defaults to None. + project_settings (Optional[dict]): project settings. Defaults to None. + + Returns: + str: name of colorspace + """ + project_settings, config_data, file_rules = _get_context_settings( + host_name, project_name, + config_data=config_data, file_rules=file_rules, + project_settings=project_settings + ) + + if not config_data: + # in case global or host color management is not enabled + return None + + # match file rule from path + colorspace_name = None + for file_rule in file_rules.values(): + pattern = file_rule["pattern"] + extension = file_rule["ext"] + ext_match = re.match( + r".*(?=.{})".format(extension), filepath + ) + file_match = re.search( + pattern, filepath + ) + + if ext_match and file_match: + colorspace_name = file_rule["colorspace"] + + return colorspace_name + + +def get_config_file_rules_colorspace_from_filepath(config_path, filepath): + """Get colorspace from file path wrapper. + + Wrapper function for getting colorspace from file path + with use of OCIO v2 file-rules. + + Args: + config_path (str): path leading to config.ocio file + filepath (str): path leading to a file + + Returns: + Any[str, None]: matching colorspace name + """ + if not compatibility_check(): + # python environment is not compatible with PyOpenColorIO + # needs to be run in subprocess + result_data = _get_wrapped_with_subprocess( + "colorspace", "get_config_file_rules_colorspace_from_filepath", + config_path=config_path, + filepath=filepath + ) + if result_data: + return result_data[0] + + # TODO: refactor this so it is not imported but part of this file + from openpype.scripts.ocio_wrapper import _get_config_file_rules_colorspace_from_filepath # noqa: E501 + + result_data = _get_config_file_rules_colorspace_from_filepath( + config_path, filepath) + + if result_data: + return result_data[0] + + +def parse_colorspace_from_filepath( + filepath, colorspaces=None, config_path=None ): """Parse colorspace name from filepath An input path can have colorspace name used as part of name or as folder name. + Example: + >>> config_path = "path/to/config.ocio" + >>> colorspaces = get_ocio_config_colorspaces(config_path) + >>> colorspace = parse_colorspace_from_filepath( + "path/to/file/acescg/file.exr", + colorspaces=colorspaces + ) + >>> print(colorspace) + acescg + Args: - path (str): path string - host_name (str): host name - project_name (str): project name - config_data (dict, optional): config path and template in dict. - Defaults to None. - project_settings (dict, optional): project settings. Defaults to None. + filepath (str): path string + colorspaces (Optional[dict[str]]): list of colorspaces + config_path (Optional[str]): path to config.ocio file Returns: str: name of colorspace """ - if not config_data: - project_settings = project_settings or get_project_settings( - project_name + def _get_colorspace_match_regex(colorspaces): + """Return a regex pattern + + Allows to search a colorspace match in a filename + + Args: + colorspaces (list): List of colorspace names + + Returns: + re.Pattern: regex pattern + """ + pattern = "|".join( + # Allow to match spaces also as underscores because the + # integrator replaces spaces with underscores in filenames + re.escape(colorspace) for colorspace in + # Sort by longest first so the regex matches longer matches + # over smaller matches, e.g. matching 'Output - sRGB' over 'sRGB' + sorted(colorspaces, key=len, reverse=True) ) - config_data = get_imageio_config( - project_name, host_name, project_settings) + return re.compile(pattern) - config_path = config_data["path"] + if not colorspaces and not config_path: + raise ValueError( + "Must provide `config_path` if `colorspaces` is not provided." + ) - # match file rule from path - colorspace_name = None - colorspaces = get_ocio_config_colorspaces(config_path) - for colorspace_key in colorspaces: - # check underscored variant of colorspace name - # since we are reformatting it in integrate.py - if colorspace_key.replace(" ", "_") in path: - colorspace_name = colorspace_key - break - if colorspace_key in path: - colorspace_name = colorspace_key - break + colorspaces = colorspaces or get_ocio_config_colorspaces(config_path) + underscored_colorspaces = { + key.replace(" ", "_"): key for key in colorspaces + if " " in key + } - if not colorspace_name: - log.info("No matching colorspace in config '{}' for path: '{}'".format( - config_path, path - )) - return None + # match colorspace from filepath + regex_pattern = _get_colorspace_match_regex( + list(colorspaces) + list(underscored_colorspaces)) + match = regex_pattern.search(filepath) + colorspace = match.group(0) if match else None - return colorspace_name + if colorspace in underscored_colorspaces: + return underscored_colorspaces[colorspace] + + if colorspace: + return colorspace + + log.info("No matching colorspace in config '{}' for path: '{}'".format( + config_path, filepath + )) + return None def validate_imageio_colorspace_in_config(config_path, colorspace_name): @@ -211,49 +402,101 @@ def validate_imageio_colorspace_in_config(config_path, colorspace_name): return True +# TODO: remove this in future - backward compatibility +@deprecated("_get_wrapped_with_subprocess") def get_data_subprocess(config_path, data_type): - """Get data via subprocess + """[Deprecated] Get data via subprocess Wrapper for Python 2 hosts. Args: config_path (str): path leading to config.ocio file """ + return _get_wrapped_with_subprocess( + "config", data_type, in_path=config_path, + ) + + +def _get_wrapped_with_subprocess(command_group, command, **kwargs): + """Get data via subprocess + + Wrapper for Python 2 hosts. + + Args: + command_group (str): command group name + command (str): command name + **kwargs: command arguments + + Returns: + Any[dict, None]: data + """ with _make_temp_json_file() as tmp_json_path: # Prepare subprocess arguments args = [ "run", get_ocio_config_script_path(), - "config", data_type, - "--in_path", config_path, - "--out_path", tmp_json_path - + command_group, command ] + + for key_, value_ in kwargs.items(): + args.extend(("--{}".format(key_), value_)) + + args.append("--out_path") + args.append(tmp_json_path) + log.info("Executing: {}".format(" ".join(args))) - process_kwargs = { - "logger": log - } - - run_openpype_process(*args, **process_kwargs) + run_openpype_process(*args, logger=log) # return all colorspaces - return_json_data = open(tmp_json_path).read() - return json.loads(return_json_data) + with open(tmp_json_path, "r") as f_: + return json.load(f_) +# TODO: this should be part of ocio_wrapper.py def compatibility_check(): - """checking if user has a compatible PyOpenColorIO >= 2. + """Making sure PyOpenColorIO is importable""" + if CachedData.has_compatible_ocio_package is not None: + return CachedData.has_compatible_ocio_package - It's achieved by checking if PyOpenColorIO is importable - and calling any version 2 specific function - """ try: - import PyOpenColorIO + import PyOpenColorIO # noqa: F401 + CachedData.has_compatible_ocio_package = True + except ImportError: + CachedData.has_compatible_ocio_package = False - # ocio versions lower than 2 will raise AttributeError - PyOpenColorIO.GetVersion() - except (ImportError, AttributeError): + # compatible + return CachedData.has_compatible_ocio_package + + +# TODO: this should be part of ocio_wrapper.py +def compatibility_check_config_version(config_path, major=1, minor=None): + """Making sure PyOpenColorIO config version is compatible""" + + if not CachedData.config_version_data.get(config_path): + if compatibility_check(): + # TODO: refactor this so it is not imported but part of this file + from openpype.scripts.ocio_wrapper import _get_version_data + + CachedData.config_version_data[config_path] = \ + _get_version_data(config_path) + + else: + # python environment is not compatible with PyOpenColorIO + # needs to be run in subprocess + CachedData.config_version_data[config_path] = \ + _get_wrapped_with_subprocess( + "config", "get_version", config_path=config_path + ) + + # check major version + if CachedData.config_version_data[config_path]["major"] != major: return False + + # check minor version + if minor and CachedData.config_version_data[config_path]["minor"] != minor: + return False + + # compatible return True @@ -269,18 +512,28 @@ def get_ocio_config_colorspaces(config_path): Returns: dict: colorspace and family in couple """ - if not compatibility_check(): - # python environment is not compatible with PyOpenColorIO - # needs to be run in subprocess - return get_colorspace_data_subprocess(config_path) + if not CachedData.ocio_config_colorspaces.get(config_path): + if not compatibility_check(): + # python environment is not compatible with PyOpenColorIO + # needs to be run in subprocess + CachedData.ocio_config_colorspaces[config_path] = \ + _get_wrapped_with_subprocess( + "config", "get_colorspace", in_path=config_path + ) + else: + # TODO: refactor this so it is not imported but part of this file + from openpype.scripts.ocio_wrapper import _get_colorspace_data - from openpype.scripts.ocio_wrapper import _get_colorspace_data + CachedData.ocio_config_colorspaces[config_path] = \ + _get_colorspace_data(config_path) - return _get_colorspace_data(config_path) + return CachedData.ocio_config_colorspaces[config_path] +# TODO: remove this in future - backward compatibility +@deprecated("_get_wrapped_with_subprocess") def get_colorspace_data_subprocess(config_path): - """Get colorspace data via subprocess + """[Deprecated] Get colorspace data via subprocess Wrapper for Python 2 hosts. @@ -290,7 +543,9 @@ def get_colorspace_data_subprocess(config_path): Returns: dict: colorspace and family in couple """ - return get_data_subprocess(config_path, "get_colorspace") + return _get_wrapped_with_subprocess( + "config", "get_colorspace", in_path=config_path + ) def get_ocio_config_views(config_path): @@ -308,15 +563,20 @@ def get_ocio_config_views(config_path): if not compatibility_check(): # python environment is not compatible with PyOpenColorIO # needs to be run in subprocess - return get_views_data_subprocess(config_path) + return _get_wrapped_with_subprocess( + "config", "get_views", in_path=config_path + ) + # TODO: refactor this so it is not imported but part of this file from openpype.scripts.ocio_wrapper import _get_views_data return _get_views_data(config_path) +# TODO: remove this in future - backward compatibility +@deprecated("_get_wrapped_with_subprocess") def get_views_data_subprocess(config_path): - """Get viewers data via subprocess + """[Deprecated] Get viewers data via subprocess Wrapper for Python 2 hosts. @@ -326,7 +586,9 @@ def get_views_data_subprocess(config_path): Returns: dict: `display/viewer` and viewer data """ - return get_data_subprocess(config_path, "get_views") + return _get_wrapped_with_subprocess( + "config", "get_views", in_path=config_path + ) def get_imageio_config( diff --git a/openpype/plugins/publish/collect_resources_path.py b/openpype/plugins/publish/collect_resources_path.py index f96dd0ae18..57a612c5ae 100644 --- a/openpype/plugins/publish/collect_resources_path.py +++ b/openpype/plugins/publish/collect_resources_path.py @@ -62,7 +62,8 @@ class CollectResourcesPath(pyblish.api.InstancePlugin): "effect", "staticMesh", "skeletalMesh", - "xgen" + "xgen", + "yeticacheUE" ] def process(self, instance): diff --git a/openpype/plugins/publish/integrate.py b/openpype/plugins/publish/integrate.py index 7e48155b9e..ce24dad1b5 100644 --- a/openpype/plugins/publish/integrate.py +++ b/openpype/plugins/publish/integrate.py @@ -139,7 +139,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "simpleUnrealTexture", "online", "uasset", - "blendScene" + "blendScene", + "yeticacheUE" ] default_template_name = "publish" diff --git a/openpype/scripts/ocio_wrapper.py b/openpype/scripts/ocio_wrapper.py index 40553d30f2..c362670126 100644 --- a/openpype/scripts/ocio_wrapper.py +++ b/openpype/scripts/ocio_wrapper.py @@ -27,7 +27,7 @@ import PyOpenColorIO as ocio @click.group() def main(): - pass + pass # noqa: WPS100 @main.group() @@ -37,7 +37,17 @@ def config(): Example of use: > pyton.exe ./ocio_wrapper.py config *args """ - pass + pass # noqa: WPS100 + + +@main.group() +def colorspace(): + """Colorspace related commands group + + Example of use: + > pyton.exe ./ocio_wrapper.py config *args + """ + pass # noqa: WPS100 @config.command( @@ -70,8 +80,8 @@ def get_colorspace(in_path, out_path): out_data = _get_colorspace_data(in_path) - with open(json_path, "w") as f: - json.dump(out_data, f) + with open(json_path, "w") as f_: + json.dump(out_data, f_) print(f"Colorspace data are saved to '{json_path}'") @@ -97,8 +107,8 @@ def _get_colorspace_data(config_path): config = ocio.Config().CreateFromFile(str(config_path)) return { - c.getName(): c.getFamily() - for c in config.getColorSpaces() + c_.getName(): c_.getFamily() + for c_ in config.getColorSpaces() } @@ -132,8 +142,8 @@ def get_views(in_path, out_path): out_data = _get_views_data(in_path) - with open(json_path, "w") as f: - json.dump(out_data, f) + with open(json_path, "w") as f_: + json.dump(out_data, f_) print(f"Viewer data are saved to '{json_path}'") @@ -157,7 +167,7 @@ def _get_views_data(config_path): config = ocio.Config().CreateFromFile(str(config_path)) - data = {} + data_ = {} for display in config.getDisplays(): for view in config.getViews(display): colorspace = config.getDisplayViewColorSpaceName(display, view) @@ -165,13 +175,148 @@ def _get_views_data(config_path): if colorspace == "": colorspace = display - data[f"{display}/{view}"] = { + data_[f"{display}/{view}"] = { "display": display, "view": view, "colorspace": colorspace } - return data + return data_ + + +@config.command( + name="get_version", + help=( + "return major and minor version from config file " + "--config_path input arg is required" + "--out_path input arg is required" + ) +) +@click.option("--config_path", required=True, + help="path where to read ocio config file", + type=click.Path(exists=True)) +@click.option("--out_path", required=True, + help="path where to write output json file", + type=click.Path()) +def get_version(config_path, out_path): + """Get version of config. + + Python 2 wrapped console command + + Args: + config_path (str): ocio config file path string + out_path (str): temp json file path string + + Example of use: + > pyton.exe ./ocio_wrapper.py config get_version \ + --config_path= --out_path= + """ + json_path = Path(out_path) + + out_data = _get_version_data(config_path) + + with open(json_path, "w") as f_: + json.dump(out_data, f_) + + print(f"Config version data are saved to '{json_path}'") + + +def _get_version_data(config_path): + """Return major and minor version info. + + Args: + config_path (str): path string leading to config.ocio + + Raises: + IOError: Input config does not exist. + + Returns: + dict: minor and major keys with values + """ + config_path = Path(config_path) + + if not config_path.is_file(): + raise IOError("Input path should be `config.ocio` file") + + config = ocio.Config().CreateFromFile(str(config_path)) + + return { + "major": config.getMajorVersion(), + "minor": config.getMinorVersion() + } + + +@colorspace.command( + name="get_config_file_rules_colorspace_from_filepath", + help=( + "return colorspace from filepath " + "--config_path - ocio config file path (input arg is required) " + "--filepath - any file path (input arg is required) " + "--out_path - temp json file path (input arg is required)" + ) +) +@click.option("--config_path", required=True, + help="path where to read ocio config file", + type=click.Path(exists=True)) +@click.option("--filepath", required=True, + help="path to file to get colorspace from", + type=click.Path()) +@click.option("--out_path", required=True, + help="path where to write output json file", + type=click.Path()) +def get_config_file_rules_colorspace_from_filepath( + config_path, filepath, out_path +): + """Get colorspace from file path wrapper. + + Python 2 wrapped console command + + Args: + config_path (str): config file path string + filepath (str): path string leading to file + out_path (str): temp json file path string + + Example of use: + > pyton.exe ./ocio_wrapper.py \ + colorspace get_config_file_rules_colorspace_from_filepath \ + --config_path= --filepath= --out_path= + """ + json_path = Path(out_path) + + colorspace = _get_config_file_rules_colorspace_from_filepath( + config_path, filepath) + + with open(json_path, "w") as f_: + json.dump(colorspace, f_) + + print(f"Colorspace name is saved to '{json_path}'") + + +def _get_config_file_rules_colorspace_from_filepath(config_path, filepath): + """Return found colorspace data found in v2 file rules. + + Args: + config_path (str): path string leading to config.ocio + filepath (str): path string leading to v2 file rules + + Raises: + IOError: Input config does not exist. + + Returns: + dict: aggregated available colorspaces + """ + config_path = Path(config_path) + + if not config_path.is_file(): + raise IOError( + f"Input path `{config_path}` should be `config.ocio` file") + + config = ocio.Config().CreateFromFile(str(config_path)) + + # TODO: use `parseColorSpaceFromString` instead if ocio v1 + colorspace = config.getColorSpaceFromFilepath(str(filepath)) + + return colorspace def _get_display_view_colorspace_name(config_path, display, view): diff --git a/openpype/tools/ayon_workfiles/models/workfiles.py b/openpype/tools/ayon_workfiles/models/workfiles.py index 316d8b2a16..4d989ed22c 100644 --- a/openpype/tools/ayon_workfiles/models/workfiles.py +++ b/openpype/tools/ayon_workfiles/models/workfiles.py @@ -48,7 +48,7 @@ def get_task_template_data(project_entity, task): return {} short_name = None task_type_name = task["taskType"] - for task_type_info in project_entity["config"]["taskTypes"]: + for task_type_info in project_entity["taskTypes"]: if task_type_info["name"] == task_type_name: short_name = task_type_info["shortName"] break diff --git a/tests/integration/hosts/maya/test_deadline_publish_in_maya.py b/tests/integration/hosts/maya/test_deadline_publish_in_maya.py index c5bf526f52..9332177944 100644 --- a/tests/integration/hosts/maya/test_deadline_publish_in_maya.py +++ b/tests/integration/hosts/maya/test_deadline_publish_in_maya.py @@ -32,7 +32,7 @@ class TestDeadlinePublishInMaya(MayaDeadlinePublishTestClass): # keep empty to locate latest installed variant or explicit APP_VARIANT = "" - TIMEOUT = 120 # publish timeout + TIMEOUT = 180 # publish timeout def test_db_asserts(self, dbcon, publish_finished): """Host and input data dependent expected results in DB.""" diff --git a/tests/unit/openpype/hosts/photoshop/test_lib.py b/tests/unit/openpype/hosts/photoshop/test_lib.py new file mode 100644 index 0000000000..ad4feb42ae --- /dev/null +++ b/tests/unit/openpype/hosts/photoshop/test_lib.py @@ -0,0 +1,25 @@ +import pytest + +from openpype.hosts.photoshop.lib import clean_subset_name + +""" +Tests cleanup of unused layer placeholder ({layer}) from subset name. +Layer differentiation might be desired in subset name, but in some cases it +might be used (in `auto_image` - only single image without layer diff., +single image instance created without toggled use of subset name etc.) +""" + + +def test_no_layer_placeholder(): + clean_subset = clean_subset_name("imageMain") + assert "imageMain" == clean_subset + + +@pytest.mark.parametrize("subset_name", + ["imageMain{Layer}", + "imageMain_{layer}", # trailing _ + "image{Layer}Main", + "image{LAYER}Main"]) +def test_not_used_layer_placeholder(subset_name): + clean_subset = clean_subset_name(subset_name) + assert "imageMain" == clean_subset diff --git a/tests/unit/openpype/pipeline/test_colorspace.py b/tests/unit/openpype/pipeline/test_colorspace.py index ac35a28303..85faa8ff5d 100644 --- a/tests/unit/openpype/pipeline/test_colorspace.py +++ b/tests/unit/openpype/pipeline/test_colorspace.py @@ -26,8 +26,9 @@ class TestPipelineColorspace(TestPipeline): Example: cd to OpenPype repo root dir - poetry run python ./start.py runtests ../tests/unit/openpype/pipeline - """ + poetry run python ./start.py runtests /tests/unit/openpype/pipeline/test_colorspace.py + """ # noqa: E501 + TEST_FILES = [ ( "1csqimz8bbNcNgxtEXklLz6GRv91D3KgA", @@ -131,14 +132,14 @@ class TestPipelineColorspace(TestPipeline): path_1 = "renderCompMain_ACES2065-1.####.exr" expected_1 = "ACES2065-1" ret_1 = colorspace.parse_colorspace_from_filepath( - path_1, "nuke", "test_project", project_settings=project_settings + path_1, config_path=config_path_asset ) assert ret_1 == expected_1, f"Not matching colorspace {expected_1}" path_2 = "renderCompMain_BMDFilm_WideGamut_Gen5.mov" expected_2 = "BMDFilm WideGamut Gen5" ret_2 = colorspace.parse_colorspace_from_filepath( - path_2, "nuke", "test_project", project_settings=project_settings + path_2, config_path=config_path_asset ) assert ret_2 == expected_2, f"Not matching colorspace {expected_2}" @@ -184,5 +185,70 @@ class TestPipelineColorspace(TestPipeline): assert expected_hiero == hiero_file_rules, ( f"Not matching file rules {expected_hiero}") + def test_get_imageio_colorspace_from_filepath_p3(self, project_settings): + """Test Colorspace from filepath with python 3 compatibility mode + + Also test ocio v2 file rules + """ + nuke_filepath = "renderCompMain_baking_h264.mp4" + hiero_filepath = "prerenderCompMain.mp4" + + expected_nuke = "Camera Rec.709" + expected_hiero = "Gamma 2.2 Rec.709 - Texture" + + nuke_colorspace = colorspace.get_colorspace_name_from_filepath( + nuke_filepath, + "nuke", + "test_project", + project_settings=project_settings + ) + assert expected_nuke == nuke_colorspace, ( + f"Not matching colorspace {expected_nuke}") + + hiero_colorspace = colorspace.get_colorspace_name_from_filepath( + hiero_filepath, + "hiero", + "test_project", + project_settings=project_settings + ) + assert expected_hiero == hiero_colorspace, ( + f"Not matching colorspace {expected_hiero}") + + def test_get_imageio_colorspace_from_filepath_python2mode( + self, project_settings): + """Test Colorspace from filepath with python 2 compatibility mode + + Also test ocio v2 file rules + """ + nuke_filepath = "renderCompMain_baking_h264.mp4" + hiero_filepath = "prerenderCompMain.mp4" + + expected_nuke = "Camera Rec.709" + expected_hiero = "Gamma 2.2 Rec.709 - Texture" + + # switch to python 2 compatibility mode + colorspace.CachedData.has_compatible_ocio_package = False + + nuke_colorspace = colorspace.get_colorspace_name_from_filepath( + nuke_filepath, + "nuke", + "test_project", + project_settings=project_settings + ) + assert expected_nuke == nuke_colorspace, ( + f"Not matching colorspace {expected_nuke}") + + hiero_colorspace = colorspace.get_colorspace_name_from_filepath( + hiero_filepath, + "hiero", + "test_project", + project_settings=project_settings + ) + assert expected_hiero == hiero_colorspace, ( + f"Not matching colorspace {expected_hiero}") + + # return to python 3 compatibility mode + colorspace.CachedData.python3compatible = None + test_case = TestPipelineColorspace()