Merge pull request #4291 from ynput/feature/OP-4643_color-v3-global-oiio-transcoder-exctractor-plugin

Global: color v3 global oiio transcoder plugin
This commit is contained in:
Petr Kalis 2023-02-23 16:44:53 +01:00 committed by GitHub
commit e7f2a3c269
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 606 additions and 3 deletions

View file

@ -1045,3 +1045,90 @@ def convert_ffprobe_fps_to_float(value):
if divisor == 0.0:
return 0.0
return dividend / divisor
def convert_colorspace(
input_path,
output_path,
config_path,
source_colorspace,
target_colorspace=None,
view=None,
display=None,
additional_command_args=None,
logger=None
):
"""Convert source file from one color space to another.
Args:
input_path (str): Path that should be converted. It is expected that
contains single file or image sequence of same type
(sequence in format 'file.FRAMESTART-FRAMEEND#.ext', see oiio docs,
eg `big.1-3#.tif`)
output_path (str): Path to output filename.
(must follow format of 'input_path', eg. single file or
sequence in 'file.FRAMESTART-FRAMEEND#.ext', `output.1-3#.tif`)
config_path (str): path to OCIO config file
source_colorspace (str): ocio valid color space of source files
target_colorspace (str): ocio valid target color space
if filled, 'view' and 'display' must be empty
view (str): name for viewer space (ocio valid)
both 'view' and 'display' must be filled (if 'target_colorspace')
display (str): name for display-referred reference space (ocio valid)
additional_command_args (list): arguments for oiiotool (like binary
depth for .dpx)
logger (logging.Logger): Logger used for logging.
Raises:
ValueError: if misconfigured
"""
if logger is None:
logger = logging.getLogger(__name__)
oiio_cmd = [
get_oiio_tools_path(),
input_path,
# Don't add any additional attributes
"--nosoftwareattrib",
"--colorconfig", config_path
]
if all([target_colorspace, view, display]):
raise ValueError("Colorspace and both screen and display"
" cannot be set together."
"Choose colorspace or screen and display")
if not target_colorspace and not all([view, display]):
raise ValueError("Both screen and display must be set.")
if additional_command_args:
oiio_cmd.extend(additional_command_args)
if target_colorspace:
oiio_cmd.extend(["--colorconvert",
source_colorspace,
target_colorspace])
if view and display:
oiio_cmd.extend(["--iscolorspace", source_colorspace])
oiio_cmd.extend(["--ociodisplay", display, view])
oiio_cmd.extend(["-o", output_path])
logger.debug("Conversion command: {}".format(" ".join(oiio_cmd)))
run_subprocess(oiio_cmd, logger=logger)
def split_cmd_args(in_args):
"""Makes sure all entered arguments are separated in individual items.
Split each argument string with " -" to identify if string contains
one or more arguments.
Args:
in_args (list): of arguments ['-n', '-d uint10']
Returns
(list): ['-n', '-d', 'unint10']
"""
splitted_args = []
for arg in in_args:
if not arg.strip():
continue
splitted_args.extend(arg.split(" "))
return splitted_args

View file

@ -19,8 +19,6 @@ oauth_config:
- chat:write.public
- files:write
- channels:read
- users:read
- usergroups:read
settings:
org_deploy_enabled: false
socket_mode_enabled: false

View file

@ -0,0 +1,365 @@
import os
import copy
import clique
import pyblish.api
from openpype.pipeline import publish
from openpype.lib import (
is_oiio_supported,
)
from openpype.lib.transcoding import (
convert_colorspace,
get_transcode_temp_directory,
)
from openpype.lib.profiles_filtering import filter_profiles
class ExtractOIIOTranscode(publish.Extractor):
"""
Extractor to convert colors from one colorspace to different.
Expects "colorspaceData" on representation. This dictionary is collected
previously and denotes that representation files should be converted.
This dict contains source colorspace information, collected by hosts.
Target colorspace is selected by profiles in the Settings, based on:
- families
- host
- task types
- task names
- subset names
Can produce one or more representations (with different extensions) based
on output definition in format:
"output_name: {
"extension": "png",
"colorspace": "ACES - ACEScg",
"display": "",
"view": "",
"tags": [],
"custom_tags": []
}
If 'extension' is empty original representation extension is used.
'output_name' will be used as name of new representation. In case of value
'passthrough' name of original representation will be used.
'colorspace' denotes target colorspace to be transcoded into. Could be
empty if transcoding should be only into display and viewer colorspace.
(In that case both 'display' and 'view' must be filled.)
"""
label = "Transcode color spaces"
order = pyblish.api.ExtractorOrder + 0.019
optional = True
# Supported extensions
supported_exts = ["exr", "jpg", "jpeg", "png", "dpx"]
# Configurable by Settings
profiles = None
options = None
def process(self, instance):
if not self.profiles:
self.log.debug("No profiles present for color transcode")
return
if "representations" not in instance.data:
self.log.debug("No representations, skipping.")
return
if not is_oiio_supported():
self.log.warning("OIIO not supported, no transcoding possible.")
return
profile = self._get_profile(instance)
if not profile:
return
new_representations = []
repres = instance.data["representations"]
for idx, repre in enumerate(list(repres)):
self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"]))
if not self._repre_is_valid(repre):
continue
added_representations = False
added_review = False
colorspace_data = repre["colorspaceData"]
source_colorspace = colorspace_data["colorspace"]
config_path = colorspace_data.get("config", {}).get("path")
if not config_path or not os.path.exists(config_path):
self.log.warning("Config file doesn't exist, skipping")
continue
for output_name, output_def in profile.get("outputs", {}).items():
new_repre = copy.deepcopy(repre)
original_staging_dir = new_repre["stagingDir"]
new_staging_dir = get_transcode_temp_directory()
new_repre["stagingDir"] = new_staging_dir
if isinstance(new_repre["files"], list):
files_to_convert = copy.deepcopy(new_repre["files"])
else:
files_to_convert = [new_repre["files"]]
output_extension = output_def["extension"]
output_extension = output_extension.replace('.', '')
self._rename_in_representation(new_repre,
files_to_convert,
output_name,
output_extension)
transcoding_type = output_def["transcoding_type"]
target_colorspace = view = display = None
if transcoding_type == "colorspace":
target_colorspace = (output_def["colorspace"] or
colorspace_data.get("colorspace"))
else:
view = output_def["view"] or colorspace_data.get("view")
display = (output_def["display"] or
colorspace_data.get("display"))
# both could be already collected by DCC,
# but could be overwritten
if view:
new_repre["colorspaceData"]["view"] = view
if display:
new_repre["colorspaceData"]["display"] = display
additional_command_args = (output_def["oiiotool_args"]
["additional_command_args"])
files_to_convert = self._translate_to_sequence(
files_to_convert)
for file_name in files_to_convert:
input_path = os.path.join(original_staging_dir,
file_name)
output_path = self._get_output_file_path(input_path,
new_staging_dir,
output_extension)
convert_colorspace(
input_path,
output_path,
config_path,
source_colorspace,
target_colorspace,
view,
display,
additional_command_args,
self.log
)
# cleanup temporary transcoded files
for file_name in new_repre["files"]:
transcoded_file_path = os.path.join(new_staging_dir,
file_name)
instance.context.data["cleanupFullPaths"].append(
transcoded_file_path)
custom_tags = output_def.get("custom_tags")
if custom_tags:
if new_repre.get("custom_tags") is None:
new_repre["custom_tags"] = []
new_repre["custom_tags"].extend(custom_tags)
# Add additional tags from output definition to representation
if new_repre.get("tags") is None:
new_repre["tags"] = []
for tag in output_def["tags"]:
if tag not in new_repre["tags"]:
new_repre["tags"].append(tag)
if tag == "review":
added_review = True
new_representations.append(new_repre)
added_representations = True
if added_representations:
self._mark_original_repre_for_deletion(repre, profile,
added_review)
for repre in tuple(instance.data["representations"]):
tags = repre.get("tags") or []
if "delete" in tags and "thumbnail" not in tags:
instance.data["representations"].remove(repre)
instance.data["representations"].extend(new_representations)
def _rename_in_representation(self, new_repre, files_to_convert,
output_name, output_extension):
"""Replace old extension with new one everywhere in representation.
Args:
new_repre (dict)
files_to_convert (list): of filenames from repre["files"],
standardized to always list
output_name (str): key of output definition from Settings,
if "<passthrough>" token used, keep original repre name
output_extension (str): extension from output definition
"""
if output_name != "passthrough":
new_repre["name"] = output_name
if not output_extension:
return
new_repre["ext"] = output_extension
renamed_files = []
for file_name in files_to_convert:
file_name, _ = os.path.splitext(file_name)
file_name = '{}.{}'.format(file_name,
output_extension)
renamed_files.append(file_name)
new_repre["files"] = renamed_files
def _rename_in_representation(self, new_repre, files_to_convert,
output_name, output_extension):
"""Replace old extension with new one everywhere in representation.
Args:
new_repre (dict)
files_to_convert (list): of filenames from repre["files"],
standardized to always list
output_name (str): key of output definition from Settings,
if "<passthrough>" token used, keep original repre name
output_extension (str): extension from output definition
"""
if output_name != "passthrough":
new_repre["name"] = output_name
if not output_extension:
return
new_repre["ext"] = output_extension
renamed_files = []
for file_name in files_to_convert:
file_name, _ = os.path.splitext(file_name)
file_name = '{}.{}'.format(file_name,
output_extension)
renamed_files.append(file_name)
new_repre["files"] = renamed_files
def _translate_to_sequence(self, files_to_convert):
"""Returns original list or list with filename formatted in single
sequence format.
Uses clique to find frame sequence, in this case it merges all frames
into sequence format (FRAMESTART-FRAMEEND#) and returns it.
If sequence not found, it returns original list
Args:
files_to_convert (list): list of file names
Returns:
(list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr]
"""
pattern = [clique.PATTERNS["frames"]]
collections, remainder = clique.assemble(
files_to_convert, patterns=pattern,
assume_padded_when_ambiguous=True)
if collections:
if len(collections) > 1:
raise ValueError(
"Too many collections {}".format(collections))
collection = collections[0]
frames = list(collection.indexes)
frame_str = "{}-{}#".format(frames[0], frames[-1])
file_name = "{}{}{}".format(collection.head, frame_str,
collection.tail)
files_to_convert = [file_name]
return files_to_convert
def _get_output_file_path(self, input_path, output_dir,
output_extension):
"""Create output file name path."""
file_name = os.path.basename(input_path)
file_name, input_extension = os.path.splitext(file_name)
if not output_extension:
output_extension = input_extension.replace(".", "")
new_file_name = '{}.{}'.format(file_name,
output_extension)
return os.path.join(output_dir, new_file_name)
def _get_profile(self, instance):
"""Returns profile if and how repre should be color transcoded."""
host_name = instance.context.data["hostName"]
family = instance.data["family"]
task_data = instance.data["anatomyData"].get("task", {})
task_name = task_data.get("name")
task_type = task_data.get("type")
subset = instance.data["subset"]
filtering_criteria = {
"hosts": host_name,
"families": family,
"task_names": task_name,
"task_types": task_type,
"subsets": subset
}
profile = filter_profiles(self.profiles, filtering_criteria,
logger=self.log)
if not profile:
self.log.info((
"Skipped instance. None of profiles in presets are for"
" Host: \"{}\" | Families: \"{}\" | Task \"{}\""
" | Task type \"{}\" | Subset \"{}\" "
).format(host_name, family, task_name, task_type, subset))
self.log.debug("profile: {}".format(profile))
return profile
def _repre_is_valid(self, repre):
"""Validation if representation should be processed.
Args:
repre (dict): Representation which should be checked.
Returns:
bool: False if can't be processed else True.
"""
if repre.get("ext") not in self.supported_exts:
self.log.debug((
"Representation '{}' of unsupported extension. Skipped."
).format(repre["name"]))
return False
if not repre.get("files"):
self.log.debug((
"Representation '{}' have empty files. Skipped."
).format(repre["name"]))
return False
if not repre.get("colorspaceData"):
self.log.debug("Representation '{}' has no colorspace data. "
"Skipped.")
return False
return True
def _mark_original_repre_for_deletion(self, repre, profile, added_review):
"""If new transcoded representation created, delete old."""
if not repre.get("tags"):
repre["tags"] = []
delete_original = profile["delete_original"]
if delete_original:
if "delete" not in repre["tags"]:
repre["tags"].append("delete")
if added_review and "review" in repre["tags"]:
repre["tags"].remove("review")

View file

@ -169,7 +169,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
"Skipped representation. All output definitions from"
" selected profile does not match to representation's"
" custom tags. \"{}\""
).format(str(tags)))
).format(str(custom_tags)))
continue
outputs_per_representations.append((repre, outputs))

View file

@ -68,6 +68,10 @@
"output": []
}
},
"ExtractOIIOTranscode": {
"enabled": true,
"profiles": []
},
"ExtractReview": {
"enabled": true,
"profiles": [

View file

@ -197,6 +197,136 @@
}
]
},
{
"type": "dict",
"collapsible": true,
"key": "ExtractOIIOTranscode",
"label": "Extract OIIO Transcode",
"checkbox_key": "enabled",
"is_group": true,
"children": [
{
"type": "label",
"label": "Configure Output Definition(s) for new representation(s). \nEmpty 'Extension' denotes keeping source extension. \nName(key) of output definition will be used as new representation name \nunless 'passthrough' value is used to keep existing name. \nFill either 'Colorspace' (for target colorspace) or \nboth 'Display' and 'View' (for display and viewer colorspaces)."
},
{
"type": "boolean",
"key": "enabled",
"label": "Enabled"
},
{
"type": "list",
"key": "profiles",
"label": "Profiles",
"object_type": {
"type": "dict",
"children": [
{
"key": "families",
"label": "Families",
"type": "list",
"object_type": "text"
},
{
"key": "hosts",
"label": "Host names",
"type": "hosts-enum",
"multiselection": true
},
{
"key": "task_types",
"label": "Task types",
"type": "task-types-enum"
},
{
"key": "task_names",
"label": "Task names",
"type": "list",
"object_type": "text"
},
{
"key": "subsets",
"label": "Subset names",
"type": "list",
"object_type": "text"
},
{
"type": "boolean",
"key": "delete_original",
"label": "Delete Original Representation"
},
{
"type": "splitter"
},
{
"key": "outputs",
"label": "Output Definitions",
"type": "dict-modifiable",
"highlight_content": true,
"object_type": {
"type": "dict",
"children": [
{
"key": "extension",
"label": "Extension",
"type": "text"
},
{
"type": "enum",
"key": "transcoding_type",
"label": "Transcoding type",
"enum_items": [
{ "colorspace": "Use Colorspace" },
{ "display": "Use Display&View" }
]
},
{
"key": "colorspace",
"label": "Colorspace",
"type": "text"
},
{
"key": "display",
"label": "Display",
"type": "text"
},
{
"key": "view",
"label": "View",
"type": "text"
},
{
"key": "oiiotool_args",
"label": "OIIOtool arguments",
"type": "dict",
"highlight_content": true,
"children": [
{
"key": "additional_command_args",
"label": "Arguments",
"type": "list",
"object_type": "text"
}
]
},
{
"type": "schema",
"name": "schema_representation_tags"
},
{
"key": "custom_tags",
"label": "Custom Tags",
"type": "list",
"object_type": "text"
}
]
}
}
]
}
}
]
},
{
"type": "dict",
"collapsible": true,

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

View file

@ -45,6 +45,25 @@ The _input pattern_ matching uses REGEX expression syntax (try [regexr.com](http
The **colorspace name** value is a raw string input and no validation is run after saving project settings. We recommend to open the specified `config.ocio` file and copy pasting the exact colorspace names.
:::
### Extract OIIO Transcode
OIIOTools transcoder plugin with configurable output presets. Any incoming representation with `colorspaceData` is convertable to single or multiple representations with different target colorspaces or display and viewer names found in linked **config.ocio** file.
`oiiotool` is used for transcoding, eg. `oiiotool` must be present in `vendor/bin/oiio` or environment variable `OPENPYPE_OIIO_PATHS` must be provided for custom oiio installation.
Notable parameters:
- **`Delete Original Representation`** - keep or remove original representation. If old representation is kept, but there is new transcoded representation with 'Create review' tag, original representation loses its 'review' tag if present.
- **`Extension`** - target extension. If left empty, original extension is used.
- **`Transcoding type`** - transcoding into colorspace or into display and viewer space could be used. Cannot use both at the same time.
- **`Colorspace`** - target colorspace, which must be available in used color config. (If `Transcoding type` is `Use Colorspace` value in configuration is used OR if empty value collected on instance from DCC).
- **`Display & View`** - display and viewer colorspace. (If `Transcoding type` is `Use Display&View` values in configuration is used OR if empty values collected on instance from DCC).
- **`Arguments`** - special additional command line arguments for `oiiotool`.
Example here describes use case for creation of new color coded review of png image sequence. Original representation's files are kept intact, review is created from transcoded files, but these files are removed in cleanup process.
![global_oiio_transcode](assets/global_oiio_transcode.png)
Another use case is to transcode in Maya only `beauty` render layers and use collected `Display` and `View` colorspaces from DCC.
![global_oiio_transcode_in_Maya](assets/global_oiio_transcode.png)n
## Profile filters