Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve parallelization for ExtractReview and ExtractOIIOTranscode

Browse source 
- Support ExtractReview convert to FFMPEG in one `oiiotool` call for sequences
- Support sequences with holes in both plug-ins by using dedicated `--frames` argument to `oiiotool` for more complex frame patterns.
- Add `--parallel-frames` argument to `oiiotool` to allow parallelizing more of the OIIO tool process, improving throughput. Note: This requires OIIO 2.5.2.0 or higher. See f40f9800c8
This commit is contained in:
Roy Nieterau 2025-04-23 19:44:39 +02:00
parent 62b835faf8
commit 98e0ec1051
2 changed files with 85 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

57
client/ayon_core/lib/transcoding.py
View file

@ -10,6 +10,8 @@ from typing import Optional
import xml.etree.ElementTree
import clique
from .execute import run_subprocess
from .vendor_bin_utils import (
get_ffmpeg_tool_args,
@ -707,7 +709,29 @@ def convert_input_paths_for_ffmpeg(
# Collect channels to export
input_arg, channels_arg = get_oiio_input_and_channel_args(input_info)
for input_path in input_paths:
# Process input files
# If a sequence of files is detected we process it in one go
# with the dedicated --frames argument for faster processing
collections, remainder = clique.assemble(
input_paths, patterns=clique.PATTERNS["frame"])
process_queue = collections + remainder
for input_item in process_queue:
if isinstance(input_item, clique.Collection):
# Support sequences with holes by supplying dedicated `--frames`
# Create `frames` string like "1001-1002,1004,1010-1012
frames: str = input_item.format("{ranges}").replace(" ", "")
# Create `filename` string like "file.%04d.exr"
input_path = input_item.format("{head}{padding}{tail}")
elif isinstance(input_item, str):
# Single filepath
frames = None
input_path = input_item
else:
raise TypeError(
f"Input is not a string or Collection: {input_item}"
)
# Prepare subprocess arguments
oiio_cmd = get_oiio_tool_args(
"oiiotool",
@ -718,6 +742,14 @@ def convert_input_paths_for_ffmpeg(
if compression:
oiio_cmd.extend(["--compression", compression])
if frames:
oiio_cmd.extend([
"--frames", frames,
# TODO: Handle potential toggle for parallel frames
# to support older OIIO releases.
"--parallel-frames"
])
oiio_cmd.extend([
input_arg, input_path,
# Tell oiiotool which channels should be put to top stack
@ -1106,6 +1138,8 @@ def convert_colorspace(
view=None,
display=None,
additional_command_args=None,
frames=None,
parallel_frames=False,
logger=None,
):
"""Convert source file from one color space to another.
@ -1114,7 +1148,7 @@ def convert_colorspace(
input_path (str): Path that should be converted. It is expected that
contains single file or image sequence of same type
(sequence in format 'file.FRAMESTART-FRAMEEND#.ext', see oiio docs,
eg `big.1-3#.tif`)
eg `big.1-3#.tif` or `big.%04d.ext` with `frames` argument)
output_path (str): Path to output filename.
(must follow format of 'input_path', eg. single file or
sequence in 'file.FRAMESTART-FRAMEEND#.ext', `output.1-3#.tif`)
@ -1128,6 +1162,11 @@ def convert_colorspace(
both 'view' and 'display' must be filled (if 'target_colorspace')
additional_command_args (list): arguments for oiiotool (like binary
depth for .dpx)
frames (Optional[str]): Complex frame range to process. This requires
input path and output path to use frame token placeholder like
e.g. file.%04d.exr
parallel_frames (bool): If True, process frames in parallel inside
the `oiiotool` process. Only supported in OIIO 2.5.20.0+.
logger (logging.Logger): Logger used for logging.
Raises:
ValueError: if misconfigured
@ -1145,9 +1184,21 @@ def convert_colorspace(
"oiiotool",
# Don't add any additional attributes
"--nosoftwareattrib",
"--colorconfig", config_path
"--colorconfig", config_path,
)
if frames:
# If `frames` is specified, then process the input and output
# as if it's a sequence of frames (must contain `%04d` as frame
# token placeholder in filepaths)
oiio_cmd.extend([
"--frames", frames,
])
if parallel_frames:
oiio_cmd.extend([
"--parallel-frames"
])
oiio_cmd.extend([
input_arg, input_path,
# Tell oiiotool which channels should be put to top stack

48
client/ayon_core/plugins/publish/extract_color_transcode.py
View file

@ -159,9 +159,18 @@ class ExtractOIIOTranscode(publish.Extractor):
files_to_convert)
self.log.debug("Files to convert: {}".format(files_to_convert))
for file_name in files_to_convert:
# Handle special case for sequences where we specify
# the --frames argument to oiiotool
frames = None
parallel_frames = False
if isinstance(file_name, tuple):
file_name, frames = file_name
# TODO: Handle potential toggle for parallel frames
# to support older OIIO releases.
parallel_frames = True
self.log.debug("Transcoding file: `{}`".format(file_name))
input_path = os.path.join(original_staging_dir,
file_name)
input_path = os.path.join(original_staging_dir, file_name)
output_path = self._get_output_file_path(input_path,
new_staging_dir,
output_extension)
@ -175,7 +184,9 @@ class ExtractOIIOTranscode(publish.Extractor):
view,
display,
additional_command_args,
self.log
frames=frames,
parallel_frames=parallel_frames,
logger=self.log
)
# cleanup temporary transcoded files
@ -256,17 +267,22 @@ class ExtractOIIOTranscode(publish.Extractor):
new_repre["files"] = renamed_files
def _translate_to_sequence(self, files_to_convert):
"""Returns original list or list with filename formatted in single
sequence format.
"""Returns original individual filepaths or list of a single two-tuple
representating sequence filename with its frames.
Uses clique to find frame sequence, in this case it merges all frames
into sequence format (FRAMESTART-FRAMEEND#) and returns it.
If sequence not found, it returns original list
into sequence format (`%04d`) together with all its frames to support
both regular sequences and sequences with holes.
If sequence not detected in input filenames, it returns original list.
Args:
files_to_convert (list): list of file names
files_to_convert (list[str]): list of file names
Returns:
(list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr]
list[Union[str, tuple[str, str]]: List of
or filepaths ['fileA.exr', 'fileB.exr']
or sequence with frames [('file.%04d.exr', '1001-1002,1004')]
"""
pattern = [clique.PATTERNS["frames"]]
collections, _ = clique.assemble(
@ -279,15 +295,13 @@ class ExtractOIIOTranscode(publish.Extractor):
"Too many collections {}".format(collections))
collection = collections[0]
frames = list(collection.indexes)
if collection.holes():
return files_to_convert
frame_str = "{}-{}#".format(frames[0], frames[-1])
file_name = "{}{}{}".format(collection.head, frame_str,
collection.tail)
files_to_convert = [file_name]
# Support sequences with holes by supplying dedicated `--frames`
# Create `frames` string like "1001-1002,1004,1010-1012
frames: str = collection.format("{ranges}").replace(" ", "")
# Create `filename` string like "file.%04d.exr"
filename = collection.format("{head}{padding}{tail}")
return [(filename, frames)]
return files_to_convert