Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into unify-and-speed-up-copying

Browse source
This commit is contained in:
Mustafa Jafar 2025-05-19 11:43:46 +03:00 • committed by GitHub
commit 8d9c8f4e91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 437 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

2
client/ayon_core/pipeline/colorspace.py
View file

@ -834,7 +834,7 @@ def _get_global_config_data(
if not product_entities_by_name:
# in case no product was found we need to use fallback
fallback_type = fallback_data["type"]
fallback_type = fallback_data["fallback_type"]
return _get_config_path_from_profile_data(
fallback_data, fallback_type, template_data
)

15
client/ayon_core/plugins/publish/extract_otio_review.py
View file

@ -54,7 +54,7 @@ class ExtractOTIOReview(
# plugin default attributes
to_width = 1280
to_height = 720
output_ext = ".jpg"
output_ext = ".png"
def process(self, instance):
# Not all hosts can import these modules.
@ -510,6 +510,12 @@ class ExtractOTIOReview(
"-tune", "stillimage"
])
if video or sequence:
command.extend([
"-vf", f"scale={self.to_width}:{self.to_height}:flags=lanczos",
"-compression_level", "5",
])
# add output attributes
command.extend([
"-start_number", str(out_frame_start)
@ -520,9 +526,10 @@ class ExtractOTIOReview(
input_extension
and self.output_ext == input_extension
):
command.extend([
"-c", "copy"
])
command.extend(["-c", "copy"])
else:
# For lossy formats, force re-encode
command.extend(["-pix_fmt", "rgba"])
# add output path at the end
command.append(output_path)

314
client/ayon_core/plugins/publish/extract_review.py
View file

@ -5,11 +5,15 @@ import json
import shutil
import subprocess
from abc import ABC, abstractmethod
from typing import Dict, Any, Optional
import tempfile
import clique
import speedcopy
import pyblish.api
from ayon_api import get_last_version_by_product_name, get_representations
from ayon_core.lib import (
get_ffmpeg_tool_args,
filter_profiles,
@ -400,15 +404,73 @@ class ExtractReview(pyblish.api.InstancePlugin):
)
temp_data = self.prepare_temp_data(instance, repre, output_def)
files_to_clean = []
new_frame_files = {}
if temp_data["input_is_sequence"]:
self.log.debug("Checking sequence to fill gaps in sequence..")
files_to_clean = self.fill_sequence_gaps(
files=temp_data["origin_repre"]["files"],
staging_dir=new_repre["stagingDir"],
start_frame=temp_data["frame_start"],
end_frame=temp_data["frame_end"]
)
files = temp_data["origin_repre"]["files"]
collections = clique.assemble(
files,
)[0]
if len(collections) != 1:
raise KnownPublishError(
"Multiple collections {} found.".format(collections))
collection = collections[0]
fill_missing_frames = _output_def["fill_missing_frames"]
if fill_missing_frames == "closest_existing":
new_frame_files = self.fill_sequence_gaps_from_existing(
collection=collection,
staging_dir=new_repre["stagingDir"],
start_frame=temp_data["frame_start"],
end_frame=temp_data["frame_end"],
)
elif fill_missing_frames == "blank":
new_frame_files = self.fill_sequence_gaps_with_blanks(
collection=collection,
staging_dir=new_repre["stagingDir"],
start_frame=temp_data["frame_start"],
end_frame=temp_data["frame_end"],
resolution_width=temp_data["resolution_width"],
resolution_height=temp_data["resolution_height"],
extension=temp_data["input_ext"],
temp_data=temp_data
)
elif fill_missing_frames == "previous_version":
new_frame_files = self.fill_sequence_gaps_with_previous(
collection=collection,
staging_dir=new_repre["stagingDir"],
instance=instance,
current_repre_name=repre["name"],
start_frame=temp_data["frame_start"],
end_frame=temp_data["frame_end"],
)
# fallback to original workflow
if new_frame_files is None:
new_frame_files = (
self.fill_sequence_gaps_from_existing(
collection=collection,
staging_dir=new_repre["stagingDir"],
start_frame=temp_data["frame_start"],
end_frame=temp_data["frame_end"],
))
elif fill_missing_frames == "only_rendered":
temp_data["explicit_input_paths"] = [
os.path.join(
new_repre["stagingDir"], file
).replace("\\", "/")
for file in files
]
frame_start = min(collection.indexes)
frame_end = max(collection.indexes)
# modify range for burnins
instance.data["frameStart"] = frame_start
instance.data["frameEnd"] = frame_end
temp_data["frame_start"] = frame_start
temp_data["frame_end"] = frame_end
temp_data["filled_files"] = new_frame_files
# create or update outputName
output_name = new_repre.get("outputName", "")
@ -465,9 +527,12 @@ class ExtractReview(pyblish.api.InstancePlugin):
run_subprocess(subprcs_cmd, shell=True, logger=self.log)
# delete files added to fill gaps
if files_to_clean:
for f in files_to_clean:
os.unlink(f)
if new_frame_files:
for filepath in new_frame_files.values():
os.unlink(filepath)
for filepath in temp_data["paths_to_remove"]:
os.unlink(filepath)
new_repre.update({
"fps": temp_data["fps"],
@ -560,6 +625,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
input_is_sequence = self.input_is_sequence(repre)
input_allow_bg = False
first_sequence_frame = None
if input_is_sequence and repre["files"]:
# Calculate first frame that should be used
cols, _ = clique.assemble(repre["files"])
@ -578,6 +644,8 @@ class ExtractReview(pyblish.api.InstancePlugin):
ext = os.path.splitext(repre["files"][0])[1].replace(".", "")
if ext.lower() in self.alpha_exts:
input_allow_bg = True
else:
ext = os.path.splitext(repre["files"])[1].replace(".", "")
return {
"fps": float(instance.data["fps"]),
@ -598,7 +666,10 @@ class ExtractReview(pyblish.api.InstancePlugin):
"input_allow_bg": input_allow_bg,
"with_audio": with_audio,
"without_handles": without_handles,
"handles_are_set": handles_are_set
"handles_are_set": handles_are_set,
"input_ext": ext,
"explicit_input_paths": [], # absolute paths to rendered files
"paths_to_remove": []
}
def _ffmpeg_arguments(
@ -680,7 +751,8 @@ class ExtractReview(pyblish.api.InstancePlugin):
if layer_name:
ffmpeg_input_args.extend(["-layer", layer_name])
if temp_data["input_is_sequence"]:
explicit_input_paths = temp_data["explicit_input_paths"]
if temp_data["input_is_sequence"] and not explicit_input_paths:
# Set start frame of input sequence (just frame in filename)
# - definition of input filepath
# - add handle start if output should be without handles
@ -707,7 +779,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
"-to", "{:0.10f}".format(duration_seconds)
])
if temp_data["output_is_sequence"]:
if temp_data["output_is_sequence"] and not explicit_input_paths:
# Set start frame of output sequence (just frame in filename)
# - this is definition of an output
ffmpeg_output_args.extend([
@ -738,10 +810,34 @@ class ExtractReview(pyblish.api.InstancePlugin):
"-frames:v", str(output_frames_len)
])
# Add video/image input path
ffmpeg_input_args.extend([
"-i", path_to_subprocess_arg(temp_data["full_input_path"])
])
if not explicit_input_paths:
# Add video/image input path
ffmpeg_input_args.extend([
"-i", path_to_subprocess_arg(temp_data["full_input_path"])
])
else:
frame_duration = 1 / temp_data["fps"]
explicit_frames_meta = tempfile.NamedTemporaryFile(
mode="w", prefix="explicit_frames", suffix=".txt", delete=False
)
explicit_frames_meta.close()
explicit_frames_path = explicit_frames_meta.name
with open(explicit_frames_path, "w") as fp:
lines = [
f"file '{path}'{os.linesep}duration {frame_duration}"
for path in temp_data["explicit_input_paths"]
]
fp.write("\n".join(lines))
temp_data["paths_to_remove"].append(explicit_frames_path)
# let ffmpeg use only rendered files, might have gaps
ffmpeg_input_args.extend([
"-f", "concat",
"-safe", "0",
"-i", path_to_subprocess_arg(explicit_frames_path),
"-r", str(temp_data["fps"])
])
# Add audio arguments if there are any. Skipped when output are images.
if not temp_data["output_ext_is_image"] and temp_data["with_audio"]:
@ -881,8 +977,159 @@ class ExtractReview(pyblish.api.InstancePlugin):
return all_args
def fill_sequence_gaps(self, files, staging_dir, start_frame, end_frame):
# type: (list, str, int, int) -> list
def fill_sequence_gaps_with_previous(
self,
collection: str,
staging_dir: str,
instance: pyblish.plugin.Instance,
current_repre_name: str,
start_frame: int,
end_frame: int
) -> Optional[Dict[int, str]]:
"""Tries to replace missing frames from ones from last version"""
repre_file_paths = self._get_last_version_files(
instance, current_repre_name)
if repre_file_paths is None:
# issues in getting last version files, falling back
return None
prev_collection = clique.assemble(
repre_file_paths,
patterns=[clique.PATTERNS["frames"]],
minimum_items=1
)[0][0]
prev_col_format = prev_collection.format("{head}{padding}{tail}")
added_files = {}
anatomy = instance.context.data["anatomy"]
col_format = collection.format("{head}{padding}{tail}")
for frame in range(start_frame, end_frame + 1):
if frame in collection.indexes:
continue
hole_fpath = os.path.join(staging_dir, col_format % frame)
previous_version_path = prev_col_format % frame
previous_version_path = anatomy.fill_root(previous_version_path)
if not os.path.exists(previous_version_path):
self.log.warning(
"Missing frame should be replaced from "
f"'{previous_version_path}' but that doesn't exist. "
"Falling back to filling from currently last rendered."
)
return None
self.log.warning(
f"Replacing missing '{hole_fpath}' with "
f"'{previous_version_path}'"
)
speedcopy.copyfile(previous_version_path, hole_fpath)
added_files[frame] = hole_fpath
return added_files
def _get_last_version_files(
self,
instance: pyblish.plugin.Instance,
current_repre_name: str,
):
product_name = instance.data["productName"]
project_name = instance.data["projectEntity"]["name"]
folder_entity = instance.data["folderEntity"]
version_entity = get_last_version_by_product_name(
project_name,
product_name,
folder_entity["id"],
fields={"id"}
)
if not version_entity:
return None
matching_repres = get_representations(
project_name,
version_ids=[version_entity["id"]],
representation_names=[current_repre_name],
fields={"files"}
)
if not matching_repres:
return None
matching_repre = list(matching_repres)[0]
repre_file_paths = [
file_info["path"]
for file_info in matching_repre["files"]
]
return repre_file_paths
def fill_sequence_gaps_with_blanks(
self,
collection: str,
staging_dir: str,
start_frame: int,
end_frame: int,
resolution_width: int,
resolution_height: int,
extension: str,
temp_data: Dict[str, Any]
) -> Optional[Dict[int, str]]:
"""Fills missing files by blank frame."""
blank_frame_path = None
added_files = {}
col_format = collection.format("{head}{padding}{tail}")
for frame in range(start_frame, end_frame + 1):
if frame in collection.indexes:
continue
hole_fpath = os.path.join(staging_dir, col_format % frame)
if blank_frame_path is None:
blank_frame_path = self._create_blank_frame(
staging_dir, extension, resolution_width, resolution_height
)
temp_data["paths_to_remove"].append(blank_frame_path)
speedcopy.copyfile(blank_frame_path, hole_fpath)
added_files[frame] = hole_fpath
return added_files
def _create_blank_frame(
self,
staging_dir,
extension,
resolution_width,
resolution_height
):
blank_frame_path = os.path.join(staging_dir, f"blank.{extension}")
command = get_ffmpeg_tool_args(
"ffmpeg",
"-f", "lavfi",
"-i", "color=c=black:s={}x{}:d=1".format(
resolution_width, resolution_height
),
"-tune", "stillimage",
"-frames:v", "1",
blank_frame_path
)
self.log.debug("Executing: {}".format(" ".join(command)))
output = run_subprocess(
command, logger=self.log
)
self.log.debug("Output: {}".format(output))
return blank_frame_path
def fill_sequence_gaps_from_existing(
self,
collection,
staging_dir: str,
start_frame: int,
end_frame: int
) -> Dict[int, str]:
"""Fill missing files in sequence by duplicating existing ones.
This will take nearest frame file and copy it with so as to fill
@ -890,40 +1137,33 @@ class ExtractReview(pyblish.api.InstancePlugin):
hole ahead.
Args:
files (list): List of representation files.
collection (clique.collection)
staging_dir (str): Path to staging directory.
start_frame (int): Sequence start (no matter what files are there)
end_frame (int): Sequence end (no matter what files are there)
Returns:
list of added files. Those should be cleaned after work
dict[int, str] of added files. Those should be cleaned after work
is done.
Raises:
KnownPublishError: if more than one collection is obtained.
"""
collections = clique.assemble(files)[0]
if len(collections) != 1:
raise KnownPublishError(
"Multiple collections {} found.".format(collections))
col = collections[0]
# Prepare which hole is filled with what frame
# - the frame is filled only with already existing frames
prev_frame = next(iter(col.indexes))
prev_frame = next(iter(collection.indexes))
hole_frame_to_nearest = {}
for frame in range(int(start_frame), int(end_frame) + 1):
if frame in col.indexes:
if frame in collection.indexes:
prev_frame = frame
else:
# Use previous frame as source for hole
hole_frame_to_nearest[frame] = prev_frame
# Calculate paths
added_files = []
col_format = col.format("{head}{padding}{tail}")
added_files = {}
col_format = collection.format("{head}{padding}{tail}")
for hole_frame, src_frame in hole_frame_to_nearest.items():
hole_fpath = os.path.join(staging_dir, col_format % hole_frame)
src_fpath = os.path.join(staging_dir, col_format % src_frame)
@ -932,7 +1172,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
"Missing previously detected file: {}".format(src_fpath))
speedcopy.copyfile(src_fpath, hole_fpath)
added_files.append(hole_fpath)
added_files[hole_frame] = hole_fpath
return added_files
@ -978,6 +1218,14 @@ class ExtractReview(pyblish.api.InstancePlugin):
# Make sure to have full path to one input file
full_input_path_single_file = full_input_path
filled_files = temp_data["filled_files"]
if filled_files:
first_frame, first_file = next(iter(filled_files.items()))
if first_file < full_input_path_single_file:
self.log.warning(f"Using filled frame: '{first_file}'")
full_input_path_single_file = first_file
temp_data["first_sequence_frame"] = first_frame
filename_suffix = output_def["filename_suffix"]
output_ext = output_def.get("ext")

87
client/ayon_core/plugins/publish/extract_thumbnail.py
View file

@ -506,27 +506,36 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
# Set video input attributes
max_int = str(2147483647)
video_data = get_ffprobe_data(video_file_path, logger=self.log)
# Use duration of the individual streams since it is returned with
# higher decimal precision than 'format.duration'. We need this
# more precise value for calculating the correct amount of frames
# for higher FPS ranges or decimal ranges, e.g. 29.97 FPS
duration = max(
float(stream.get("duration", 0))
for stream in video_data["streams"]
if stream.get("codec_type") == "video"
)
cmd_args = [
"-y",
"-ss", str(duration * self.duration_split),
# Get duration or use a safe default (single frame)
duration = 0
for stream in video_data["streams"]:
if stream.get("codec_type") == "video":
stream_duration = float(stream.get("duration", 0))
if stream_duration > duration:
duration = stream_duration
# For very short videos, just use the first frame
# Calculate seek position safely
seek_position = 0.0
# Only use timestamp calculation for videos longer than 0.1 seconds
if duration > 0.1:
seek_position = duration * self.duration_split
# Build command args
cmd_args = []
if seek_position > 0.0:
cmd_args.extend(["-ss", str(seek_position)])
# Add generic ffmpeg commands
cmd_args.extend([
"-i", video_file_path,
"-analyzeduration", max_int,
"-probesize", max_int,
"-frames:v", "1"
]
# add output file path
cmd_args.append(output_thumb_file_path)
"-y",
"-frames:v", "1",
output_thumb_file_path
])
# create ffmpeg command
cmd = get_ffmpeg_tool_args(
@ -537,15 +546,53 @@ class ExtractThumbnail(pyblish.api.InstancePlugin):
# run subprocess
self.log.debug("Executing: {}".format(" ".join(cmd)))
run_subprocess(cmd, logger=self.log)
self.log.debug(
"Thumbnail created: {}".format(output_thumb_file_path))
return output_thumb_file_path
# Verify the output file was created
if (
os.path.exists(output_thumb_file_path)
and os.path.getsize(output_thumb_file_path) > 0
):
self.log.debug(
"Thumbnail created: {}".format(output_thumb_file_path))
return output_thumb_file_path
self.log.warning("Output file was not created or is empty")
# Try to create thumbnail without offset
# - skip if offset did not happen
if "-ss" not in cmd_args:
return None
self.log.debug("Trying fallback without offset")
# Remove -ss and its value
ss_index = cmd_args.index("-ss")
cmd_args.pop(ss_index) # Remove -ss
cmd_args.pop(ss_index) # Remove the timestamp value
# Create new command and try again
cmd = get_ffmpeg_tool_args("ffmpeg", *cmd_args)
self.log.debug("Fallback command: {}".format(" ".join(cmd)))
run_subprocess(cmd, logger=self.log)
if (
os.path.exists(output_thumb_file_path)
and os.path.getsize(output_thumb_file_path) > 0
):
self.log.debug("Fallback thumbnail created")
return output_thumb_file_path
return None
except RuntimeError as error:
self.log.warning(
"Failed intermediate thumb source using ffmpeg: {}".format(
error)
)
return None
finally:
# Remove output file if is empty
if (
os.path.exists(output_thumb_file_path)
and os.path.getsize(output_thumb_file_path) == 0
):
os.remove(output_thumb_file_path)
def _get_resolution_arg(
self,