From 9fb629d562a25606f3d43e39a3043aa7e35afb12 Mon Sep 17 00:00:00 2001 From: jezscha Date: Fri, 7 May 2021 12:24:09 +0000 Subject: [PATCH 1/9] Create draft PR for #1378 From caf35d6d70ba743f5ab2f6a52dc07d636982342a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 13:01:41 +0200 Subject: [PATCH 2/9] Hiero: fixing audio collection --- openpype/hosts/hiero/otio/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/hiero/otio/utils.py b/openpype/hosts/hiero/otio/utils.py index f882a5d1f2..4c5d46bd51 100644 --- a/openpype/hosts/hiero/otio/utils.py +++ b/openpype/hosts/hiero/otio/utils.py @@ -68,7 +68,11 @@ def get_rate(item): return None num, den = item.framerate().toRational() - rate = float(num) / float(den) + + try: + rate = float(num) / float(den) + except ZeroDivisionError: + return None if rate.is_integer(): return rate From b97c246310c11d412aa7d3bb46d50a5175808ab2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 13:02:44 +0200 Subject: [PATCH 3/9] Hiero: adding audio family to instance --- openpype/hosts/hiero/plugins/publish/precollect_instances.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/hosts/hiero/plugins/publish/precollect_instances.py b/openpype/hosts/hiero/plugins/publish/precollect_instances.py index a1dee711b7..4cb6011a10 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_instances.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_instances.py @@ -57,6 +57,10 @@ class PrecollectInstances(pyblish.api.ContextPlugin): families = [str(f) for f in tag_data["families"]] families.insert(0, str(family)) + # add audio to families + if tag_data["audio"]: + families.append("audio") + # form label label = asset if asset != clip_name: From 26fc0b49e4155ccca73676f1ce7394734d84a3ad Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 13:03:13 +0200 Subject: [PATCH 4/9] Global: updating editorial lib with frames to timecode --- openpype/lib/__init__.py | 2 ++ openpype/lib/editorial.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index 457ceb1d56..838c5aa7a1 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -139,6 +139,7 @@ from .editorial import ( trim_media_range, range_from_frames, frames_to_secons, + frames_to_timecode, make_sequence_collection ) @@ -246,5 +247,6 @@ __all__ = [ "trim_media_range", "range_from_frames", "frames_to_secons", + "frames_to_timecode", "make_sequence_collection" ] diff --git a/openpype/lib/editorial.py b/openpype/lib/editorial.py index 1dbc4d7954..bf9a0cb506 100644 --- a/openpype/lib/editorial.py +++ b/openpype/lib/editorial.py @@ -137,6 +137,11 @@ def frames_to_secons(frames, framerate): return _ot.to_seconds(rt) +def frames_to_timecode(frames, framerate): + rt = _ot.from_frames(frames, framerate) + return _ot.to_timecode(rt) + + def make_sequence_collection(path, otio_range, metadata): """ Make collection from path otio range and otio metadata. From 11679000074325f045f2aa7a5871d02526d72be2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 13:03:35 +0200 Subject: [PATCH 5/9] Global: extract otio audio clips --- .../publish/extract_otio_audio_tracks.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 openpype/plugins/publish/extract_otio_audio_tracks.py diff --git a/openpype/plugins/publish/extract_otio_audio_tracks.py b/openpype/plugins/publish/extract_otio_audio_tracks.py new file mode 100644 index 0000000000..ade53abeef --- /dev/null +++ b/openpype/plugins/publish/extract_otio_audio_tracks.py @@ -0,0 +1,263 @@ +import os +import pyblish +import openpype.api +from openpype.lib import ( + get_ffmpeg_tool_path +) +import tempfile +import opentimelineio as otio + + +class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): + """Extract Audio tracks from OTIO timeline. + + Process will merge all found audio tracks into one long .wav file at frist + stage. Then it will trim it into individual short audio files relative to + asset length and add it to each marked instance data representation. This + is influenced by instance data audio attribute """ + + order = pyblish.api.CollectorOrder - 0.571 + label = "Extract OTIO Audio Tracks" + hosts = ["hiero"] + + # FFmpeg tools paths + ffmpeg_path = get_ffmpeg_tool_path("ffmpeg") + + def process(self, context): + """Convert otio audio track's content to audio representations + + Args: + context (pyblish.Context): context of publisher + """ + + # get sequence + otio_timeline = context.data["otioTimeline"] + + # temp file + audio_temp_fpath = self.create_temp_file("audio") + + # get all audio inputs from otio timeline + audio_inputs = self.get_audio_track_items(otio_timeline) + + # create empty audio with longest duration + empty = self.create_empty(audio_inputs) + + # add empty to list of audio inputs + audio_inputs.insert(0, empty) + + # create cmd + cmd = self.ffmpeg_path + " " + cmd += self.create_cmd(audio_inputs) + cmd += audio_temp_fpath + + # run subprocess + self.log.debug("Executing: {}".format(cmd)) + openpype.api.run_subprocess( + cmd, shell=True, logger=self.log + ) + + # remove empty + os.remove(empty["mediaPath"]) + + # split the long audio file to peces devided by isntances + audio_instances = self.get_audio_instances(context) + + # cut instance framerange and add to representations + self.create_audio_representations(audio_temp_fpath, audio_instances) + + def create_audio_representations(self, audio_file, instances): + for inst in instances: + # create empty representation attr + if "representations" not in inst.data: + inst.data["representations"] = [] + + name = inst.data["name"] + + # frameranges + timeline_in_h = inst.data["clipInH"] + timeline_out_h = inst.data["clipOutH"] + fps = inst.data["fps"] + + # seconds + duration = (timeline_out_h - timeline_in_h) + 1 + start_sec = float(timeline_in_h / fps) + duration_sec = float(duration / fps) + + # temp audio file + audio_fpath = self.create_temp_file(name) + + cmd = " ".join([ + self.ffmpeg_path, + "-ss {}".format(start_sec), + "-t {}".format(duration_sec), + "-i {}".format(audio_file), + audio_fpath + ]) + + # run subprocess + self.log.debug("Executing: {}".format(cmd)) + openpype.api.run_subprocess( + cmd, shell=True, logger=self.log + ) + + # add to representations + inst.data["representations"].append({ + "files": os.path.basename(audio_fpath), + "name": "wav", + "ext": "wav", + "stagingDir": os.path.dirname(audio_fpath), + "frameStart": 0, + "frameEnd": duration + }) + + def get_audio_instances(self, context): + """Return only instances which are having audio in families + + Args: + context (pyblish.context): context of publisher + + Returns: + list: list of selected instances + """ + return [ + _i for _i in context + if bool("audio" in _i.data.get("families", [])) + ] + + def get_audio_track_items(self, otio_timeline): + """Get all audio clips form OTIO audio tracks + + Args: + otio_timeline (otio.schema.timeline): timeline object + + Returns: + list: list of audio clip dictionaries + """ + output = [] + # go trough all audio tracks + for otio_track in otio_timeline.tracks: + if "Audio" not in otio_track.kind: + continue + self.log.debug("_" * 50) + playhead = 0 + for otio_clip in otio_track: + self.log.debug(otio_clip) + if isinstance(otio_clip, otio.schema.Gap): + playhead += otio_clip.source_range.duration.value + elif isinstance(otio_clip, otio.schema.Clip): + start = otio_clip.source_range.start_time.value + duration = otio_clip.source_range.duration.value + fps = otio_clip.source_range.start_time.rate + media_path = otio_clip.media_reference.target_url + input = { + "mediaPath": media_path, + "delayFrame": playhead, + "startFrame": start, + "durationFrame": duration, + "delayMilSec": int(float(playhead / fps) * 1000), + "startSec": float(start / fps), + "durationSec": float(duration / fps), + "fps": fps + } + if input not in output: + output.append(input) + self.log.debug("__ input: {}".format(input)) + playhead += otio_clip.source_range.duration.value + + return output + + def create_empty(self, inputs): + """Create an empty audio file used as duration placeholder + + Args: + inputs (list): list of audio clip dictionaries + + Returns: + dict: audio clip dictionary + """ + # temp file + empty_fpath = self.create_temp_file("empty") + + # get all end frames + end_secs = [(_i["delayFrame"] + _i["durationFrame"]) / _i["fps"] + for _i in inputs] + # get the max of end frames + max_duration_sec = max(end_secs) + + # create empty cmd + cmd = " ".join([ + self.ffmpeg_path, + "-f lavfi", + "-i anullsrc=channel_layout=stereo:sample_rate=48000", + "-t {}".format(max_duration_sec), + empty_fpath + ]) + + # generate empty with ffmpeg + # run subprocess + self.log.debug("Executing: {}".format(cmd)) + + openpype.api.run_subprocess( + cmd, shell=True, logger=self.log + ) + + # return dict with output + return { + "mediaPath": empty_fpath, + "delayMilSec": 0, + "startSec": 0.00, + "durationSec": max_duration_sec + } + + def create_cmd(self, inputs): + """Creating multiple input cmd string + + Args: + inputs (list): list of input dicts. Order mater. + + Returns: + str: the command body + + """ + # create cmd segments + _inputs = "" + _filters = "-filter_complex \"" + _channels = "" + for index, input in enumerate(inputs): + input_format = input.copy() + input_format.update({"i": index}) + _inputs += ( + "-ss {startSec} " + "-t {durationSec} " + "-i \"{mediaPath}\" " + ).format(**input_format) + + _filters += "[{i}]adelay={delayMilSec}:all=1[r{i}]; ".format( + **input_format) + _channels += "[r{}]".format(index) + + # merge all cmd segments together + cmd = _inputs + _filters + _channels + cmd += str( + "amix=inputs={inputs}:duration=first:" + "dropout_transition=1000,volume={inputs}[a]\" " + ).format(inputs=len(inputs)) + cmd += "-map \"[a]\" " + + return cmd + + def create_temp_file(self, name): + """Create temp wav file + + Args: + name (str): name to be used in file name + + Returns: + str: temp fpath + """ + return os.path.normpath( + tempfile.mktemp( + prefix="pyblish_tmp_{}_".format(name), + suffix=".wav" + ) + ) From 6a485fc88756a4df573c2c1d093f44555e8eddb6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 15:53:11 +0200 Subject: [PATCH 6/9] Hiero: improving instace pre-collector so it is creating audio instances --- .../plugins/publish/precollect_instances.py | 62 +++++++++++++++++-- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/hiero/plugins/publish/precollect_instances.py b/openpype/hosts/hiero/plugins/publish/precollect_instances.py index 4cb6011a10..8cccdec99a 100644 --- a/openpype/hosts/hiero/plugins/publish/precollect_instances.py +++ b/openpype/hosts/hiero/plugins/publish/precollect_instances.py @@ -24,7 +24,7 @@ class PrecollectInstances(pyblish.api.ContextPlugin): for track_item in selected_timeline_items: - data = dict() + data = {} clip_name = track_item.name() # get openpype tag data @@ -43,6 +43,11 @@ class PrecollectInstances(pyblish.api.ContextPlugin): tag_data["handleEnd"] = min( tag_data["handleEnd"], int(track_item.handleOutLength())) + # add audio to families + with_audio = False + if tag_data.pop("audio"): + with_audio = True + # add tag data to instance data data.update({ k: v for k, v in tag_data.items() @@ -57,10 +62,6 @@ class PrecollectInstances(pyblish.api.ContextPlugin): families = [str(f) for f in tag_data["families"]] families.insert(0, str(family)) - # add audio to families - if tag_data["audio"]: - families.append("audio") - # form label label = asset if asset != clip_name: @@ -98,6 +99,17 @@ class PrecollectInstances(pyblish.api.ContextPlugin): self.log.debug( "_ instance.data: {}".format(pformat(instance.data))) + if not with_audio: + return + + # create audio subset instance + self.create_audio_instance(context, **data) + + # add audioReview attribute to plate instance data + # if reviewTrack is on + if tag_data.get("reviewTrack") is not None: + instance.data["reviewAudio"] = True + def get_resolution_to_data(self, data, context): assert data.get("otioClip"), "Missing `otioClip` data" @@ -163,6 +175,46 @@ class PrecollectInstances(pyblish.api.ContextPlugin): self.log.debug( "_ instance.data: {}".format(pformat(instance.data))) + def create_audio_instance(self, context, **data): + master_layer = data.get("heroTrack") + + if not master_layer: + return + + asset = data.get("asset") + item = data.get("item") + clip_name = item.name() + + asset = data["asset"] + subset = "audioMain" + + # insert family into families + family = "audio" + + # form label + label = asset + if asset != clip_name: + label += " ({}) ".format(clip_name) + label += " {}".format(subset) + label += " [{}]".format(family) + + data.update({ + "name": "{}_{}".format(asset, subset), + "label": label, + "subset": subset, + "asset": asset, + "family": family, + "families": ["clip"] + }) + # remove review track attr if any + data.pop("reviewTrack") + + # create instance + instance = context.create_instance(**data) + self.log.info("Creating instance: {}".format(instance)) + self.log.debug( + "_ instance.data: {}".format(pformat(instance.data))) + def get_otio_clip_instance_data(self, otio_timeline, track_item): """ Return otio objects for timeline, track and clip From b0c72f38bff062f1ad6cec5114d4d2019cb4988e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 15:53:30 +0200 Subject: [PATCH 7/9] global: dont do anything with audio instance --- openpype/plugins/publish/collect_otio_subset_resources.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index d687c1920a..cebfc90630 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -22,6 +22,10 @@ class CollectOcioSubsetResources(pyblish.api.InstancePlugin): hosts = ["resolve", "hiero"] def process(self, instance): + + if "audio" in instance.data["family"]: + return + if not instance.data.get("representations"): instance.data["representations"] = list() version_data = dict() From 6f39b0f046e5358a520e429505ecfd7fec51e907 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 15:53:57 +0200 Subject: [PATCH 8/9] Global: finalizing otio audio extractor --- .../publish/extract_otio_audio_tracks.py | 106 +++++++++++------- 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/openpype/plugins/publish/extract_otio_audio_tracks.py b/openpype/plugins/publish/extract_otio_audio_tracks.py index ade53abeef..23e9fcd03b 100644 --- a/openpype/plugins/publish/extract_otio_audio_tracks.py +++ b/openpype/plugins/publish/extract_otio_audio_tracks.py @@ -16,9 +16,9 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): asset length and add it to each marked instance data representation. This is influenced by instance data audio attribute """ - order = pyblish.api.CollectorOrder - 0.571 + order = pyblish.api.ExtractorOrder - 0.44 label = "Extract OTIO Audio Tracks" - hosts = ["hiero"] + hosts = ["hiero", "resolve"] # FFmpeg tools paths ffmpeg_path = get_ffmpeg_tool_path("ffmpeg") @@ -29,6 +29,13 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): Args: context (pyblish.Context): context of publisher """ + # split the long audio file to peces devided by isntances + audio_instances = self.get_audio_instances(context) + self.log.debug("Audio instances: {}".format(len(audio_instances))) + + if len(audio_instances) < 1: + self.log.info("No audio instances available") + return # get sequence otio_timeline = context.data["otioTimeline"] @@ -59,56 +66,77 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): # remove empty os.remove(empty["mediaPath"]) - # split the long audio file to peces devided by isntances - audio_instances = self.get_audio_instances(context) - # cut instance framerange and add to representations - self.create_audio_representations(audio_temp_fpath, audio_instances) + self.add_audio_to_instances(audio_temp_fpath, audio_instances) - def create_audio_representations(self, audio_file, instances): + # remove full mixed audio file + os.remove(audio_temp_fpath) + + def add_audio_to_instances(self, audio_file, instances): + created_files = [] for inst in instances: - # create empty representation attr - if "representations" not in inst.data: - inst.data["representations"] = [] + name = inst.data["asset"] - name = inst.data["name"] + recycling_file = [f for f in created_files if name in f] # frameranges timeline_in_h = inst.data["clipInH"] timeline_out_h = inst.data["clipOutH"] fps = inst.data["fps"] - # seconds + # create duration duration = (timeline_out_h - timeline_in_h) + 1 - start_sec = float(timeline_in_h / fps) - duration_sec = float(duration / fps) - # temp audio file - audio_fpath = self.create_temp_file(name) + # ffmpeg generate new file only if doesnt exists already + if not recycling_file: + # convert to seconds + start_sec = float(timeline_in_h / fps) + duration_sec = float(duration / fps) - cmd = " ".join([ - self.ffmpeg_path, - "-ss {}".format(start_sec), - "-t {}".format(duration_sec), - "-i {}".format(audio_file), - audio_fpath - ]) + # temp audio file + audio_fpath = self.create_temp_file(name) - # run subprocess - self.log.debug("Executing: {}".format(cmd)) - openpype.api.run_subprocess( - cmd, shell=True, logger=self.log - ) + cmd = " ".join([ + self.ffmpeg_path, + "-ss {}".format(start_sec), + "-t {}".format(duration_sec), + "-i {}".format(audio_file), + audio_fpath + ]) - # add to representations - inst.data["representations"].append({ - "files": os.path.basename(audio_fpath), - "name": "wav", - "ext": "wav", - "stagingDir": os.path.dirname(audio_fpath), - "frameStart": 0, - "frameEnd": duration - }) + # run subprocess + self.log.debug("Executing: {}".format(cmd)) + openpype.api.run_subprocess( + cmd, shell=True, logger=self.log + ) + else: + audio_fpath = recycling_file.pop() + + if "audio" in (inst.data["families"] + [inst.data["family"]]): + # create empty representation attr + if "representations" not in inst.data: + inst.data["representations"] = [] + # add to representations + inst.data["representations"].append({ + "files": os.path.basename(audio_fpath), + "name": "wav", + "ext": "wav", + "stagingDir": os.path.dirname(audio_fpath), + "frameStart": 0, + "frameEnd": duration + }) + + elif "reviewAudio" in inst.data.keys(): + audio_attr = inst.data.get("audio") or [] + audio_attr.append({ + "filename": audio_fpath, + "offset": 0 + }) + inst.data["audio"] = audio_attr + + # add generated audio file to created files for recycling + if audio_fpath not in created_files: + created_files.append(audio_fpath) def get_audio_instances(self, context): """Return only instances which are having audio in families @@ -121,7 +149,9 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): """ return [ _i for _i in context - if bool("audio" in _i.data.get("families", [])) + if bool("audio" in ( + _i.data.get("families", []) + [_i.data["family"]]) + ) or _i.data.get("reviewAudio") ] def get_audio_track_items(self, otio_timeline): From 121a338be17f0fd737c7cf5ae0e92075e7167547 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 12 May 2021 15:56:31 +0200 Subject: [PATCH 9/9] hound: suggestions --- openpype/plugins/publish/extract_otio_audio_tracks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/extract_otio_audio_tracks.py b/openpype/plugins/publish/extract_otio_audio_tracks.py index 23e9fcd03b..43e40097f7 100644 --- a/openpype/plugins/publish/extract_otio_audio_tracks.py +++ b/openpype/plugins/publish/extract_otio_audio_tracks.py @@ -149,9 +149,11 @@ class ExtractOtioAudioTracks(pyblish.api.ContextPlugin): """ return [ _i for _i in context + # filter only those with audio family + # and also with reviewAudio data key if bool("audio" in ( _i.data.get("families", []) + [_i.data["family"]]) - ) or _i.data.get("reviewAudio") + ) or _i.data.get("reviewAudio") ] def get_audio_track_items(self, otio_timeline):