Merge branch 'develop' of https://github.com/pypeclub/OpenPype into feature/multiverse

openpype/plugins/publish/collect_scene_loaded_versions.py

@@ -1,3 +1,4 @@
+from bson.objectid import ObjectId
 
 import pyblish.api
 from avalon import api, io
@@ -35,7 +36,7 @@ class CollectSceneLoadedVersions(pyblish.api.ContextPlugin):
 
         loaded_versions = []
         _containers = list(host.ls())
-        _repr_ids = [io.ObjectId(c["representation"]) for c in _containers]
+        _repr_ids = [ObjectId(c["representation"]) for c in _containers]
         version_by_repr = {
             str(doc["_id"]): doc["parent"] for doc in
             io.find({"_id": {"$in": _repr_ids}}, projection={"parent": 1})
@@ -46,7 +47,7 @@ class CollectSceneLoadedVersions(pyblish.api.ContextPlugin):
             # may have more then one representation that are same version
             version = {
                 "subsetName": con["name"],
-                "representation": io.ObjectId(con["representation"]),
+                "representation": ObjectId(con["representation"]),
                 "version": version_by_repr[con["representation"]],  # _id
             }
             loaded_versions.append(version)

openpype/plugins/publish/extract_hierarchy_avalon.py

@@ -64,7 +64,7 @@ class ExtractHierarchyToAvalon(pyblish.api.ContextPlugin):
                     data["tasks"] = tasks
                 parents = []
                 visualParent = None
-                # do not store project"s id as visualParent (silo asset)
+                # do not store project"s id as visualParent
                 if self.project is not None:
                     if self.project["_id"] != parent["_id"]:
                         visualParent = parent["_id"]

openpype/plugins/publish/extract_review.py

@@ -13,7 +13,7 @@ import pyblish.api
 import openpype.api
 from openpype.lib import (
     get_ffmpeg_tool_path,
-    ffprobe_streams,
+    get_ffprobe_streams,
 
     path_to_subprocess_arg,
 
@@ -747,10 +747,14 @@ class ExtractReview(pyblish.api.InstancePlugin):
         collections = clique.assemble(files)[0]
         assert len(collections) == 1, "Multiple collections found."
         col = collections[0]
-        # do nothing if sequence is complete
-        if list(col.indexes)[0] == start_frame and \
-                list(col.indexes)[-1] == end_frame and \
-                col.is_contiguous():
+
+        # do nothing if no gap is found in input range
+        not_gap = True
+        for fr in range(start_frame, end_frame + 1):
+            if fr not in col.indexes:
+                not_gap = False
+
+        if not_gap:
             return []
 
         holes = col.holes()
@@ -1146,7 +1150,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
         # NOTE Skipped using instance's resolution
         full_input_path_single_file = temp_data["full_input_path_single_file"]
         try:
-            streams = ffprobe_streams(
+            streams = get_ffprobe_streams(
                 full_input_path_single_file, self.log
             )
         except Exception as exc:
@@ -1188,8 +1192,8 @@ class ExtractReview(pyblish.api.InstancePlugin):
 
         # NOTE Setting only one of `width` or `heigth` is not allowed
         # - settings value can't have None but has value of 0
-        output_width = output_width or output_def.get("width") or None
-        output_height = output_height or output_def.get("height") or None
+        output_width = output_def.get("width") or output_width or None
+        output_height = output_def.get("height") or output_height or None
 
         # Overscal color
         overscan_color_value = "black"

openpype/plugins/publish/extract_review_slate.py

@@ -1,7 +1,14 @@
 import os
 import openpype.api
-import openpype.lib
 import pyblish
+from openpype.lib import (
+    path_to_subprocess_arg,
+    get_ffmpeg_tool_path,
+    get_ffprobe_data,
+    get_ffprobe_streams,
+    get_ffmpeg_codec_args,
+    get_ffmpeg_format_args,
+)
 
 
 class ExtractReviewSlate(openpype.api.Extractor):
@@ -24,9 +31,9 @@ class ExtractReviewSlate(openpype.api.Extractor):
 
         suffix = "_slate"
         slate_path = inst_data.get("slateFrame")
-        ffmpeg_path = openpype.lib.get_ffmpeg_tool_path("ffmpeg")
+        ffmpeg_path = get_ffmpeg_tool_path("ffmpeg")
 
-        slate_streams = openpype.lib.ffprobe_streams(slate_path, self.log)
+        slate_streams = get_ffprobe_streams(slate_path, self.log)
         # Try to find first stream with defined 'width' and 'height'
         # - this is to avoid order of streams where audio can be as first
         # - there may be a better way (checking `codec_type`?)+
@@ -66,7 +73,7 @@ class ExtractReviewSlate(openpype.api.Extractor):
                 os.path.normpath(stagingdir), repre["files"])
             self.log.debug("__ input_path: {}".format(input_path))
 
-            video_streams = openpype.lib.ffprobe_streams(
+            video_streams = get_ffprobe_streams(
                 input_path, self.log
             )
 
@@ -143,7 +150,7 @@ class ExtractReviewSlate(openpype.api.Extractor):
             else:
                 input_args.extend(repre["outputDef"].get('input', []))
             input_args.append("-loop 1 -i {}".format(
-                openpype.lib.path_to_subprocess_arg(slate_path)
+                path_to_subprocess_arg(slate_path)
             ))
             input_args.extend([
                 "-r {}".format(fps),
@@ -157,7 +164,7 @@ class ExtractReviewSlate(openpype.api.Extractor):
                 output_args.extend(repre["_profile"].get('output', []))
             else:
                 # Codecs are copied from source for whole input
-                codec_args = self.codec_args(repre)
+                codec_args = self._get_codec_args(repre)
                 output_args.extend(codec_args)
 
             # make sure colors are correct
@@ -216,12 +223,12 @@ class ExtractReviewSlate(openpype.api.Extractor):
 
             slate_v_path = slate_path.replace(".png", ext)
             output_args.append(
-                openpype.lib.path_to_subprocess_arg(slate_v_path)
+                path_to_subprocess_arg(slate_v_path)
             )
             _remove_at_end.append(slate_v_path)
 
             slate_args = [
-                openpype.lib.path_to_subprocess_arg(ffmpeg_path),
+                path_to_subprocess_arg(ffmpeg_path),
                 " ".join(input_args),
                 " ".join(output_args)
             ]
@@ -331,7 +338,7 @@ class ExtractReviewSlate(openpype.api.Extractor):
 
         return vf_back
 
-    def codec_args(self, repre):
+    def _get_codec_args(self, repre):
         """Detect possible codec arguments from representation."""
         codec_args = []
 
@@ -345,7 +352,7 @@ class ExtractReviewSlate(openpype.api.Extractor):
 
         try:
             # Get information about input file via ffprobe tool
-            streams = openpype.lib.ffprobe_streams(full_input_path, self.log)
+            ffprobe_data = get_ffprobe_data(full_input_path, self.log)
         except Exception:
             self.log.warning(
                 "Could not get codec data from input.",
@@ -353,42 +360,14 @@ class ExtractReviewSlate(openpype.api.Extractor):
             )
             return codec_args
 
-        # Try to find first stream that is not an audio
-        no_audio_stream = None
-        for stream in streams:
-            if stream.get("codec_type") != "audio":
-                no_audio_stream = stream
-                break
+        source_ffmpeg_cmd = repre.get("ffmpeg_cmd")
+        codec_args.extend(
+            get_ffmpeg_format_args(ffprobe_data, source_ffmpeg_cmd)
+        )
+        codec_args.extend(
+            get_ffmpeg_codec_args(
+                ffprobe_data, source_ffmpeg_cmd, logger=self.log
+            )
+        )
 
-        if no_audio_stream is None:
-            self.log.warning((
-                "Couldn't find stream that is not an audio from file \"{}\""
-            ).format(full_input_path))
-            return codec_args
-
-        codec_name = no_audio_stream.get("codec_name")
-        if codec_name:
-            codec_args.append("-codec:v {}".format(codec_name))
-
-        profile_name = no_audio_stream.get("profile")
-        if profile_name:
-            # Rest of arguments is prores_kw specific
-            if codec_name == "prores_ks":
-                codec_tag_to_profile_map = {
-                    "apco": "proxy",
-                    "apcs": "lt",
-                    "apcn": "standard",
-                    "apch": "hq",
-                    "ap4h": "4444",
-                    "ap4x": "4444xq"
-                }
-                codec_tag_str = no_audio_stream.get("codec_tag_string")
-                if codec_tag_str:
-                    profile = codec_tag_to_profile_map.get(codec_tag_str)
-                    if profile:
-                        codec_args.extend(["-profile:v", profile])
-
-        pix_fmt = no_audio_stream.get("pix_fmt")
-        if pix_fmt:
-            codec_args.append("-pix_fmt {}".format(pix_fmt))
         return codec_args

openpype/plugins/publish/integrate_hero_version.py

@@ -4,6 +4,7 @@ import clique
 import errno
 import shutil
 
+from bson.objectid import ObjectId
 from pymongo import InsertOne, ReplaceOne
 import pyblish.api
 from avalon import api, io, schema
@@ -161,7 +162,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
         if old_version:
             new_version_id = old_version["_id"]
         else:
-            new_version_id = io.ObjectId()
+            new_version_id = ObjectId()
 
         new_hero_version = {
             "_id": new_version_id,
@@ -384,7 +385,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
 
                 # Create representation
                 else:
-                    repre["_id"] = io.ObjectId()
+                    repre["_id"] = ObjectId()
                     bulk_writes.append(
                         InsertOne(repre)
                     )
@@ -420,7 +421,7 @@ class IntegrateHeroVersion(pyblish.api.InstancePlugin):
 
                 else:
                     repre["old_id"] = repre["_id"]
-                    repre["_id"] = io.ObjectId()
+                    repre["_id"] = ObjectId()
                     repre["type"] = "archived_representation"
                     bulk_writes.append(
                         InsertOne(repre)

openpype/plugins/publish/integrate_inputlinks.py

@@ -1,8 +1,10 @@
-
 from collections import OrderedDict
-from avalon import io
+
+from bson.objectid import ObjectId
 import pyblish.api
 
+from avalon import io
+
 
 class IntegrateInputLinks(pyblish.api.ContextPlugin):
     """Connecting version level dependency links"""
@@ -104,7 +106,7 @@ class IntegrateInputLinks(pyblish.api.ContextPlugin):
         # future.
         link = OrderedDict()
         link["type"] = link_type
-        link["id"] = io.ObjectId(input_id)
+        link["id"] = ObjectId(input_id)
         link["linkedBy"] = "publish"
 
         if "inputLinks" not in version_doc["data"]:

openpype/plugins/publish/integrate_new.py

@@ -9,17 +9,19 @@ import six
 import re
 import shutil
 
+from bson.objectid import ObjectId
 from pymongo import DeleteOne, InsertOne
 import pyblish.api
 from avalon import io
-from avalon.api import format_template_with_optional_keys
 import openpype.api
 from datetime import datetime
 # from pype.modules import ModulesManager
 from openpype.lib.profiles_filtering import filter_profiles
 from openpype.lib import (
     prepare_template_data,
-    create_hard_link
+    create_hard_link,
+    StringTemplate,
+    TemplateUnsolved
 )
 
 # this is needed until speedcopy for linux is fixed
@@ -295,7 +297,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 bulk_writes.append(DeleteOne({"_id": repre_id}))
 
                 repre["orig_id"] = repre_id
-                repre["_id"] = io.ObjectId()
+                repre["_id"] = ObjectId()
                 repre["type"] = "archived_representation"
                 bulk_writes.append(InsertOne(repre))
 
@@ -574,7 +576,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
             # Create new id if existing representations does not match
             if repre_id is None:
-                repre_id = io.ObjectId()
+                repre_id = ObjectId()
 
             data = repre.get("data") or {}
             data.update({'path': dst, 'template': template})
@@ -783,7 +785,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         families = [instance.data["family"]]
         families.extend(instance.data.get("families", []))
         io.update_many(
-            {"type": "subset", "_id": io.ObjectId(subset["_id"])},
+            {"type": "subset", "_id": ObjectId(subset["_id"])},
             {"$set": {"data.families": families}}
         )
 
@@ -808,7 +810,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if subset_group:
             io.update_many({
                 'type': 'subset',
-                '_id': io.ObjectId(subset_id)
+                '_id': ObjectId(subset_id)
             }, {'$set': {'data.subsetGroup': subset_group}})
 
     def _get_subset_group(self, instance):
@@ -856,9 +858,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         fill_pairs = prepare_template_data(fill_pairs)
 
         try:
-            filled_template = \
-                format_template_with_optional_keys(fill_pairs, template)
-        except KeyError:
+            filled_template = StringTemplate.format_strict_template(
+                template, fill_pairs
+            )
+        except (KeyError, TemplateUnsolved):
             keys = []
             if fill_pairs:
                 keys = fill_pairs.keys()
@@ -1054,7 +1057,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         sync_project_presets = None
 
         rec = {
-            "_id": io.ObjectId(),
+            "_id": ObjectId(),
             "path": path
         }
         if size: