Merge develop

openpype/plugins/publish/cleanup.py

@@ -93,6 +93,10 @@ class CleanUp(pyblish.api.InstancePlugin):
             self.log.info("No staging directory found: %s" % staging_dir)
             return
 
+        if instance.data.get("stagingDir_persistent"):
+            self.log.info("Staging dir: %s should be persistent" % staging_dir)
+            return
+
         self.log.info("Removing staging directory {}".format(staging_dir))
         shutil.rmtree(staging_dir)
 

openpype/plugins/publish/cleanup_farm.py

@@ -37,7 +37,7 @@ class CleanUpFarm(pyblish.api.ContextPlugin):
         dirpaths_to_remove = set()
         for instance in context:
             staging_dir = instance.data.get("stagingDir")
-            if staging_dir:
+            if staging_dir and not instance.data.get("stagingDir_persistent"):
                 dirpaths_to_remove.add(os.path.normpath(staging_dir))
 
             if "representations" in instance.data:

openpype/plugins/publish/collect_comment.py

@@ -29,7 +29,7 @@ from openpype.pipeline.publish import OpenPypePyblishPluginMixin
 
 
 class CollectInstanceCommentDef(
-    pyblish.api.ContextPlugin,
+    pyblish.api.InstancePlugin,
     OpenPypePyblishPluginMixin
 ):
     label = "Comment per instance"

openpype/plugins/publish/collect_custom_staging_dir.py

@@ -0,0 +1,67 @@
+"""
+Requires:
+    anatomy
+
+
+Provides:
+    instance.data     -> stagingDir (folder path)
+                      -> stagingDir_persistent (bool)
+"""
+import copy
+import os.path
+
+import pyblish.api
+
+from openpype.pipeline.publish.lib import get_custom_staging_dir_info
+
+
+class CollectCustomStagingDir(pyblish.api.InstancePlugin):
+    """Looks through profiles if stagingDir should be persistent and in special
+    location.
+
+    Transient staging dir could be useful in specific use cases where is
+    desirable to have temporary renders in specific, persistent folders, could
+    be on disks optimized for speed for example.
+
+    It is studio responsibility to clean up obsolete folders with data.
+
+    Location of the folder is configured in `project_anatomy/templates/others`.
+    ('transient' key is expected, with 'folder' key)
+
+    Which family/task type/subset is applicable is configured in:
+    `project_settings/global/tools/publish/custom_staging_dir_profiles`
+
+    """
+    label = "Collect Custom Staging Directory"
+    order = pyblish.api.CollectorOrder + 0.4990
+
+    template_key = "transient"
+
+    def process(self, instance):
+        family = instance.data["family"]
+        subset_name = instance.data["subset"]
+        host_name = instance.context.data["hostName"]
+        project_name = instance.context.data["projectName"]
+
+        anatomy = instance.context.data["anatomy"]
+        anatomy_data = copy.deepcopy(instance.data["anatomyData"])
+        task = anatomy_data.get("task", {})
+
+        transient_tml, is_persistent = get_custom_staging_dir_info(
+            project_name, host_name, family, task.get("name"),
+            task.get("type"), subset_name, anatomy=anatomy, log=self.log)
+        result_str = "Not adding"
+        if transient_tml:
+            anatomy_data["root"] = anatomy.roots
+            scene_name = instance.context.data.get("currentFile")
+            if scene_name:
+                anatomy_data["scene_name"] = os.path.basename(scene_name)
+            transient_dir = transient_tml.format(**anatomy_data)
+            instance.data["stagingDir"] = transient_dir
+
+            instance.data["stagingDir_persistent"] = is_persistent
+            result_str = "Adding '{}' as".format(transient_dir)
+
+        self.log.info("{} custom staging dir for instance with '{}'".format(
+            result_str, family
+        ))

openpype/plugins/publish/extract_burnin.py

@@ -16,9 +16,7 @@ from openpype.lib import (
 
     get_transcode_temp_directory,
     convert_input_paths_for_ffmpeg,
-    should_convert_for_ffmpeg,
-
-    CREATE_NO_WINDOW
+    should_convert_for_ffmpeg
 )
 from openpype.lib.profiles_filtering import filter_profiles
 
@@ -255,6 +253,9 @@ class ExtractBurnin(publish.Extractor):
             # Add context data burnin_data.
             burnin_data["custom"] = custom_data
 
+            # Add data members.
+            burnin_data.update(instance.data.get("burninDataMembers", {}))
+
             # Add source camera name to burnin data
             camera_name = repre.get("camera_name")
             if camera_name:
@@ -339,8 +340,6 @@ class ExtractBurnin(publish.Extractor):
                     "logger": self.log,
                     "env": {}
                 }
-                if platform.system().lower() == "windows":
-                    process_kwargs["creationflags"] = CREATE_NO_WINDOW
 
                 run_openpype_process(*args, **process_kwargs)
                 # Remove the temporary json
@@ -730,7 +729,6 @@ class ExtractBurnin(publish.Extractor):
             return filtered_burnin_defs
 
         families = self.families_from_instance(instance)
-        low_families = [family.lower() for family in families]
 
         for filename_suffix, orig_burnin_def in burnin_defs.items():
             burnin_def = copy.deepcopy(orig_burnin_def)
@@ -741,7 +739,7 @@ class ExtractBurnin(publish.Extractor):
 
             families_filters = def_filter["families"]
             if not self.families_filter_validation(
-                low_families, families_filters
+                families, families_filters
             ):
                 self.log.debug((
                     "Skipped burnin definition \"{}\". Family"
@@ -778,31 +776,19 @@ class ExtractBurnin(publish.Extractor):
         return filtered_burnin_defs
 
     def families_filter_validation(self, families, output_families_filter):
-        """Determine if entered families intersect with families filters.
+        """Determines if entered families intersect with families filters.
 
         All family values are lowered to avoid unexpected results.
         """
-        if not output_families_filter:
+
+        families_filter_lower = set(family.lower() for family in
+                                    output_families_filter
+                                    # Exclude empty filter values
+                                    if family)
+        if not families_filter_lower:
             return True
-
-        for family_filter in output_families_filter:
-            if not family_filter:
-                continue
-
-            if not isinstance(family_filter, (list, tuple)):
-                if family_filter.lower() not in families:
-                    continue
-                return True
-
-            valid = True
-            for family in family_filter:
-                if family.lower() not in families:
-                    valid = False
-                    break
-
-            if valid:
-                return True
-        return False
+        return any(family.lower() in families_filter_lower
+                   for family in families)
 
     def families_from_instance(self, instance):
         """Return all families of entered instance."""

openpype/plugins/publish/integrate.py

@@ -24,7 +24,10 @@ from openpype.client import (
     get_version_by_name,
 )
 from openpype.lib import source_hash
-from openpype.lib.file_transaction import FileTransaction
+from openpype.lib.file_transaction import (
+    FileTransaction,
+    DuplicateDestinationError
+)
 from openpype.pipeline.publish import (
     KnownPublishError,
     get_publish_template_name,
@@ -80,10 +83,12 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
     order = pyblish.api.IntegratorOrder
     families = ["workfile",
                 "pointcache",
+                "pointcloud",
                 "proxyAbc",
                 "camera",
                 "animation",
                 "model",
+                "maxScene",
                 "mayaAscii",
                 "mayaScene",
                 "setdress",
@@ -168,9 +173,18 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
             ).format(instance.data["family"]))
             return
 
-        file_transactions = FileTransaction(log=self.log)
+        file_transactions = FileTransaction(log=self.log,
+                                            # Enforce unique transfers
+                                            allow_queue_replacements=False)
         try:
             self.register(instance, file_transactions, filtered_repres)
+        except DuplicateDestinationError as exc:
+            # Raise DuplicateDestinationError as KnownPublishError
+            # and rollback the transactions
+            file_transactions.rollback()
+            six.reraise(KnownPublishError,
+                        KnownPublishError(exc),
+                        sys.exc_info()[2])
         except Exception:
             # clean destination
             # todo: preferably we'd also rollback *any* changes to the database

openpype/plugins/publish/integrate_legacy.py

@@ -76,10 +76,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     order = pyblish.api.IntegratorOrder + 0.00001
     families = ["workfile",
                 "pointcache",
+                "pointcloud",
                 "proxyAbc",
                 "camera",
                 "animation",
                 "model",
+                "maxScene",
                 "mayaAscii",
                 "mayaScene",
                 "setdress",

openpype/plugins/publish/preintegrate_thumbnail_representation.py

@@ -60,6 +60,8 @@ class PreIntegrateThumbnails(pyblish.api.InstancePlugin):
         if not found_profile:
             return
 
+        thumbnail_repre.setdefault("tags", [])
+
         if not found_profile["integrate_thumbnail"]:
             if "delete" not in thumbnail_repre["tags"]:
                 thumbnail_repre["tags"].append("delete")