Global: plugins cleanup plugin will leave beauty rendered files (#4790)

* OP-1066 - add expected files in Deadline into explicit cleanup

Implicit cleanup doesn't work correctly, safest option is for DL submissions to mark only files that should be rendered to be deleted after successful publish.

* OP-1066 - moved collecting of expected files into collector

Parsing of json didn't have context implemented, it is easier to mark expected files in collector.

* OP-4793 - delete full stagingDir

Reviews might be extracted into staging dir, should be removed too.

* Revert "OP-4793 - delete full stagingDir"

This reverts commit 8b002191e1ad3b31a0cbe439ca1158946c43b049.

* OP-1066 - added function to mark representation files to be cleaned up

Should be applicable for all new representations, as reviews, thumbnails, to clean up their intermediate  files.

* OP-1066 - moved files to better file

Cleaned up occurences where not necessary.

* OP-1066 - removed unused import

* OP-1066 - removed unnecessary setdefault

* OP-1066 - removed unnecessary logging

* OP-1066 - cleanup metadata json

Try to cleanup parent folder if empty.

openpype/pipeline/publish/lib.py

@@ -847,3 +847,22 @@ def _validate_transient_template(project_name, template_name, anatomy):
         raise ValueError(("There is not set \"folder\" template in \"{}\" anatomy"  # noqa
                              " for project \"{}\"."
                          ).format(template_name, project_name))
+
+
+def add_repre_files_for_cleanup(instance, repre):
+    """ Explicitly mark repre files to be deleted.
+
+    Should be used on intermediate files (eg. review, thumbnails) to be
+    explicitly deleted.
+    """
+    files = repre["files"]
+    staging_dir = repre.get("stagingDir")
+    if not staging_dir:
+        return
+
+    if isinstance(files, str):
+        files = [files]
+
+    for file_name in files:
+        expected_file = os.path.join(staging_dir, file_name)
+        instance.context.data["cleanupFullPaths"].append(expected_file)

openpype/plugins/publish/collect_rendered_files.py

@@ -13,6 +13,7 @@ import json
 import pyblish.api
 
 from openpype.pipeline import legacy_io, KnownPublishError
+from openpype.pipeline.publish.lib import add_repre_files_for_cleanup
 
 
 class CollectRenderedFiles(pyblish.api.ContextPlugin):
@@ -89,6 +90,7 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin):
 
         # now we can just add instances from json file and we are done
         for instance_data in data.get("instances"):
+
             self.log.info("  - processing instance for {}".format(
                 instance_data.get("subset")))
             instance = self._context.create_instance(
@@ -107,6 +109,8 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin):
                 self._fill_staging_dir(repre_data, anatomy)
                 representations.append(repre_data)
 
+                add_repre_files_for_cleanup(instance, repre_data)
+
             instance.data["representations"] = representations
 
             # add audio if in metadata data
@@ -157,6 +161,8 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin):
                     os.environ.update(session_data)
                     session_is_set = True
                 self._process_path(data, anatomy)
+                context.data["cleanupFullPaths"].append(path)
+                context.data["cleanupEmptyDirs"].append(os.path.dirname(path))
         except Exception as e:
             self.log.error(e, exc_info=True)
             raise Exception("Error") from e

openpype/plugins/publish/extract_burnin.py

@@ -19,6 +19,7 @@ from openpype.lib import (
     should_convert_for_ffmpeg
 )
 from openpype.lib.profiles_filtering import filter_profiles
+from openpype.pipeline.publish.lib import add_repre_files_for_cleanup
 
 
 class ExtractBurnin(publish.Extractor):
@@ -353,6 +354,8 @@ class ExtractBurnin(publish.Extractor):
                 # Add new representation to instance
                 instance.data["representations"].append(new_repre)
 
+                add_repre_files_for_cleanup(instance, new_repre)
+
             # Cleanup temp staging dir after procesisng of output definitions
             if do_convert:
                 temp_dir = repre["stagingDir"]

openpype/plugins/publish/extract_review.py

@@ -24,6 +24,7 @@ from openpype.lib.transcoding import (
     get_transcode_temp_directory,
 )
 from openpype.pipeline.publish import KnownPublishError
+from openpype.pipeline.publish.lib import add_repre_files_for_cleanup
 
 
 class ExtractReview(pyblish.api.InstancePlugin):
@@ -425,6 +426,8 @@ class ExtractReview(pyblish.api.InstancePlugin):
             )
             instance.data["representations"].append(new_repre)
 
+            add_repre_files_for_cleanup(instance, new_repre)
+
     def input_is_sequence(self, repre):
         """Deduce from representation data if input is sequence."""
         # TODO GLOBAL ISSUE - Find better way how to find out if input