From e5733450e428f7f26e5bfe76fc9fe1e80b42b9f2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 May 2023 12:18:57 +0200 Subject: [PATCH] Global: plugins cleanup plugin will leave beauty rendered files (#4790) * OP-1066 - add expected files in Deadline into explicit cleanup Implicit cleanup doesn't work correctly, safest option is for DL submissions to mark only files that should be rendered to be deleted after successful publish. * OP-1066 - moved collecting of expected files into collector Parsing of json didn't have context implemented, it is easier to mark expected files in collector. * OP-4793 - delete full stagingDir Reviews might be extracted into staging dir, should be removed too. * Revert "OP-4793 - delete full stagingDir" This reverts commit 8b002191e1ad3b31a0cbe439ca1158946c43b049. * OP-1066 - added function to mark representation files to be cleaned up Should be applicable for all new representations, as reviews, thumbnails, to clean up their intermediate files. * OP-1066 - moved files to better file Cleaned up occurences where not necessary. * OP-1066 - removed unused import * OP-1066 - removed unnecessary setdefault * OP-1066 - removed unnecessary logging * OP-1066 - cleanup metadata json Try to cleanup parent folder if empty. --- openpype/pipeline/publish/lib.py | 19 +++++++++++++++++++ .../plugins/publish/collect_rendered_files.py | 6 ++++++ openpype/plugins/publish/extract_burnin.py | 3 +++ openpype/plugins/publish/extract_review.py | 3 +++ 4 files changed, 31 insertions(+) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index 63a856e326..b55f813b5e 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -847,3 +847,22 @@ def _validate_transient_template(project_name, template_name, anatomy): raise ValueError(("There is not set \"folder\" template in \"{}\" anatomy" # noqa " for project \"{}\"." ).format(template_name, project_name)) + + +def add_repre_files_for_cleanup(instance, repre): + """ Explicitly mark repre files to be deleted. + + Should be used on intermediate files (eg. review, thumbnails) to be + explicitly deleted. + """ + files = repre["files"] + staging_dir = repre.get("stagingDir") + if not staging_dir: + return + + if isinstance(files, str): + files = [files] + + for file_name in files: + expected_file = os.path.join(staging_dir, file_name) + instance.context.data["cleanupFullPaths"].append(expected_file) diff --git a/openpype/plugins/publish/collect_rendered_files.py b/openpype/plugins/publish/collect_rendered_files.py index 8f8d0a5eeb..6c8d1e9ca5 100644 --- a/openpype/plugins/publish/collect_rendered_files.py +++ b/openpype/plugins/publish/collect_rendered_files.py @@ -13,6 +13,7 @@ import json import pyblish.api from openpype.pipeline import legacy_io, KnownPublishError +from openpype.pipeline.publish.lib import add_repre_files_for_cleanup class CollectRenderedFiles(pyblish.api.ContextPlugin): @@ -89,6 +90,7 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin): # now we can just add instances from json file and we are done for instance_data in data.get("instances"): + self.log.info(" - processing instance for {}".format( instance_data.get("subset"))) instance = self._context.create_instance( @@ -107,6 +109,8 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin): self._fill_staging_dir(repre_data, anatomy) representations.append(repre_data) + add_repre_files_for_cleanup(instance, repre_data) + instance.data["representations"] = representations # add audio if in metadata data @@ -157,6 +161,8 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin): os.environ.update(session_data) session_is_set = True self._process_path(data, anatomy) + context.data["cleanupFullPaths"].append(path) + context.data["cleanupEmptyDirs"].append(os.path.dirname(path)) except Exception as e: self.log.error(e, exc_info=True) raise Exception("Error") from e diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 10b366dcd6..6a8ae958d2 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -19,6 +19,7 @@ from openpype.lib import ( should_convert_for_ffmpeg ) from openpype.lib.profiles_filtering import filter_profiles +from openpype.pipeline.publish.lib import add_repre_files_for_cleanup class ExtractBurnin(publish.Extractor): @@ -353,6 +354,8 @@ class ExtractBurnin(publish.Extractor): # Add new representation to instance instance.data["representations"].append(new_repre) + add_repre_files_for_cleanup(instance, new_repre) + # Cleanup temp staging dir after procesisng of output definitions if do_convert: temp_dir = repre["stagingDir"] diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index a68addda7d..fa58c03df1 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -24,6 +24,7 @@ from openpype.lib.transcoding import ( get_transcode_temp_directory, ) from openpype.pipeline.publish import KnownPublishError +from openpype.pipeline.publish.lib import add_repre_files_for_cleanup class ExtractReview(pyblish.api.InstancePlugin): @@ -425,6 +426,8 @@ class ExtractReview(pyblish.api.InstancePlugin): ) instance.data["representations"].append(new_repre) + add_repre_files_for_cleanup(instance, new_repre) + def input_is_sequence(self, repre): """Deduce from representation data if input is sequence.""" # TODO GLOBAL ISSUE - Find better way how to find out if input