From 10c0004c80ca5437f37a7729f3acfc31b2344f94 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 4 Apr 2024 15:41:58 +0800 Subject: [PATCH 01/11] delete old versions loader action does not work --- client/ayon_core/pipeline/load/plugins.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/client/ayon_core/pipeline/load/plugins.py b/client/ayon_core/pipeline/load/plugins.py index 064af4ddc1..29edc73b57 100644 --- a/client/ayon_core/pipeline/load/plugins.py +++ b/client/ayon_core/pipeline/load/plugins.py @@ -220,19 +220,6 @@ class LoaderPlugin(list): """ return cls.options or [] - @property - def fname(self): - """Backwards compatibility with deprecation warning""" - - self.log.warning(( - "DEPRECATION WARNING: Source - Loader plugin {}." - " The 'fname' property on the Loader plugin will be removed in" - " future versions of OpenPype. Planned version to drop the support" - " is 3.16.6 or 3.17.0." - ).format(self.__class__.__name__)) - if hasattr(self, "_fname"): - return self._fname - class ProductLoaderPlugin(LoaderPlugin): """Load product into host application From 6c8b4e33bce15361690d446f0b204040abc42c04 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 4 Apr 2024 16:23:16 +0800 Subject: [PATCH 02/11] should be removing delete_old_versions.py from the load plugins --- client/ayon_core/pipeline/load/plugins.py | 13 + .../plugins/load/delete_old_versions.py | 501 ------------------ 2 files changed, 13 insertions(+), 501 deletions(-) delete mode 100644 client/ayon_core/plugins/load/delete_old_versions.py diff --git a/client/ayon_core/pipeline/load/plugins.py b/client/ayon_core/pipeline/load/plugins.py index 29edc73b57..064af4ddc1 100644 --- a/client/ayon_core/pipeline/load/plugins.py +++ b/client/ayon_core/pipeline/load/plugins.py @@ -220,6 +220,19 @@ class LoaderPlugin(list): """ return cls.options or [] + @property + def fname(self): + """Backwards compatibility with deprecation warning""" + + self.log.warning(( + "DEPRECATION WARNING: Source - Loader plugin {}." + " The 'fname' property on the Loader plugin will be removed in" + " future versions of OpenPype. Planned version to drop the support" + " is 3.16.6 or 3.17.0." + ).format(self.__class__.__name__)) + if hasattr(self, "_fname"): + return self._fname + class ProductLoaderPlugin(LoaderPlugin): """Load product into host application diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py deleted file mode 100644 index 04873d8b5c..0000000000 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ /dev/null @@ -1,501 +0,0 @@ -# TODO This plugin is not converted for AYON -# -# import collections -# import os -# import uuid -# -# import clique -# import ayon_api -# from pymongo import UpdateOne -# import qargparse -# from qtpy import QtWidgets, QtCore -# -# from ayon_core import style -# from ayon_core.addon import AddonsManager -# from ayon_core.lib import format_file_size -# from ayon_core.pipeline import load, Anatomy -# from ayon_core.pipeline.load import ( -# get_representation_path_with_anatomy, -# InvalidRepresentationContext, -# ) -# -# -# class DeleteOldVersions(load.ProductLoaderPlugin): -# """Deletes specific number of old version""" -# -# is_multiple_contexts_compatible = True -# sequence_splitter = "__sequence_splitter__" -# -# representations = ["*"] -# product_types = {"*"} -# tool_names = ["library_loader"] -# -# label = "Delete Old Versions" -# order = 35 -# icon = "trash" -# color = "#d8d8d8" -# -# options = [ -# qargparse.Integer( -# "versions_to_keep", default=2, min=0, help="Versions to keep:" -# ), -# qargparse.Boolean( -# "remove_publish_folder", help="Remove publish folder:" -# ) -# ] -# -# def delete_whole_dir_paths(self, dir_paths, delete=True): -# size = 0 -# -# for dir_path in dir_paths: -# # Delete all files and fodlers in dir path -# for root, dirs, files in os.walk(dir_path, topdown=False): -# for name in files: -# file_path = os.path.join(root, name) -# size += os.path.getsize(file_path) -# if delete: -# os.remove(file_path) -# self.log.debug("Removed file: {}".format(file_path)) -# -# for name in dirs: -# if delete: -# os.rmdir(os.path.join(root, name)) -# -# if not delete: -# continue -# -# # Delete even the folder and it's parents folders if they are empty -# while True: -# if not os.path.exists(dir_path): -# dir_path = os.path.dirname(dir_path) -# continue -# -# if len(os.listdir(dir_path)) != 0: -# break -# -# os.rmdir(os.path.join(dir_path)) -# -# return size -# -# def path_from_representation(self, representation, anatomy): -# try: -# context = representation["context"] -# except KeyError: -# return (None, None) -# -# try: -# path = get_representation_path_with_anatomy( -# representation, anatomy -# ) -# except InvalidRepresentationContext: -# return (None, None) -# -# sequence_path = None -# if "frame" in context: -# context["frame"] = self.sequence_splitter -# sequence_path = get_representation_path_with_anatomy( -# representation, anatomy -# ) -# -# if sequence_path: -# sequence_path = sequence_path.normalized() -# -# return (path.normalized(), sequence_path) -# -# def delete_only_repre_files(self, dir_paths, file_paths, delete=True): -# size = 0 -# -# for dir_id, dir_path in dir_paths.items(): -# dir_files = os.listdir(dir_path) -# collections, remainders = clique.assemble(dir_files) -# for file_path, seq_path in file_paths[dir_id]: -# file_path_base = os.path.split(file_path)[1] -# # Just remove file if `frame` key was not in context or -# # filled path is in remainders (single file sequence) -# if not seq_path or file_path_base in remainders: -# if not os.path.exists(file_path): -# self.log.debug( -# "File was not found: {}".format(file_path) -# ) -# continue -# -# size += os.path.getsize(file_path) -# -# if delete: -# os.remove(file_path) -# self.log.debug("Removed file: {}".format(file_path)) -# -# if file_path_base in remainders: -# remainders.remove(file_path_base) -# continue -# -# seq_path_base = os.path.split(seq_path)[1] -# head, tail = seq_path_base.split(self.sequence_splitter) -# -# final_col = None -# for collection in collections: -# if head != collection.head or tail != collection.tail: -# continue -# final_col = collection -# break -# -# if final_col is not None: -# # Fill full path to head -# final_col.head = os.path.join(dir_path, final_col.head) -# for _file_path in final_col: -# if os.path.exists(_file_path): -# -# size += os.path.getsize(_file_path) -# -# if delete: -# os.remove(_file_path) -# self.log.debug( -# "Removed file: {}".format(_file_path) -# ) -# -# _seq_path = final_col.format("{head}{padding}{tail}") -# self.log.debug("Removed files: {}".format(_seq_path)) -# collections.remove(final_col) -# -# elif os.path.exists(file_path): -# size += os.path.getsize(file_path) -# -# if delete: -# os.remove(file_path) -# self.log.debug("Removed file: {}".format(file_path)) -# else: -# self.log.debug( -# "File was not found: {}".format(file_path) -# ) -# -# # Delete as much as possible parent folders -# if not delete: -# return size -# -# for dir_path in dir_paths.values(): -# while True: -# if not os.path.exists(dir_path): -# dir_path = os.path.dirname(dir_path) -# continue -# -# if len(os.listdir(dir_path)) != 0: -# break -# -# self.log.debug("Removed folder: {}".format(dir_path)) -# os.rmdir(dir_path) -# -# return size -# -# def message(self, text): -# msgBox = QtWidgets.QMessageBox() -# msgBox.setText(text) -# msgBox.setStyleSheet(style.load_stylesheet()) -# msgBox.setWindowFlags( -# msgBox.windowFlags() | QtCore.Qt.FramelessWindowHint -# ) -# msgBox.exec_() -# -# def get_data(self, context, versions_count): -# product_entity = context["product"] -# folder_entity = context["folder"] -# project_name = context["project"]["name"] -# anatomy = Anatomy(project_name) -# -# versions = list(ayon_api.get_versions( -# project_name, product_ids=[product_entity["id"]] -# )) -# -# versions_by_parent = collections.defaultdict(list) -# for ent in versions: -# versions_by_parent[ent["productId"]].append(ent) -# -# def sort_func(ent): -# return int(ent["version"]) -# -# all_last_versions = [] -# for _parent_id, _versions in versions_by_parent.items(): -# for idx, version in enumerate( -# sorted(_versions, key=sort_func, reverse=True) -# ): -# if idx >= versions_count: -# break -# all_last_versions.append(version) -# -# self.log.debug("Collected versions ({})".format(len(versions))) -# -# # Filter latest versions -# for version in all_last_versions: -# versions.remove(version) -# -# # Update versions_by_parent without filtered versions -# versions_by_parent = collections.defaultdict(list) -# for ent in versions: -# versions_by_parent[ent["productId"]].append(ent) -# -# # Filter already deleted versions -# versions_to_pop = [] -# for version in versions: -# version_tags = version["data"].get("tags") -# if version_tags and "deleted" in version_tags: -# versions_to_pop.append(version) -# -# for version in versions_to_pop: -# msg = "Folder: \"{}\" | Product: \"{}\" | Version: \"{}\"".format( -# folder_entity["path"], -# product_entity["name"], -# version["version"] -# ) -# self.log.debug(( -# "Skipping version. Already tagged as `deleted`. < {} >" -# ).format(msg)) -# versions.remove(version) -# -# version_ids = [ent["id"] for ent in versions] -# -# self.log.debug( -# "Filtered versions to delete ({})".format(len(version_ids)) -# ) -# -# if not version_ids: -# msg = "Skipping processing. Nothing to delete on {}/{}".format( -# folder_entity["path"], product_entity["name"] -# ) -# self.log.info(msg) -# print(msg) -# return -# -# repres = list(ayon_api.get_representations( -# project_name, version_ids=version_ids -# )) -# -# self.log.debug( -# "Collected representations to remove ({})".format(len(repres)) -# ) -# -# dir_paths = {} -# file_paths_by_dir = collections.defaultdict(list) -# for repre in repres: -# file_path, seq_path = self.path_from_representation( -# repre, anatomy -# ) -# if file_path is None: -# self.log.debug(( -# "Could not format path for represenation \"{}\"" -# ).format(str(repre))) -# continue -# -# dir_path = os.path.dirname(file_path) -# dir_id = None -# for _dir_id, _dir_path in dir_paths.items(): -# if _dir_path == dir_path: -# dir_id = _dir_id -# break -# -# if dir_id is None: -# dir_id = uuid.uuid4() -# dir_paths[dir_id] = dir_path -# -# file_paths_by_dir[dir_id].append([file_path, seq_path]) -# -# dir_ids_to_pop = [] -# for dir_id, dir_path in dir_paths.items(): -# if os.path.exists(dir_path): -# continue -# -# dir_ids_to_pop.append(dir_id) -# -# # Pop dirs from both dictionaries -# for dir_id in dir_ids_to_pop: -# dir_paths.pop(dir_id) -# paths = file_paths_by_dir.pop(dir_id) -# # TODO report of missing directories? -# paths_msg = ", ".join([ -# "'{}'".format(path[0].replace("\\", "/")) for path in paths -# ]) -# self.log.debug(( -# "Folder does not exist. Deleting it's files skipped: {}" -# ).format(paths_msg)) -# -# return { -# "dir_paths": dir_paths, -# "file_paths_by_dir": file_paths_by_dir, -# "versions": versions, -# "folder": folder_entity, -# "product": product_entity, -# "archive_product": versions_count == 0 -# } -# -# def main(self, project_name, data, remove_publish_folder): -# # Size of files. -# size = 0 -# if not data: -# return size -# -# if remove_publish_folder: -# size = self.delete_whole_dir_paths(data["dir_paths"].values()) -# else: -# size = self.delete_only_repre_files( -# data["dir_paths"], data["file_paths_by_dir"] -# ) -# -# mongo_changes_bulk = [] -# for version in data["versions"]: -# orig_version_tags = version["data"].get("tags") or [] -# version_tags = [tag for tag in orig_version_tags] -# if "deleted" not in version_tags: -# version_tags.append("deleted") -# -# if version_tags == orig_version_tags: -# continue -# -# update_query = {"id": version["id"]} -# update_data = {"$set": {"data.tags": version_tags}} -# mongo_changes_bulk.append(UpdateOne(update_query, update_data)) -# -# if data["archive_product"]: -# mongo_changes_bulk.append(UpdateOne( -# { -# "id": data["product"]["id"], -# "type": "subset" -# }, -# {"$set": {"type": "archived_subset"}} -# )) -# -# if mongo_changes_bulk: -# dbcon = AvalonMongoDB() -# dbcon.Session["AYON_PROJECT_NAME"] = project_name -# dbcon.install() -# dbcon.bulk_write(mongo_changes_bulk) -# dbcon.uninstall() -# -# self._ftrack_delete_versions(data) -# -# return size -# -# def _ftrack_delete_versions(self, data): -# """Delete version on ftrack. -# -# Handling of ftrack logic in this plugin is not ideal. But in OP3 it is -# almost impossible to solve the issue other way. -# -# Note: -# Asset versions on ftrack are not deleted but marked as -# "not published" which cause that they're invisible. -# -# Args: -# data (dict): Data sent to product loader with full context. -# """ -# -# # First check for ftrack id on folder entity -# # - skip if ther is none -# ftrack_id = data["folder"]["attrib"].get("ftrackId") -# if not ftrack_id: -# self.log.info(( -# "Folder does not have filled ftrack id. Skipped delete" -# " of ftrack version." -# )) -# return -# -# # Check if ftrack module is enabled -# addons_manager = AddonsManager() -# ftrack_addon = addons_manager.get("ftrack") -# if not ftrack_addon or not ftrack_addon.enabled: -# return -# -# import ftrack_api -# -# session = ftrack_api.Session() -# product_name = data["product"]["name"] -# versions = { -# '"{}"'.format(version_doc["name"]) -# for version_doc in data["versions"] -# } -# asset_versions = session.query( -# ( -# "select id, is_published from AssetVersion where" -# " asset.parent.id is \"{}\"" -# " and asset.name is \"{}\"" -# " and version in ({})" -# ).format( -# ftrack_id, -# product_name, -# ",".join(versions) -# ) -# ).all() -# -# # Set attribute `is_published` to `False` on ftrack AssetVersions -# for asset_version in asset_versions: -# asset_version["is_published"] = False -# -# try: -# session.commit() -# -# except Exception: -# msg = ( -# "Could not set `is_published` attribute to `False`" -# " for selected AssetVersions." -# ) -# self.log.error(msg) -# self.message(msg) -# -# def load(self, contexts, name=None, namespace=None, options=None): -# try: -# size = 0 -# for count, context in enumerate(contexts): -# versions_to_keep = 2 -# remove_publish_folder = False -# if options: -# versions_to_keep = options.get( -# "versions_to_keep", versions_to_keep -# ) -# remove_publish_folder = options.get( -# "remove_publish_folder", remove_publish_folder -# ) -# -# data = self.get_data(context, versions_to_keep) -# if not data: -# continue -# -# project_name = context["project"]["name"] -# size += self.main(project_name, data, remove_publish_folder) -# print("Progressing {}/{}".format(count + 1, len(contexts))) -# -# msg = "Total size of files: {}".format(format_file_size(size)) -# self.log.info(msg) -# self.message(msg) -# -# except Exception: -# self.log.error("Failed to delete versions.", exc_info=True) -# -# -# class CalculateOldVersions(DeleteOldVersions): -# """Calculate file size of old versions""" -# label = "Calculate Old Versions" -# order = 30 -# tool_names = ["library_loader"] -# -# options = [ -# qargparse.Integer( -# "versions_to_keep", default=2, min=0, help="Versions to keep:" -# ), -# qargparse.Boolean( -# "remove_publish_folder", help="Remove publish folder:" -# ) -# ] -# -# def main(self, project_name, data, remove_publish_folder): -# size = 0 -# -# if not data: -# return size -# -# if remove_publish_folder: -# size = self.delete_whole_dir_paths( -# data["dir_paths"].values(), delete=False -# ) -# else: -# size = self.delete_only_repre_files( -# data["dir_paths"], data["file_paths_by_dir"], delete=False -# ) -# -# return size From be694cba5619f8cb7ac66cb378435614f52a98d8 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Thu, 4 Apr 2024 17:43:24 +0800 Subject: [PATCH 03/11] remove the old versions loader action should be working --- .../plugins/load/delete_old_versions.py | 478 ++++++++++++++++++ 1 file changed, 478 insertions(+) create mode 100644 client/ayon_core/plugins/load/delete_old_versions.py diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py new file mode 100644 index 0000000000..4f591a503a --- /dev/null +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -0,0 +1,478 @@ +import collections +import os +import uuid + +import clique +import ayon_api +import qargparse +from qtpy import QtWidgets, QtCore + +from ayon_core import style +from ayon_core.addon import AddonsManager +from ayon_core.lib import format_file_size +from ayon_core.pipeline import load, Anatomy +from ayon_core.pipeline.load import ( + get_representation_path_with_anatomy, + InvalidRepresentationContext, +) + + +class DeleteOldVersions(load.ProductLoaderPlugin): + """Deletes specific number of old version""" + + is_multiple_contexts_compatible = True + sequence_splitter = "__sequence_splitter__" + + representations = ["*"] + product_types = {"*"} + tool_names = ["library_loader"] + + label = "Delete Old Versions" + order = 35 + icon = "trash" + color = "#d8d8d8" + + options = [ + qargparse.Integer( + "versions_to_keep", default=2, min=0, help="Versions to keep:" + ), + qargparse.Boolean( + "remove_publish_folder", help="Remove publish folder:" + ) + ] + + def delete_whole_dir_paths(self, dir_paths, delete=True): + size = 0 + + for dir_path in dir_paths: + # Delete all files and fodlers in dir path + for root, dirs, files in os.walk(dir_path, topdown=False): + for name in files: + file_path = os.path.join(root, name) + size += os.path.getsize(file_path) + if delete: + os.remove(file_path) + self.log.debug("Removed file: {}".format(file_path)) + + for name in dirs: + if delete: + os.rmdir(os.path.join(root, name)) + + if not delete: + continue + + # Delete even the folder and it's parents folders if they are empty + while True: + if not os.path.exists(dir_path): + dir_path = os.path.dirname(dir_path) + continue + + if len(os.listdir(dir_path)) != 0: + break + + os.rmdir(os.path.join(dir_path)) + + return size + + def path_from_representation(self, representation, anatomy): + try: + context = representation["context"] + except KeyError: + return (None, None) + + try: + path = get_representation_path_with_anatomy( + representation, anatomy + ) + except InvalidRepresentationContext: + return (None, None) + + sequence_path = None + if "frame" in context: + context["frame"] = self.sequence_splitter + sequence_path = get_representation_path_with_anatomy( + representation, anatomy + ) + + if sequence_path: + sequence_path = sequence_path.normalized() + + return (path.normalized(), sequence_path) + + def delete_only_repre_files(self, dir_paths, file_paths, delete=True): + size = 0 + + for dir_id, dir_path in dir_paths.items(): + dir_files = os.listdir(dir_path) + collections, remainders = clique.assemble(dir_files) + for file_path, seq_path in file_paths[dir_id]: + file_path_base = os.path.split(file_path)[1] + # Just remove file if `frame` key was not in context or + # filled path is in remainders (single file sequence) + if not seq_path or file_path_base in remainders: + if not os.path.exists(file_path): + self.log.debug( + "File was not found: {}".format(file_path) + ) + continue + + size += os.path.getsize(file_path) + + if delete: + os.remove(file_path) + self.log.debug("Removed file: {}".format(file_path)) + + if file_path_base in remainders: + remainders.remove(file_path_base) + continue + + seq_path_base = os.path.split(seq_path)[1] + head, tail = seq_path_base.split(self.sequence_splitter) + + final_col = None + for collection in collections: + if head != collection.head or tail != collection.tail: + continue + final_col = collection + break + + if final_col is not None: + # Fill full path to head + final_col.head = os.path.join(dir_path, final_col.head) + for _file_path in final_col: + if os.path.exists(_file_path): + + size += os.path.getsize(_file_path) + + if delete: + os.remove(_file_path) + self.log.debug( + "Removed file: {}".format(_file_path) + ) + + _seq_path = final_col.format("{head}{padding}{tail}") + self.log.debug("Removed files: {}".format(_seq_path)) + collections.remove(final_col) + + elif os.path.exists(file_path): + size += os.path.getsize(file_path) + + if delete: + os.remove(file_path) + self.log.debug("Removed file: {}".format(file_path)) + else: + self.log.debug( + "File was not found: {}".format(file_path) + ) + + # Delete as much as possible parent folders + if not delete: + return size + + for dir_path in dir_paths.values(): + while True: + if not os.path.exists(dir_path): + dir_path = os.path.dirname(dir_path) + continue + + if len(os.listdir(dir_path)) != 0: + break + + self.log.debug("Removed folder: {}".format(dir_path)) + os.rmdir(dir_path) + + return size + + def message(self, text): + msgBox = QtWidgets.QMessageBox() + msgBox.setText(text) + msgBox.setStyleSheet(style.load_stylesheet()) + msgBox.setWindowFlags( + msgBox.windowFlags() | QtCore.Qt.FramelessWindowHint + ) + msgBox.exec_() + + def get_data(self, context, versions_count): + product_entity = context["product"] + folder_entity = context["folder"] + project_name = context["project"]["name"] + anatomy = Anatomy(project_name) + + versions = list(ayon_api.get_versions( + project_name, product_ids=[product_entity["id"]] + )) + self.log.debug( + "Version Number ({})".format(len(versions)) + ) + versions_by_parent = collections.defaultdict(list) + for ent in versions: + versions_by_parent[ent["productId"]].append(ent) + + def sort_func(ent): + return int(ent["version"]) + + all_last_versions = [] + for _parent_id, _versions in versions_by_parent.items(): + for idx, version in enumerate( + sorted(_versions, key=sort_func, reverse=True) + ): + if idx >= versions_count: + break + all_last_versions.append(version) + + self.log.debug("Collected versions ({})".format(len(versions))) + + # Filter latest versions + for version in all_last_versions: + versions.remove(version) + + # Update versions_by_parent without filtered versions + versions_by_parent = collections.defaultdict(list) + for ent in versions: + versions_by_parent[ent["productId"]].append(ent) + + # Filter already deleted versions + versions_to_pop = [] + for version in versions: + version_tags = version["data"].get("tags") + if version_tags and "deleted" in version_tags: + versions_to_pop.append(version) + + for version in versions_to_pop: + msg = "Folder: \"{}\" | Product: \"{}\" | Version: \"{}\"".format( + folder_entity["path"], + product_entity["name"], + version["version"] + ) + self.log.debug(( + "Skipping version. Already tagged as `deleted`. < {} >" + ).format(msg)) + versions.remove(version) + + version_ids = [ent["id"] for ent in versions] + + self.log.debug( + "Filtered versions to delete ({})".format(len(version_ids)) + ) + + if not version_ids: + msg = "Skipping processing. Nothing to delete on {}/{}".format( + folder_entity["path"], product_entity["name"] + ) + self.log.info(msg) + print(msg) + return + + repres = list(ayon_api.get_representations( + project_name, version_ids=version_ids + )) + + self.log.debug( + "Collected representations to remove ({})".format(len(repres)) + ) + + dir_paths = {} + file_paths_by_dir = collections.defaultdict(list) + for repre in repres: + file_path, seq_path = self.path_from_representation( + repre, anatomy + ) + if file_path is None: + self.log.debug(( + "Could not format path for represenation \"{}\"" + ).format(str(repre))) + continue + + dir_path = os.path.dirname(file_path) + dir_id = None + for _dir_id, _dir_path in dir_paths.items(): + if _dir_path == dir_path: + dir_id = _dir_id + break + + if dir_id is None: + dir_id = uuid.uuid4() + dir_paths[dir_id] = dir_path + + file_paths_by_dir[dir_id].append([file_path, seq_path]) + + dir_ids_to_pop = [] + for dir_id, dir_path in dir_paths.items(): + if os.path.exists(dir_path): + continue + + dir_ids_to_pop.append(dir_id) + + # Pop dirs from both dictionaries + for dir_id in dir_ids_to_pop: + dir_paths.pop(dir_id) + paths = file_paths_by_dir.pop(dir_id) + # TODO report of missing directories? + paths_msg = ", ".join([ + "'{}'".format(path[0].replace("\\", "/")) for path in paths + ]) + self.log.debug(( + "Folder does not exist. Deleting it's files skipped: {}" + ).format(paths_msg)) + + return { + "dir_paths": dir_paths, + "file_paths_by_dir": file_paths_by_dir, + "versions": versions, + "folder": folder_entity, + "product": product_entity, + "archive_product": versions_count == 0 + } + + def main(self, data, remove_publish_folder): + # Size of files. + size = 0 + if not data: + return size + + if remove_publish_folder: + size = self.delete_whole_dir_paths(data["dir_paths"].values()) + else: + size = self.delete_only_repre_files( + data["dir_paths"], data["file_paths_by_dir"] + ) + + for version in data["versions"]: + orig_version_tags = version["data"].get("tags") or [] + version_tags = [tag for tag in orig_version_tags] + if "deleted" not in version_tags: + version_tags.append("deleted") + + if version_tags == orig_version_tags: + continue + + self._ftrack_delete_versions(data) + + return size + + def _ftrack_delete_versions(self, data): + """Delete version on ftrack. + + Handling of ftrack logic in this plugin is not ideal. But in OP3 it is + almost impossible to solve the issue other way. + + Note: + Asset versions on ftrack are not deleted but marked as + "not published" which cause that they're invisible. + + Args: + data (dict): Data sent to product loader with full context. + """ + + # First check for ftrack id on folder entity + # - skip if ther is none + ftrack_id = data["folder"]["attrib"].get("ftrackId") + if not ftrack_id: + self.log.info(( + "Folder does not have filled ftrack id. Skipped delete" + " of ftrack version." + )) + return + + # Check if ftrack module is enabled + addons_manager = AddonsManager() + ftrack_addon = addons_manager.get("ftrack") + if not ftrack_addon or not ftrack_addon.enabled: + return + + import ftrack_api + + session = ftrack_api.Session() + product_name = data["product"]["name"] + versions = { + '"{}"'.format(version_doc["name"]) + for version_doc in data["versions"] + } + asset_versions = session.query( + ( + "select id, is_published from AssetVersion where" + " asset.parent.id is \"{}\"" + " and asset.name is \"{}\"" + " and version in ({})" + ).format( + ftrack_id, + product_name, + ",".join(versions) + ) + ).all() + + # Set attribute `is_published` to `False` on ftrack AssetVersions + for asset_version in asset_versions: + asset_version["is_published"] = False + + try: + session.commit() + + except Exception: + msg = ( + "Could not set `is_published` attribute to `False`" + " for selected AssetVersions." + ) + self.log.error(msg) + self.message(msg) + + def load(self, contexts, name=None, namespace=None, options=None): + try: + size = 0 + for count, context in enumerate(contexts): + versions_to_keep = 2 + remove_publish_folder = False + if options: + versions_to_keep = options.get( + "versions_to_keep", versions_to_keep + ) + remove_publish_folder = options.get( + "remove_publish_folder", remove_publish_folder + ) + + data = self.get_data(context, versions_to_keep) + if not data: + continue + + size += self.main(data, remove_publish_folder) + print("Progressing {}/{}".format(count + 1, len(contexts))) + + msg = "Total size of files: {}".format(format_file_size(size)) + self.log.info(msg) + self.message(msg) + + except Exception: + self.log.error("Failed to delete versions.", exc_info=True) + + +class CalculateOldVersions(DeleteOldVersions): + """Calculate file size of old versions""" + label = "Calculate Old Versions" + order = 30 + tool_names = ["library_loader"] + + options = [ + qargparse.Integer( + "versions_to_keep", default=2, min=0, help="Versions to keep:" + ), + qargparse.Boolean( + "remove_publish_folder", help="Remove publish folder:" + ) + ] + + def main(self, data, remove_publish_folder): + size = 0 + + if not data: + return size + + if remove_publish_folder: + size = self.delete_whole_dir_paths( + data["dir_paths"].values(), delete=False + ) + else: + size = self.delete_only_repre_files( + data["dir_paths"], data["file_paths_by_dir"], delete=False + ) + + return size From 914367dba226b7424ef9f149e1faf68e44b82a25 Mon Sep 17 00:00:00 2001 From: Kayla Man <64118225+moonyuet@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:54:01 +0800 Subject: [PATCH 04/11] Update client/ayon_core/plugins/load/delete_old_versions.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/plugins/load/delete_old_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 4f591a503a..79958af447 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -196,7 +196,7 @@ class DeleteOldVersions(load.ProductLoaderPlugin): product_entity = context["product"] folder_entity = context["folder"] project_name = context["project"]["name"] - anatomy = Anatomy(project_name) + anatomy = Anatomy(project_name, project_entity=context["project"]) versions = list(ayon_api.get_versions( project_name, product_ids=[product_entity["id"]] From 03e7a85e204248012271a995ec33d3d11b528a62 Mon Sep 17 00:00:00 2001 From: Kayla Man <64118225+moonyuet@users.noreply.github.com> Date: Tue, 9 Apr 2024 16:54:12 +0800 Subject: [PATCH 05/11] Update client/ayon_core/plugins/load/delete_old_versions.py Co-authored-by: Roy Nieterau --- client/ayon_core/plugins/load/delete_old_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 79958af447..28e76b73d6 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -312,7 +312,7 @@ class DeleteOldVersions(load.ProductLoaderPlugin): "'{}'".format(path[0].replace("\\", "/")) for path in paths ]) self.log.debug(( - "Folder does not exist. Deleting it's files skipped: {}" + "Folder does not exist. Deleting its files skipped: {}" ).format(paths_msg)) return { From f6480281b40f8132446e09dea337e1786dee9efe Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 9 Apr 2024 17:40:06 +0800 Subject: [PATCH 06/11] remove ftrack-related functions --- .../plugins/load/delete_old_versions.py | 69 ------------------- 1 file changed, 69 deletions(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 28e76b73d6..4e2747e1dc 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -8,7 +8,6 @@ import qargparse from qtpy import QtWidgets, QtCore from ayon_core import style -from ayon_core.addon import AddonsManager from ayon_core.lib import format_file_size from ayon_core.pipeline import load, Anatomy from ayon_core.pipeline.load import ( @@ -346,76 +345,8 @@ class DeleteOldVersions(load.ProductLoaderPlugin): if version_tags == orig_version_tags: continue - self._ftrack_delete_versions(data) - return size - def _ftrack_delete_versions(self, data): - """Delete version on ftrack. - - Handling of ftrack logic in this plugin is not ideal. But in OP3 it is - almost impossible to solve the issue other way. - - Note: - Asset versions on ftrack are not deleted but marked as - "not published" which cause that they're invisible. - - Args: - data (dict): Data sent to product loader with full context. - """ - - # First check for ftrack id on folder entity - # - skip if ther is none - ftrack_id = data["folder"]["attrib"].get("ftrackId") - if not ftrack_id: - self.log.info(( - "Folder does not have filled ftrack id. Skipped delete" - " of ftrack version." - )) - return - - # Check if ftrack module is enabled - addons_manager = AddonsManager() - ftrack_addon = addons_manager.get("ftrack") - if not ftrack_addon or not ftrack_addon.enabled: - return - - import ftrack_api - - session = ftrack_api.Session() - product_name = data["product"]["name"] - versions = { - '"{}"'.format(version_doc["name"]) - for version_doc in data["versions"] - } - asset_versions = session.query( - ( - "select id, is_published from AssetVersion where" - " asset.parent.id is \"{}\"" - " and asset.name is \"{}\"" - " and version in ({})" - ).format( - ftrack_id, - product_name, - ",".join(versions) - ) - ).all() - - # Set attribute `is_published` to `False` on ftrack AssetVersions - for asset_version in asset_versions: - asset_version["is_published"] = False - - try: - session.commit() - - except Exception: - msg = ( - "Could not set `is_published` attribute to `False`" - " for selected AssetVersions." - ) - self.log.error(msg) - self.message(msg) - def load(self, contexts, name=None, namespace=None, options=None): try: size = 0 From 043b528155450ca81dba41ca8d7a49667f6f282f Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 9 Apr 2024 18:43:59 +0800 Subject: [PATCH 07/11] uses OperationsSessions from ayon_api to delete version version_tags --- .../plugins/load/delete_old_versions.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 4e2747e1dc..ee116a71cf 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -4,6 +4,7 @@ import uuid import clique import ayon_api +from ayon_api.operations import OperationsSession import qargparse from qtpy import QtWidgets, QtCore @@ -231,20 +232,16 @@ class DeleteOldVersions(load.ProductLoaderPlugin): versions_by_parent[ent["productId"]].append(ent) # Filter already deleted versions - versions_to_pop = [] for version in versions: - version_tags = version["data"].get("tags") - if version_tags and "deleted" in version_tags: - versions_to_pop.append(version) - - for version in versions_to_pop: + if version["active"] or "deleted" in version["tags"]: + continue msg = "Folder: \"{}\" | Product: \"{}\" | Version: \"{}\"".format( folder_entity["path"], product_entity["name"], version["version"] ) self.log.debug(( - "Skipping version. Already tagged as `deleted`. < {} >" + "Skipping version. Already tagged as inactive. < {} >" ).format(msg)) versions.remove(version) @@ -323,7 +320,7 @@ class DeleteOldVersions(load.ProductLoaderPlugin): "archive_product": versions_count == 0 } - def main(self, data, remove_publish_folder): + def main(self, project_name, data, remove_publish_folder): # Size of files. size = 0 if not data: @@ -336,14 +333,25 @@ class DeleteOldVersions(load.ProductLoaderPlugin): data["dir_paths"], data["file_paths_by_dir"] ) + op_session = OperationsSession() for version in data["versions"]: - orig_version_tags = version["data"].get("tags") or [] - version_tags = [tag for tag in orig_version_tags] + orig_version_tags = version["tags"] + version_tags = list(orig_version_tags) + changes = {} if "deleted" not in version_tags: version_tags.append("deleted") + changes["tags"] = version_tags - if version_tags == orig_version_tags: + if version["active"]: + changes["active"] = False + + if not changes: continue + op_session.update_entity( + project_name, "version", version["id"], changes + ) + + op_session.commit() return size @@ -364,8 +372,8 @@ class DeleteOldVersions(load.ProductLoaderPlugin): data = self.get_data(context, versions_to_keep) if not data: continue - - size += self.main(data, remove_publish_folder) + project_name = context["project"]["name"] + size += self.main(project_name, data, remove_publish_folder) print("Progressing {}/{}".format(count + 1, len(contexts))) msg = "Total size of files: {}".format(format_file_size(size)) From c8f4f3681ac0290b13debe1283bdf1e47426d3c1 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Tue, 9 Apr 2024 18:45:23 +0800 Subject: [PATCH 08/11] make sure to skip hero version --- client/ayon_core/plugins/load/delete_old_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index ee116a71cf..bfbccce33d 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -199,7 +199,10 @@ class DeleteOldVersions(load.ProductLoaderPlugin): anatomy = Anatomy(project_name, project_entity=context["project"]) versions = list(ayon_api.get_versions( - project_name, product_ids=[product_entity["id"]] + project_name, + product_ids=[product_entity["id"]], + active=None, + hero=False )) self.log.debug( "Version Number ({})".format(len(versions)) From cf5d7d9a7d12c65f723cfbd044d9d68c01023a72 Mon Sep 17 00:00:00 2001 From: Kayla Man <64118225+moonyuet@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:52:34 +0800 Subject: [PATCH 09/11] Update client/ayon_core/plugins/load/delete_old_versions.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/plugins/load/delete_old_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index bfbccce33d..78603104d8 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -236,7 +236,7 @@ class DeleteOldVersions(load.ProductLoaderPlugin): # Filter already deleted versions for version in versions: - if version["active"] or "deleted" in version["tags"]: + if "deleted" in version["tags"]: continue msg = "Folder: \"{}\" | Product: \"{}\" | Version: \"{}\"".format( folder_entity["path"], From 364dee88f1a99e3410b6cc0d31566ceb25485b67 Mon Sep 17 00:00:00 2001 From: moonyuet Date: Wed, 10 Apr 2024 22:03:37 +0800 Subject: [PATCH 10/11] add the version tags if there is not one --- client/ayon_core/plugins/load/delete_old_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index 78603104d8..fd331ec14b 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -198,11 +198,14 @@ class DeleteOldVersions(load.ProductLoaderPlugin): project_name = context["project"]["name"] anatomy = Anatomy(project_name, project_entity=context["project"]) + version_fields = ayon_api.get_default_fields_for_type("version") + version_fields.add("tags") versions = list(ayon_api.get_versions( project_name, product_ids=[product_entity["id"]], active=None, - hero=False + hero=False, + fields=version_fields )) self.log.debug( "Version Number ({})".format(len(versions)) From 95a69a1d8d994c5345c78d1354e32a333983eef2 Mon Sep 17 00:00:00 2001 From: Kayla Man Date: Wed, 17 Apr 2024 13:31:13 +0800 Subject: [PATCH 11/11] make sure deleting old version should remove the 'right' folder --- client/ayon_core/plugins/load/delete_old_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/plugins/load/delete_old_versions.py b/client/ayon_core/plugins/load/delete_old_versions.py index f432829860..62302e7123 100644 --- a/client/ayon_core/plugins/load/delete_old_versions.py +++ b/client/ayon_core/plugins/load/delete_old_versions.py @@ -238,9 +238,12 @@ class DeleteOldVersions(load.ProductLoaderPlugin): versions_by_parent[ent["productId"]].append(ent) # Filter already deleted versions + versions_to_pop = [] for version in versions: if "deleted" in version["tags"]: - continue + versions_to_pop.append(version) + + for version in versions_to_pop: msg = "Folder: \"{}\" | Product: \"{}\" | Version: \"{}\"".format( folder_entity["path"], product_entity["name"],