ayon-core/openpype/plugins/load/delete_old_versions.py

import collections
import os
import uuid

import clique
from pymongo import UpdateOne
import qargparse
from qtpy import QtWidgets, QtCore

from openpype import style
from openpype.client import get_versions, get_representations
from openpype.modules import ModulesManager
from openpype.lib import format_file_size
from openpype.pipeline import load, AvalonMongoDB, Anatomy
from openpype.pipeline.load import (
    get_representation_path_with_anatomy,
    InvalidRepresentationContext,
)


class DeleteOldVersions(load.SubsetLoaderPlugin):
    """Deletes specific number of old version"""

    is_multiple_contexts_compatible = True
    sequence_splitter = "__sequence_splitter__"

    representations = ["*"]
    families = ["*"]
    tool_names = ["library_loader"]

    label = "Delete Old Versions"
    order = 35
    icon = "trash"
    color = "#d8d8d8"

    options = [
        qargparse.Integer(
            "versions_to_keep", default=2, min=0, help="Versions to keep:"
        ),
        qargparse.Boolean(
            "remove_publish_folder", help="Remove publish folder:"
        )
    ]

    def delete_whole_dir_paths(self, dir_paths, delete=True):
        size = 0

        for dir_path in dir_paths:
            # Delete all files and fodlers in dir path
            for root, dirs, files in os.walk(dir_path, topdown=False):
                for name in files:
                    file_path = os.path.join(root, name)
                    size += os.path.getsize(file_path)
                    if delete:
                        os.remove(file_path)
                        self.log.debug("Removed file: {}".format(file_path))

                for name in dirs:
                    if delete:
                        os.rmdir(os.path.join(root, name))

            if not delete:
                continue

            # Delete even the folder and it's parents folders if they are empty
            while True:
                if not os.path.exists(dir_path):
                    dir_path = os.path.dirname(dir_path)
                    continue

                if len(os.listdir(dir_path)) != 0:
                    break

                os.rmdir(os.path.join(dir_path))

        return size

    def path_from_representation(self, representation, anatomy):
        try:
            context = representation["context"]
        except KeyError:
            return (None, None)

        try:
            path = get_representation_path_with_anatomy(
                representation, anatomy
            )
        except InvalidRepresentationContext:
            return (None, None)

        sequence_path = None
        if "frame" in context:
            context["frame"] = self.sequence_splitter
            sequence_path = get_representation_path_with_anatomy(
                representation, anatomy
            )

        if sequence_path:
            sequence_path = sequence_path.normalized()

        return (path.normalized(), sequence_path)

    def delete_only_repre_files(self, dir_paths, file_paths, delete=True):
        size = 0

        for dir_id, dir_path in dir_paths.items():
            dir_files = os.listdir(dir_path)
            collections, remainders = clique.assemble(dir_files)
            for file_path, seq_path in file_paths[dir_id]:
                file_path_base = os.path.split(file_path)[1]
                # Just remove file if `frame` key was not in context or
                # filled path is in remainders (single file sequence)
                if not seq_path or file_path_base in remainders:
                    if not os.path.exists(file_path):
                        self.log.debug(
                            "File was not found: {}".format(file_path)
                        )
                        continue

                    size += os.path.getsize(file_path)

                    if delete:
                        os.remove(file_path)
                        self.log.debug("Removed file: {}".format(file_path))

                    if file_path_base in remainders:
                        remainders.remove(file_path_base)
                    continue

                seq_path_base = os.path.split(seq_path)[1]
                head, tail = seq_path_base.split(self.sequence_splitter)

                final_col = None
                for collection in collections:
                    if head != collection.head or tail != collection.tail:
                        continue
                    final_col = collection
                    break

                if final_col is not None:
                    # Fill full path to head
                    final_col.head = os.path.join(dir_path, final_col.head)
                    for _file_path in final_col:
                        if os.path.exists(_file_path):

                            size += os.path.getsize(_file_path)

                            if delete:
                                os.remove(_file_path)
                                self.log.debug(
                                    "Removed file: {}".format(_file_path)
                                )

                    _seq_path = final_col.format("{head}{padding}{tail}")
                    self.log.debug("Removed files: {}".format(_seq_path))
                    collections.remove(final_col)

                elif os.path.exists(file_path):
                    size += os.path.getsize(file_path)

                    if delete:
                        os.remove(file_path)
                        self.log.debug("Removed file: {}".format(file_path))
                else:
                    self.log.debug(
                        "File was not found: {}".format(file_path)
                    )

        # Delete as much as possible parent folders
        if not delete:
            return size

        for dir_path in dir_paths.values():
            while True:
                if not os.path.exists(dir_path):
                    dir_path = os.path.dirname(dir_path)
                    continue

                if len(os.listdir(dir_path)) != 0:
                    break

                self.log.debug("Removed folder: {}".format(dir_path))
                os.rmdir(dir_path)

        return size

    def message(self, text):
        msgBox = QtWidgets.QMessageBox()
        msgBox.setText(text)
        msgBox.setStyleSheet(style.load_stylesheet())
        msgBox.setWindowFlags(
            msgBox.windowFlags() | QtCore.Qt.FramelessWindowHint
        )
        msgBox.exec_()

    def get_data(self, context, versions_count):
        subset = context["subset"]
        asset = context["asset"]
        project_name = context["project"]["name"]
        anatomy = Anatomy(project_name)

        versions = list(get_versions(project_name, subset_ids=[subset["_id"]]))

        versions_by_parent = collections.defaultdict(list)
        for ent in versions:
            versions_by_parent[ent["parent"]].append(ent)

        def sort_func(ent):
            return int(ent["name"])

        all_last_versions = []
        for _parent_id, _versions in versions_by_parent.items():
            for idx, version in enumerate(
                sorted(_versions, key=sort_func, reverse=True)
            ):
                if idx >= versions_count:
                    break
                all_last_versions.append(version)

        self.log.debug("Collected versions ({})".format(len(versions)))

        # Filter latest versions
        for version in all_last_versions:
            versions.remove(version)

        # Update versions_by_parent without filtered versions
        versions_by_parent = collections.defaultdict(list)
        for ent in versions:
            versions_by_parent[ent["parent"]].append(ent)

        # Filter already deleted versions
        versions_to_pop = []
        for version in versions:
            version_tags = version["data"].get("tags")
            if version_tags and "deleted" in version_tags:
                versions_to_pop.append(version)

        for version in versions_to_pop:
            msg = "Asset: \"{}\" | Subset: \"{}\" | Version: \"{}\"".format(
                asset["name"], subset["name"], version["name"]
            )
            self.log.debug((
                "Skipping version. Already tagged as `deleted`. < {} >"
            ).format(msg))
            versions.remove(version)

        version_ids = [ent["_id"] for ent in versions]

        self.log.debug(
            "Filtered versions to delete ({})".format(len(version_ids))
        )

        if not version_ids:
            msg = "Skipping processing. Nothing to delete on {}/{}".format(
                asset["name"], subset["name"]
            )
            self.log.info(msg)
            print(msg)
            return

        repres = list(get_representations(
            project_name, version_ids=version_ids
        ))

        self.log.debug(
            "Collected representations to remove ({})".format(len(repres))
        )

        dir_paths = {}
        file_paths_by_dir = collections.defaultdict(list)
        for repre in repres:
            file_path, seq_path = self.path_from_representation(repre, anatomy)
            if file_path is None:
                self.log.debug((
                    "Could not format path for represenation \"{}\""
                ).format(str(repre)))
                continue

            dir_path = os.path.dirname(file_path)
            dir_id = None
            for _dir_id, _dir_path in dir_paths.items():
                if _dir_path == dir_path:
                    dir_id = _dir_id
                    break

            if dir_id is None:
                dir_id = uuid.uuid4()
                dir_paths[dir_id] = dir_path

            file_paths_by_dir[dir_id].append([file_path, seq_path])

        dir_ids_to_pop = []
        for dir_id, dir_path in dir_paths.items():
            if os.path.exists(dir_path):
                continue

            dir_ids_to_pop.append(dir_id)

        # Pop dirs from both dictionaries
        for dir_id in dir_ids_to_pop:
            dir_paths.pop(dir_id)
            paths = file_paths_by_dir.pop(dir_id)
            # TODO report of missing directories?
            paths_msg = ", ".join([
                "'{}'".format(path[0].replace("\\", "/")) for path in paths
            ])
            self.log.debug((
                "Folder does not exist. Deleting it's files skipped: {}"
            ).format(paths_msg))

        data = {
            "dir_paths": dir_paths,
            "file_paths_by_dir": file_paths_by_dir,
            "versions": versions,
            "asset": asset,
            "subset": subset,
            "archive_subset": versions_count == 0
        }

        return data

    def main(self, project_name, data, remove_publish_folder):
        # Size of files.
        size = 0
        if not data:
            return size

        if remove_publish_folder:
            size = self.delete_whole_dir_paths(data["dir_paths"].values())
        else:
            size = self.delete_only_repre_files(
                data["dir_paths"], data["file_paths_by_dir"]
            )

        mongo_changes_bulk = []
        for version in data["versions"]:
            orig_version_tags = version["data"].get("tags") or []
            version_tags = [tag for tag in orig_version_tags]
            if "deleted" not in version_tags:
                version_tags.append("deleted")

            if version_tags == orig_version_tags:
                continue

            update_query = {"_id": version["_id"]}
            update_data = {"$set": {"data.tags": version_tags}}
            mongo_changes_bulk.append(UpdateOne(update_query, update_data))

        if data["archive_subset"]:
            mongo_changes_bulk.append(UpdateOne(
                {
                    "_id": data["subset"]["_id"],
                    "type": "subset"
                },
                {"$set": {"type": "archived_subset"}}
            ))

        if mongo_changes_bulk:
            dbcon = AvalonMongoDB()
            dbcon.Session["AVALON_PROJECT"] = project_name
            dbcon.install()
            dbcon.bulk_write(mongo_changes_bulk)
            dbcon.uninstall()

        self._ftrack_delete_versions(data)

        return size

    def _ftrack_delete_versions(self, data):
        """Delete version on ftrack.

        Handling of ftrack logic in this plugin is not ideal. But in OP3 it is
        almost impossible to solve the issue other way.

        Note:
            Asset versions on ftrack are not deleted but marked as
                "not published" which cause that they're invisible.

        Args:
            data (dict): Data sent to subset loader with full context.
        """

        # First check for ftrack id on asset document
        #   - skip if ther is none
        asset_ftrack_id = data["asset"]["data"].get("ftrackId")
        if not asset_ftrack_id:
            self.log.info((
                "Asset does not have filled ftrack id. Skipped delete"
                " of ftrack version."
            ))
            return

        # Check if ftrack module is enabled
        modules_manager = ModulesManager()
        ftrack_module = modules_manager.modules_by_name.get("ftrack")
        if not ftrack_module or not ftrack_module.enabled:
            return

        import ftrack_api

        session = ftrack_api.Session()
        subset_name = data["subset"]["name"]
        versions = {
            '"{}"'.format(version_doc["name"])
            for version_doc in data["versions"]
        }
        asset_versions = session.query(
            (
                "select id, is_published from AssetVersion where"
                " asset.parent.id is \"{}\""
                " and asset.name is \"{}\""
                " and version in ({})"
            ).format(
                asset_ftrack_id,
                subset_name,
                ",".join(versions)
            )
        ).all()

        # Set attribute `is_published` to `False` on ftrack AssetVersions
        for asset_version in asset_versions:
            asset_version["is_published"] = False

        try:
            session.commit()

        except Exception:
            msg = (
                "Could not set `is_published` attribute to `False`"
                " for selected AssetVersions."
            )
            self.log.error(msg)
            self.message(msg)

    def load(self, contexts, name=None, namespace=None, options=None):
        try:
            size = 0
            for count, context in enumerate(contexts):
                versions_to_keep = 2
                remove_publish_folder = False
                if options:
                    versions_to_keep = options.get(
                        "versions_to_keep", versions_to_keep
                    )
                    remove_publish_folder = options.get(
                        "remove_publish_folder", remove_publish_folder
                    )

                data = self.get_data(context, versions_to_keep)
                if not data:
                    continue

                project_name = context["project"]["name"]
                size += self.main(project_name, data, remove_publish_folder)
                print("Progressing {}/{}".format(count + 1, len(contexts)))

            msg = "Total size of files: {}".format(format_file_size(size))
            self.log.info(msg)
            self.message(msg)

        except Exception:
            self.log.error("Failed to delete versions.", exc_info=True)


class CalculateOldVersions(DeleteOldVersions):
    """Calculate file size of old versions"""
    label = "Calculate Old Versions"
    order = 30
    tool_names = ["library_loader"]

    options = [
        qargparse.Integer(
            "versions_to_keep", default=2, min=0, help="Versions to keep:"
        ),
        qargparse.Boolean(
            "remove_publish_folder", help="Remove publish folder:"
        )
    ]

    def main(self, project_name, data, remove_publish_folder):
        size = 0

        if not data:
            return size

        if remove_publish_folder:
            size = self.delete_whole_dir_paths(
                data["dir_paths"].values(), delete=False
            )
        else:
            size = self.delete_only_repre_files(
                data["dir_paths"], data["file_paths_by_dir"], delete=False
            )

        return size