Merge pull request #1622 from ynput/bugfix/902-ay-3875_ayon-integrate-hero-for-review

Integrate Hero: Use FileTransaction in integrate plugin

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -35,6 +35,7 @@ body:
       label: Version
       description: What version are you running? Look to AYON Tray
       options:
+        - 1.7.0
         - 1.6.13
         - 1.6.12
         - 1.6.11

client/ayon_core/plugins/publish/extract_thumbnail_from_source.py

@@ -160,9 +160,14 @@ class ExtractThumbnailFromSource(pyblish.api.InstancePlugin):
         dst_path: str,
     ) -> bool:
         self.log.debug("Outputting thumbnail with OIIO: {}".format(dst_path))
-        resolution_args = self._get_resolution_args(
+        try:
-            "oiiotool", src_path
+            resolution_args = self._get_resolution_args(
-        )
+                "oiiotool", src_path
+            )
+        except Exception:
+            self.log.warning("Failed to get resolution args for OIIO.")
+            return False
+
         oiio_cmd = get_oiio_tool_args("oiiotool", "-a", src_path)
         if resolution_args:
             # resize must be before -o
@@ -188,9 +193,13 @@ class ExtractThumbnailFromSource(pyblish.api.InstancePlugin):
         src_path: str,
         dst_path: str,
     ) -> bool:
-        resolution_args = self._get_resolution_args(
+        try:
-            "ffmpeg", src_path
+            resolution_args = self._get_resolution_args(
-        )
+                "ffmpeg", src_path
+            )
+        except Exception:
+            self.log.warning("Failed to get resolution args for ffmpeg.")
+            return False
 
         max_int = str(2147483647)
         ffmpeg_cmd = get_ffmpeg_tool_args(

client/ayon_core/plugins/publish/integrate_hero_version.py

@@ -1,11 +1,8 @@
 import os
+import sys
 import copy
-import errno
 import itertools
 import shutil
-from concurrent.futures import ThreadPoolExecutor
-
-from speedcopy import copyfile
 
 import clique
 import pyblish.api
@@ -16,11 +13,15 @@ from ayon_api.operations import (
 )
 from ayon_api.utils import create_entity_id
 
-from ayon_core.lib import create_hard_link, source_hash
+from ayon_core.lib import source_hash
-from ayon_core.lib.file_transaction import wait_for_future_errors
+from ayon_core.lib.file_transaction import (
+    FileTransaction,
+    DuplicateDestinationError,
+)
 from ayon_core.pipeline.publish import (
     get_publish_template_name,
     OptionalPyblishPluginMixin,
+    KnownPublishError,
 )
 
 
@@ -421,19 +422,40 @@ class IntegrateHeroVersion(
                     (repre_entity, dst_paths)
                 )
 
-            self.path_checks = []
+            file_transactions = FileTransaction(
+                log=self.log,
+                # Enforce unique transfers
+                allow_queue_replacements=False
+            )
+            mode = FileTransaction.MODE_COPY
+            if self.use_hardlinks:
+                mode = FileTransaction.MODE_LINK
 
-            # Copy(hardlink) paths of source and destination files
+            try:
-            # TODO should we *only* create hardlinks?
+                for src_path, dst_path in itertools.chain(
-            # TODO should we keep files for deletion until this is successful?
+                    src_to_dst_file_paths,
-            with ThreadPoolExecutor(max_workers=8) as executor:
+                    other_file_paths_mapping
-                futures = [
+                ):
-                    executor.submit(self.copy_file, src_path, dst_path)
+                    file_transactions.add(src_path, dst_path, mode=mode)
-                    for src_path, dst_path in itertools.chain(
+
-                        src_to_dst_file_paths, other_file_paths_mapping
+                self.log.debug("Integrating source files to destination ...")
-                    )
+                file_transactions.process()
-                ]
+
-                wait_for_future_errors(executor, futures)
+            except DuplicateDestinationError as exc:
+                # Raise DuplicateDestinationError as KnownPublishError
+                # and rollback the transactions
+                file_transactions.rollback()
+                raise KnownPublishError(exc).with_traceback(sys.exc_info()[2])
+
+            except Exception as exc:
+                # Rollback the transactions
+                file_transactions.rollback()
+                self.log.critical("Error when copying files", exc_info=True)
+                raise exc
+
+            # Finalizing can't rollback safely so no use for moving it to
+            # the try, except.
+            file_transactions.finalize()
 
             # Update prepared representation etity data with files
             #   and integrate it to server.
@@ -622,48 +644,6 @@ class IntegrateHeroVersion(
             ).format(path))
         return path
 
-    def copy_file(self, src_path, dst_path):
-        # TODO check drives if are the same to check if cas hardlink
-        dirname = os.path.dirname(dst_path)
-
-        try:
-            os.makedirs(dirname)
-            self.log.debug("Folder(s) created: \"{}\"".format(dirname))
-        except OSError as exc:
-            if exc.errno != errno.EEXIST:
-                self.log.error("An unexpected error occurred.", exc_info=True)
-                raise
-
-            self.log.debug("Folder already exists: \"{}\"".format(dirname))
-
-        if self.use_hardlinks:
-            # First try hardlink and copy if paths are cross drive
-            self.log.debug("Hardlinking file \"{}\" to \"{}\"".format(
-                src_path, dst_path
-            ))
-            try:
-                create_hard_link(src_path, dst_path)
-                # Return when successful
-                return
-
-            except OSError as exc:
-                # re-raise exception if different than
-                # EXDEV - cross drive path
-                # EINVAL - wrong format, must be NTFS
-                self.log.debug(
-                    "Hardlink failed with errno:'{}'".format(exc.errno))
-                if exc.errno not in [errno.EXDEV, errno.EINVAL]:
-                    raise
-
-            self.log.debug(
-                "Hardlinking failed, falling back to regular copy...")
-
-        self.log.debug("Copying file \"{}\" to \"{}\"".format(
-            src_path, dst_path
-        ))
-
-        copyfile(src_path, dst_path)
-
     def version_from_representations(self, project_name, repres):
         for repre in repres:
             version = ayon_api.get_version_by_id(

client/ayon_core/tools/push_to_project/models/integrate.py

@@ -1045,10 +1045,23 @@ class ProjectPushItemProcess:
         copied_tags = self._get_transferable_tags(src_version_entity)
         copied_status = self._get_transferable_status(src_version_entity)
 
+        description_parts = []
+        dst_attr_description = dst_attrib.get("description")
+        if dst_attr_description:
+            description_parts.append(dst_attr_description)
+
+        description = self._create_src_version_description(
+            self._item.src_project_name,
+            src_version_entity
+        )
+        if description:
+            description_parts.append(description)
+
+        dst_attrib["description"] = "\n\n".join(description_parts)
+
         version_entity = new_version_entity(
             dst_version,
             product_id,
-            author=src_version_entity["author"],
             status=copied_status,
             tags=copied_tags,
             task_id=self._task_info.get("id"),
@@ -1370,6 +1383,30 @@ class ProjectPushItemProcess:
             return copied_status["name"]
         return None
 
+    def _create_src_version_description(
+            self,
+            src_project_name: str,
+            src_version_entity: dict[str, Any]
+    ) -> str:
+        """Creates description text about source version."""
+        src_version_id = src_version_entity["id"]
+        src_author = src_version_entity["author"]
+        query = "&".join([
+            f"project={src_project_name}",
+            "type=version",
+            f"id={src_version_id}"
+        ])
+        version_url = (
+            f"{ayon_api.get_base_url()}"
+            f"/projects/{src_project_name}/products?{query}"
+        )
+        description = (
+            f"Version copied from from  {version_url} "
+            f"created by '{src_author}', "
+        )
+
+        return description
+
 
 class IntegrateModel:
     def __init__(self, controller):

client/ayon_core/version.py

@@ -1,3 +1,3 @@
 # -*- coding: utf-8 -*-
 """Package declaring AYON addon 'core' version."""
-__version__ = "1.7.0"
+__version__ = "1.7.0+dev"

package.py

@@ -1,6 +1,6 @@
 name = "core"
 title = "Core"
-version = "1.7.0"
+version = "1.7.0+dev"
 
 client_dir = "ayon_core"
 

pyproject.toml

@@ -5,7 +5,7 @@
 
 [tool.poetry]
 name = "ayon-core"
-version = "1.7.0"
+version = "1.7.0+dev"
 description = ""
 authors = ["Ynput Team <team@ynput.io>"]
 readme = "README.md"