Merge pull request #4184 from pypeclub/feature/OP-4566_File-transactions-Handle-cases-when-source-and-destination-is-the-same

File transactions: Source path is destination path

openpype/lib/file_transaction.py

@@ -14,9 +14,9 @@ else:
 
 
 class FileTransaction(object):
-    """
+    """File transaction with rollback options.
 
-    The file transaction is a three step process.
+    The file transaction is a three-step process.
 
     1) Rename any existing files to a "temporary backup" during `process()`
     2) Copy the files to final destination during `process()`
@@ -39,14 +39,12 @@ class FileTransaction(object):
 
     Warning:
         Any folders created during the transfer will not be removed.
-
     """
 
     MODE_COPY = 0
     MODE_HARDLINK = 1
 
     def __init__(self, log=None):
-
         if log is None:
             log = logging.getLogger("FileTransaction")
 
@@ -63,49 +61,64 @@ class FileTransaction(object):
         self._backup_to_original = {}
 
     def add(self, src, dst, mode=MODE_COPY):
-        """Add a new file to transfer queue"""
+        """Add a new file to transfer queue.
+
+        Args:
+            src (str): Source path.
+            dst (str): Destination path.
+            mode (MODE_COPY, MODE_HARDLINK): Transfer mode.
+        """
+
         opts = {"mode": mode}
 
-        src = os.path.abspath(src)
-        dst = os.path.abspath(dst)
+        src = os.path.normpath(os.path.abspath(src))
+        dst = os.path.normpath(os.path.abspath(dst))
 
         if dst in self._transfers:
             queued_src = self._transfers[dst][0]
             if src == queued_src:
-                self.log.debug("File transfer was already "
-                               "in queue: {} -> {}".format(src, dst))
+                self.log.debug(
+                    "File transfer was already in queue: {} -> {}".format(
+                        src, dst))
                 return
             else:
                 self.log.warning("File transfer in queue replaced..")
-                self.log.debug("Removed from queue: "
-                               "{} -> {}".format(queued_src, dst))
-                self.log.debug("Added to queue: {} -> {}".format(src, dst))
+                self.log.debug(
+                    "Removed from queue: {} -> {} replaced by {} -> {}".format(
+                        queued_src, dst, src, dst))
 
         self._transfers[dst] = (src, opts)
 
     def process(self):
-
         # Backup any existing files
-        for dst in self._transfers.keys():
-            if os.path.exists(dst):
-                # Backup original file
-                # todo: add timestamp or uuid to ensure unique
-                backup = dst + ".bak"
-                self._backup_to_original[backup] = dst
-                self.log.debug("Backup existing file: "
-                               "{} -> {}".format(dst, backup))
-                os.rename(dst, backup)
+        for dst, (src, _) in self._transfers.items():
+            if dst == src or not os.path.exists(dst):
+                continue
+
+            # Backup original file
+            # todo: add timestamp or uuid to ensure unique
+            backup = dst + ".bak"
+            self._backup_to_original[backup] = dst
+            self.log.debug(
+                "Backup existing file: {} -> {}".format(dst, backup))
+            os.rename(dst, backup)
 
         # Copy the files to transfer
         for dst, (src, opts) in self._transfers.items():
+            if dst == src:
+                self.log.debug(
+                    "Source and destionation are same files {} -> {}".format(
+                        src, dst))
+                continue
+
             self._create_folder_for_file(dst)
 
             if opts["mode"] == self.MODE_COPY:
                 self.log.debug("Copying file ... {} -> {}".format(src, dst))
                 copyfile(src, dst)
             elif opts["mode"] == self.MODE_HARDLINK:
-                self.log.debug("Hardlinking file ... {} -> {}".format(src,
-                                                                      dst))
+                self.log.debug("Hardlinking file ... {} -> {}".format(
+                    src, dst))
                 create_hard_link(src, dst)
 
             self._transferred.append(dst)
@@ -116,23 +129,21 @@ class FileTransaction(object):
             try:
                 os.remove(backup)
             except OSError:
-                self.log.error("Failed to remove backup file: "
-                               "{}".format(backup),
-                               exc_info=True)
+                self.log.error(
+                    "Failed to remove backup file: {}".format(backup),
+                    exc_info=True)
 
     def rollback(self):
-
         errors = 0
-
         # Rollback any transferred files
         for path in self._transferred:
             try:
                 os.remove(path)
             except OSError:
                 errors += 1
-                self.log.error("Failed to rollback created file: "
-                               "{}".format(path),
-                               exc_info=True)
+                self.log.error(
+                    "Failed to rollback created file: {}".format(path),
+                    exc_info=True)
 
         # Rollback the backups
         for backup, original in self._backup_to_original.items():
@@ -140,13 +151,15 @@ class FileTransaction(object):
                 os.rename(backup, original)
             except OSError:
                 errors += 1
-                self.log.error("Failed to restore original file: "
-                               "{} -> {}".format(backup, original),
-                               exc_info=True)
+                self.log.error(
+                    "Failed to restore original file: {} -> {}".format(
+                        backup, original),
+                    exc_info=True)
 
         if errors:
-            self.log.error("{} errors occurred during "
-                           "rollback.".format(errors), exc_info=True)
+            self.log.error(
+                "{} errors occurred during rollback.".format(errors),
+                exc_info=True)
             six.reraise(*sys.exc_info())
 
     @property

openpype/plugins/publish/integrate.py

@@ -291,9 +291,6 @@ class IntegrateAsset(pyblish.api.InstancePlugin):
                 instance)
 
             for src, dst in prepared["transfers"]:
-                if src == dst:
-                    continue
-
                 # todo: add support for hardlink transfers
                 file_transactions.add(src, dst)