From cb5b4ab9485a32f3346fa6236897e82f76a51bb2 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Wed, 10 Jun 2020 20:02:12 +0200
Subject: [PATCH 01/15] Initial commit of adding 'files' section to
 representation - WIP

All representation integrated in DB should contain all attached
resources. This will be used in future synchronization implementation.
---
 pype/plugins/global/publish/integrate_new.py | 139 ++++++++++++++++++-
 1 file changed, 134 insertions(+), 5 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index bd908901cc..60809b864f 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -91,18 +91,28 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     default_template_name = "publish"
     template_name_profiles = None
 
+    integrated_file_sizes = {} # file_url : file_size
+
     def process(self, instance):
 
         if [ef for ef in self.exclude_families
                 if instance.data["family"] in ef]:
             return
 
-        self.register(instance)
+        self.log.info("IntegrateAssetNew.process:")
+        import json
+        self.log.info("instance: {}".format(json.dumps(instance.__dict__, default=str)))
 
-        self.log.info("Integrating Asset in to the database ...")
-        self.log.info("instance.data: {}".format(instance.data))
-        if instance.data.get('transfer', True):
-            self.integrate(instance)
+        try:
+            self.register(instance)
+            self.log.info("Integrated Asset in to the database ...")
+            self.log.info("instance.data: {}".format(instance.data))
+        except Exception:
+            # clean destination
+            # !TODO fix exceptions.WindowsError
+            e = sys.exc_info()[0]
+            self.log.critical("Error when registering {}".format(e))
+            self.clean_destination_files(self.integrated_file_sizes)
 
     def register(self, instance):
         # Required environment variables
@@ -472,6 +482,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     dst_padding_exp % int(repre.get("frameStart"))
                 )
 
+            if instance.data.get('transfer', True):
+                # could throw exception, will be caught in 'process'
+                # all integration to DB is being done together lower, so no rollback needed
+                self.log.info("Integrating source files to destination ...")
+                self.integrated_file_sizes = self.integrate(instance)
+                self.log.debug("Integrated files {}".format(self.integrated_file_sizes))
+                #TODO instance.data["transfers"].remove([src, dst]) # array needs to be changed to tuple
+
+            # get 'files' information for representation and all attached resources
+            self.log.debug("Preparing files information ..")
+            representation["files"] = self.get_files_info(dst, instance, self.integrated_file_sizes)
+
             self.log.debug("__ representation: {}".format(representation))
             destination_list.append(dst)
             self.log.debug("__ destination_list: {}".format(destination_list))
@@ -509,7 +531,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
             Args:
                 instance: the instance to integrate
+            Returns:
+                integrated_file_sizes: dictionary of destination file url and its size in bytes
         """
+        integrated_file_sizes = {} # store destination url and size for reporting and rollback
         transfers = instance.data.get("transfers", list())
 
         for src, dest in transfers:
@@ -519,6 +544,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         transfers = instance.data.get("transfers", list())
         for src, dest in transfers:
             self.copy_file(src, dest)
+            if os.path.exists(dest):
+                # TODO needs to be updated during site implementation
+                integrated_file_sizes[dest] = os.path.getsize(dest)
+
 
         # Produce hardlinked copies
         # Note: hardlink can only be produced between two files on the same
@@ -529,6 +558,11 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         for src, dest in hardlinks:
             self.log.debug("Hardlinking file .. {} -> {}".format(src, dest))
             self.hardlink_file(src, dest)
+            if os.path.exists(dest):
+                # TODO needs to be updated during site implementation
+                integrated_file_sizes[dest] = os.path.getsize(dest)
+
+        return integrated_file_sizes
 
     def copy_file(self, src, dst):
         """ Copy given source to destination
@@ -764,3 +798,98 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             ).format(family, task_name, template_name))
 
         return template_name
+
+    def get_files_info(self, dst, instance, integrated_file_sizes):
+        """ Prepare 'files' portion of representation to store all attached files (representatio, textures, json etc.)
+            This information is used in synchronization of mentioned files
+
+            Args:
+                dst: destination path for representation
+                instance: the instance to integrate
+            Returns:
+                files: list of dictionaries with representation and all resources
+        """
+        self.log.debug("get_files_info:")
+        # add asset and all resources to 'files'
+        files = []
+        # file info for representation - should be always
+        rec = self.prepare_file_info(dst, integrated_file_sizes[dst], 'temphash')  # TODO
+        files.append(rec)
+
+        # file info for resources
+        resource_files = self.get_resource_files_info(instance, integrated_file_sizes)
+        if resource_files: # do not append empty list
+            files.extend(resource_files)
+
+        return files
+
+    def get_resource_files_info(self, instance, integrated_file_sizes):
+        """ Prepare 'files' portion for attached resources,
+            could be empty if no resources (like textures) present
+
+        Arguments:
+            instance: the current instance being published
+        Returns:
+            output_resources: array of dictionaries to be added to 'files' key in representation
+        """
+        # TODO check if sourceHashes is viable or transfers (they are updated during loop though)
+        resources = instance.data.get("sourceHashes", {})
+        self.log.debug("get_resource_files_info: {}".format(resources))
+        output_resources = []
+        for resource_info, resource_path in resources.items():
+            # TODO - hash or use self.integrated_file_size
+            file_name,file_time,file_size,file_args = resource_info.split("|")
+            output_resources.append(self.prepare_file_info(resource_path, file_size, 'temphash'))
+
+        return output_resources
+
+    def prepare_file_info(self, path, size = None, hash = None, sites = None):
+        """ Prepare information for one file (asset or resource)
+
+        Arguments:
+            path: destination url of published file
+            size(optional): size of file in bytes
+            hash(optional): hash of file for synchronization validation
+            sites(optional): array of published locations, ['studio'] by default,
+                                expected ['studio', 'site1', 'gdrive1']
+        Returns:
+            rec: dictionary with filled info
+        """
+
+        rec = {  # TODO update collection step to extend with necessary values
+            "_id": io.ObjectId(),
+            "path": path
+        }
+        if size:
+            rec["size"] = size
+
+        if hash:
+            rec["hash"] = hash
+
+        if sites:
+            rec["sites"] = sites
+        else:
+            rec["sites"] = ["studio"]
+
+        self.log.debug("prepare_file_info: {}".format(rec))
+
+        return rec
+
+    def clean_destination_files(self, integrated_file_sizes):
+        """ Clean destination files
+            Called when error happened during integrating to DB or to disk
+            Used to clean unwanted files
+
+        Arguments:
+            integrated_file_sizes: disctionary of uploaded files, file urls as keys
+        """
+        if integrated_file_sizes:
+            for file_url, file_size in integrated_file_sizes.items():
+                try:
+                    self.log.debug("Removing file...{}".format(file_url))
+                    os.remove(file_url) # needs to be changed to Factory when sites implemented
+                except FileNotFoundError:
+                    pass # file not there, nothing to delete
+                except OSError as e:
+                    self.log.critical("Cannot remove file {}".format(file_url))
+                    self.log.critical(e)
\ No newline at end of file

From f1ea60f98312627af0bdc9d24a938d0c74315c94 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 11 Jun 2020 09:43:57 +0200
Subject: [PATCH 02/15] Fix - removed double copy of already copied file

Fix - remove double creation of hardlink resulting in WindowError
---
 pype/plugins/global/publish/integrate_new.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 60809b864f..740de4d930 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -544,9 +544,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         transfers = instance.data.get("transfers", list())
         for src, dest in transfers:
             self.copy_file(src, dest)
-            if os.path.exists(dest):
-                # TODO needs to be updated during site implementation
-                integrated_file_sizes[dest] = os.path.getsize(dest)
+            # TODO needs to be updated during site implementation
+            integrated_file_sizes[dest] = os.path.getsize(dest)
+            # already copied, delete from transfers to limit double copy TODO double check
+            instance.data.get("transfers", list()).remove([src, dest])
 
 
         # Produce hardlinked copies
@@ -557,10 +558,11 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         hardlinks = instance.data.get("hardlinks", list())
         for src, dest in hardlinks:
             self.log.debug("Hardlinking file .. {} -> {}".format(src, dest))
-            self.hardlink_file(src, dest)
-            if os.path.exists(dest):
-                # TODO needs to be updated during site implementation
-                integrated_file_sizes[dest] = os.path.getsize(dest)
+            if not os.path.exists(dest):
+                self.hardlink_file(src, dest)
+
+            # TODO needs to be updated during site implementation
+            integrated_file_sizes[dest] = os.path.getsize(dest)
 
         return integrated_file_sizes
 

From 91c3f479239779da46cae5349b06b2408df83efd Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 11 Jun 2020 18:32:36 +0200
Subject: [PATCH 03/15] Added creation of temporary files in destination first

Fix - integrated_file_sizes got carried over
Fix - create orig_transfers to fall back to original transfers
Fix - updated get_files_info logic - using transfers and hardlinks
Fix - added rootless_path into representation
Fix - updated file handling if errors
---
 pype/plugins/global/publish/integrate_new.py | 182 +++++++++++--------
 1 file changed, 110 insertions(+), 72 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 740de4d930..3f906e3455 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -91,28 +91,30 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     default_template_name = "publish"
     template_name_profiles = None
 
-    integrated_file_sizes = {} # file_url : file_size
+    integrated_file_sizes = {} # file_url : file_size of all published and uploaded files
+
+    TMP_FILE_EXT = 'tmp' # suffix to denote temporary files, use without '.'
 
     def process(self, instance):
-
+        self.integrated_file_sizes = {}
         if [ef for ef in self.exclude_families
                 if instance.data["family"] in ef]:
             return
 
         self.log.info("IntegrateAssetNew.process:")
         import json
-        self.log.info("instance: {}".format(json.dumps(instance.__dict__, default=str)))
+        self.log.debug("instance: {}".format(json.dumps(instance.__dict__, default=str)))
 
         try:
             self.register(instance)
             self.log.info("Integrated Asset in to the database ...")
             self.log.info("instance.data: {}".format(instance.data))
-        except Exception:
+            self.handle_destination_files(self.integrated_file_sizes, instance, 'finalize')
+        except Exception as e:
             # clean destination
-            # !TODO fix exceptions.WindowsError
-            e = sys.exc_info()[0]
-            self.log.critical("Error when registering {}".format(e))
-            self.clean_destination_files(self.integrated_file_sizes)
+            self.log.critical("Error when registering",  exc_info=True)
+            self.handle_destination_files(self.integrated_file_sizes, instance, 'remove')
+            raise
 
     def register(self, instance):
         # Required environment variables
@@ -273,13 +275,20 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         representations = []
         destination_list = []
 
+        orig_transfers = []
         if 'transfers' not in instance.data:
             instance.data['transfers'] = []
+        else:
+            orig_transfers = list(instance.data['transfers'])
 
         template_name = self.template_name_from_instance(instance)
 
         published_representations = {}
         for idx, repre in enumerate(instance.data["representations"]):
+            # reset transfers for next representation
+            # instance.data['transfers'] is used as a global variable in current codebase
+            instance.data['transfers'] = list(orig_transfers)
+
             published_files = []
 
             # create template data for Anatomy
@@ -482,17 +491,22 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     dst_padding_exp % int(repre.get("frameStart"))
                 )
 
-            if instance.data.get('transfer', True):
+            # any file that should be physically copied is expected in 'transfers' or 'hardlinks'
+            # both have same interface [[source_url, destination_url], [source_url...]]
+            if instance.data.get('transfers', False) or instance.data.get('hardlinks', False):
                 # could throw exception, will be caught in 'process'
                 # all integration to DB is being done together lower, so no rollback needed
-                self.log.info("Integrating source files to destination ...")
-                self.integrated_file_sizes = self.integrate(instance)
+                self.log.debug("Integrating source files to destination ...")
+                self.integrated_file_sizes.update(self.integrate(instance))
                 self.log.debug("Integrated files {}".format(self.integrated_file_sizes))
-                #TODO instance.data["transfers"].remove([src, dst]) # array needs to be changed to tuple
+
+            import random
+            if random.choice([True, False, True, True]):
+                raise Exception("Monkey attack!!!")
 
             # get 'files' information for representation and all attached resources
             self.log.debug("Preparing files information ..")
-            representation["files"] = self.get_files_info(dst, instance, self.integrated_file_sizes)
+            representation["files"] = self.get_files_info(instance, self.integrated_file_sizes)
 
             self.log.debug("__ representation: {}".format(representation))
             destination_list.append(dst)
@@ -535,20 +549,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 integrated_file_sizes: dictionary of destination file url and its size in bytes
         """
         integrated_file_sizes = {} # store destination url and size for reporting and rollback
-        transfers = instance.data.get("transfers", list())
-
+        transfers = list(instance.data.get("transfers", list()))
         for src, dest in transfers:
             if os.path.normpath(src) != os.path.normpath(dest):
+                dest = self.get_dest_temp_url(dest)
                 self.copy_file(src, dest)
-
-        transfers = instance.data.get("transfers", list())
-        for src, dest in transfers:
-            self.copy_file(src, dest)
-            # TODO needs to be updated during site implementation
-            integrated_file_sizes[dest] = os.path.getsize(dest)
-            # already copied, delete from transfers to limit double copy TODO double check
-            instance.data.get("transfers", list()).remove([src, dest])
-
+                # TODO needs to be updated during site implementation
+                integrated_file_sizes[dest] = os.path.getsize(dest)
 
         # Produce hardlinked copies
         # Note: hardlink can only be produced between two files on the same
@@ -557,6 +564,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         # to ensure publishes remain safe and non-edited.
         hardlinks = instance.data.get("hardlinks", list())
         for src, dest in hardlinks:
+            dest = self.get_dest_temp_url(dest)
             self.log.debug("Hardlinking file .. {} -> {}".format(src, dest))
             if not os.path.exists(dest):
                 self.hardlink_file(src, dest)
@@ -692,16 +700,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         else:
             source = context.data["currentFile"]
             anatomy = instance.context.data["anatomy"]
-            success, rootless_path = (
-                anatomy.find_root_template_from_path(source)
-            )
-            if success:
-                source = rootless_path
-            else:
-                self.log.warning((
-                    "Could not find root path for remapping \"{}\"."
-                    " This may cause issues on farm."
-                ).format(source))
+            source = self.get_rootless_path(anatomy, source)
 
         self.log.debug("Source: {}".format(source))
         version_data = {
@@ -801,55 +800,75 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
         return template_name
 
-    def get_files_info(self, dst, instance, integrated_file_sizes):
-        """ Prepare 'files' portion of representation to store all attached files (representatio, textures, json etc.)
-            This information is used in synchronization of mentioned files
 
-            Args:
-                dst: destination path for representation
-                instance: the instance to integrate
-            Returns:
-                files: list of dictionaries with representation and all resources
+    def get_rootless_path(self, anatomy, path):
+        """  Returns, if possible, path without absolute portion from host (eg. 'c:\' or '/opt/..')
+             This information is host dependent and shouldn't be captured.
+             Example:
+                 'c:/projects/MyProject1/Assets/publish...' > '{root}/MyProject1/Assets...'
+
+        Args:
+                anatomy: anatomy part from instance
+                path: path (absolute)
+        Returns:
+                path: modified path if possible, or unmodified path + warning logged
         """
-        self.log.debug("get_files_info:")
-        # add asset and all resources to 'files'
-        files = []
-        # file info for representation - should be always
-        rec = self.prepare_file_info(dst, integrated_file_sizes[dst], 'temphash')  # TODO
-        files.append(rec)
+        success, rootless_path = (
+            anatomy.find_root_template_from_path(path)
+        )
+        if success:
+            path = rootless_path
+        else:
+            self.log.warning((
+                                 "Could not find root path for remapping \"{}\"."
+                                 " This may cause issues on farm."
+                             ).format(path))
+        return path
 
-        # file info for resources
-        resource_files = self.get_resource_files_info(instance, integrated_file_sizes)
-        if resource_files: # do not append empty list
-            files.extend(resource_files)
-
-        return files
-
-    def get_resource_files_info(self, instance, integrated_file_sizes):
-        """ Prepare 'files' portion for attached resources,
-            could be empty if no resources (like textures) present
+    def get_files_info(self, instance, integrated_file_sizes):
+        """ Prepare 'files' portion for attached resources and main asset.
+            Combining records from 'transfers' and 'hardlinks' parts from instance.
+            All attached resources should be added, currently without Context info.
 
         Arguments:
             instance: the current instance being published
+            integrated_file_sizes: dictionary of destination path (absolute) and its file size
         Returns:
             output_resources: array of dictionaries to be added to 'files' key in representation
         """
-        # TODO check if sourceHashes is viable or transfers (they are updated during loop though)
-        resources = instance.data.get("sourceHashes", {})
-        self.log.debug("get_resource_files_info: {}".format(resources))
+        resources = list(instance.data.get("transfers", []))
+        resources.extend(list(instance.data.get("hardlinks", [])))
+
+        self.log.debug("get_resource_files_info.resources: {}".format(resources))
+
         output_resources = []
-        for resource_info, resource_path in resources.items():
+        anatomy = instance.context.data["anatomy"]
+        for src, dest in resources:
             # TODO - hash or use self.integrated_file_size
-            file_name,file_time,file_size,file_args = resource_info.split("|")
-            output_resources.append(self.prepare_file_info(resource_path, file_size, 'temphash'))
+            path = self.get_rootless_path(anatomy, dest)
+            dest = self.get_dest_temp_url(dest)
+            output_resources.append(self.prepare_file_info(path, integrated_file_sizes[dest], 'temphash'))
 
         return output_resources
 
+    def get_dest_temp_url(self, dest):
+        """ Enhance destination path with TMP_FILE_EXT to denote temporary file.
+            Temporary files will be renamed after successful registration into DB and full copy to destination
+
+        Arguments:
+            dest: destination url of published file (absolute)
+        Returns:
+            dest: destination path + '.TMP_FILE_EXT'
+        """
+        if self.TMP_FILE_EXT and '.{}'.format(self.TMP_FILE_EXT) not in dest:
+            dest += '.{}'.format(self.TMP_FILE_EXT)
+        return dest
+
     def prepare_file_info(self, path, size = None, hash = None, sites = None):
         """ Prepare information for one file (asset or resource)
 
         Arguments:
-            path: destination url of published file
+            path: destination url of published file (rootless)
             size(optional): size of file in bytes
             hash(optional): hash of file for synchronization validation
             sites(optional): array of published locations, ['studio'] by default,
@@ -873,25 +892,44 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         else:
             rec["sites"] = ["studio"]
 
-        self.log.debug("prepare_file_info: {}".format(rec))
-
         return rec
 
-    def clean_destination_files(self, integrated_file_sizes):
+    def handle_destination_files(self, integrated_file_sizes, instance, mode):
         """ Clean destination files
             Called when error happened during integrating to DB or to disk
             Used to clean unwanted files
 
         Arguments:
             integrated_file_sizes: disctionary of uploaded files, file urls as keys
+            instance: processed instance - for publish directories
+            mode: 'remove' - clean files,'finalize' - rename files, remove TMP_FILE_EXT suffix denoting temp file
         """
         if integrated_file_sizes:
             for file_url, file_size in integrated_file_sizes.items():
                 try:
-                    self.log.debug("Removing file...{}".format(file_url))
-                    os.remove(file_url) # needs to be changed to Factory when sites implemented
+                    if mode == 'remove':
+                        self.log.debug("Removing file...{}".format(file_url))
+                        os.remove(file_url) # needs to be changed to Factory when sites implemented
+                    if mode == 'finalize':
+                        self.log.debug("Renaming file...{}".format(file_url))
+                        import re
+                        os.rename(file_url, re.sub('\.{}$'.format(self.TMP_FILE_EXT), '', file_url))  # needs to be changed to Factory when sites implemented
+
                 except FileNotFoundError:
                     pass # file not there, nothing to delete
                 except OSError as e:
-                    self.log.critical("Cannot remove file {}".format(file_url))
-                    self.log.critical(e)
\ No newline at end of file
+                    self.log.critical("Cannot {} file {}".format(mode, file_url), exc_info=True)
+                    raise
+
+        if mode == 'remove':
+            try:
+                publishDir = instance.data.get('publishDir', '')
+                resourcesDir = instance.data.get('resourcesDir', '')
+                if resourcesDir:
+                    os.remove(resourcesDir)
+                if publishDir:
+                    os.remove(publishDir)
+            except OSError as e:
+                self.log.critical("Cannot remove destination directory {} or {}".format(publishDir, resourcesDir), exc_info=True)
+                raise
+

From 417358dc7fff007eec1b14efdd21ded9a7053dc4 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 11 Jun 2020 18:34:25 +0200
Subject: [PATCH 04/15] Housekeeping - removed parts for testing and debugging

Removed delete of publishDir and resourceDir, probably better to keep
there empty then deleting
---
 pype/plugins/global/publish/integrate_new.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 3f906e3455..e3e71f2dc1 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -101,10 +101,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 if instance.data["family"] in ef]:
             return
 
-        self.log.info("IntegrateAssetNew.process:")
-        import json
-        self.log.debug("instance: {}".format(json.dumps(instance.__dict__, default=str)))
-
         try:
             self.register(instance)
             self.log.info("Integrated Asset in to the database ...")
@@ -500,10 +496,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 self.integrated_file_sizes.update(self.integrate(instance))
                 self.log.debug("Integrated files {}".format(self.integrated_file_sizes))
 
-            import random
-            if random.choice([True, False, True, True]):
-                raise Exception("Monkey attack!!!")
-
             # get 'files' information for representation and all attached resources
             self.log.debug("Preparing files information ..")
             representation["files"] = self.get_files_info(instance, self.integrated_file_sizes)
@@ -921,15 +913,3 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     self.log.critical("Cannot {} file {}".format(mode, file_url), exc_info=True)
                     raise
 
-        if mode == 'remove':
-            try:
-                publishDir = instance.data.get('publishDir', '')
-                resourcesDir = instance.data.get('resourcesDir', '')
-                if resourcesDir:
-                    os.remove(resourcesDir)
-                if publishDir:
-                    os.remove(publishDir)
-            except OSError as e:
-                self.log.critical("Cannot remove destination directory {} or {}".format(publishDir, resourcesDir), exc_info=True)
-                raise
-

From 2e811e448fc1265c0ba1793550dc9cba592e3cfc Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 11 Jun 2020 18:58:45 +0200
Subject: [PATCH 05/15] Taming of Hound

---
 pype/plugins/global/publish/integrate_new.py | 86 +++++++++++++-------
 1 file changed, 55 insertions(+), 31 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index e3e71f2dc1..0fa12f965d 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -91,7 +91,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     default_template_name = "publish"
     template_name_profiles = None
 
-    integrated_file_sizes = {} # file_url : file_size of all published and uploaded files
+    # file_url : file_size of all published and uploaded files
+    integrated_file_sizes = {}
 
     TMP_FILE_EXT = 'tmp' # suffix to denote temporary files, use without '.'
 
@@ -105,11 +106,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             self.register(instance)
             self.log.info("Integrated Asset in to the database ...")
             self.log.info("instance.data: {}".format(instance.data))
-            self.handle_destination_files(self.integrated_file_sizes, instance, 'finalize')
+            self.handle_destination_files(self.integrated_file_sizes,
+                                          instance, 'finalize')
         except Exception as e:
             # clean destination
             self.log.critical("Error when registering",  exc_info=True)
-            self.handle_destination_files(self.integrated_file_sizes, instance, 'remove')
+            self.handle_destination_files(self.integrated_file_sizes,
+                                          instance, 'remove')
             raise
 
     def register(self, instance):
@@ -282,7 +285,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         published_representations = {}
         for idx, repre in enumerate(instance.data["representations"]):
             # reset transfers for next representation
-            # instance.data['transfers'] is used as a global variable in current codebase
+            # instance.data['transfers'] is used as a global variable
+            # in current codebase
             instance.data['transfers'] = list(orig_transfers)
 
             published_files = []
@@ -487,16 +491,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     dst_padding_exp % int(repre.get("frameStart"))
                 )
 
-            # any file that should be physically copied is expected in 'transfers' or 'hardlinks'
-            # both have same interface [[source_url, destination_url], [source_url...]]
-            if instance.data.get('transfers', False) or instance.data.get('hardlinks', False):
+            # any file that should be physically copied is expected in
+            # 'transfers' or 'hardlinks'
+            if instance.data.get('transfers', False) or \
+               instance.data.get('hardlinks', False):
                 # could throw exception, will be caught in 'process'
-                # all integration to DB is being done together lower, so no rollback needed
+                # all integration to DB is being done together lower,
+                # so no rollback needed
                 self.log.debug("Integrating source files to destination ...")
                 self.integrated_file_sizes.update(self.integrate(instance))
                 self.log.debug("Integrated files {}".format(self.integrated_file_sizes))
 
-            # get 'files' information for representation and all attached resources
+            # get 'files' info for representation and all attached resources
             self.log.debug("Preparing files information ..")
             representation["files"] = self.get_files_info(instance, self.integrated_file_sizes)
 
@@ -538,9 +544,11 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             Args:
                 instance: the instance to integrate
             Returns:
-                integrated_file_sizes: dictionary of destination file url and its size in bytes
+                integrated_file_sizes: dictionary of destination file url and
+                its size in bytes
         """
-        integrated_file_sizes = {} # store destination url and size for reporting and rollback
+        # store destination url and size for reporting and rollback
+        integrated_file_sizes = {}
         transfers = list(instance.data.get("transfers", list()))
         for src, dest in transfers:
             if os.path.normpath(src) != os.path.normpath(dest):
@@ -794,16 +802,19 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
 
     def get_rootless_path(self, anatomy, path):
-        """  Returns, if possible, path without absolute portion from host (eg. 'c:\' or '/opt/..')
+        """  Returns, if possible, path without absolute portion from host
+             (eg. 'c:\' or '/opt/..')
              This information is host dependent and shouldn't be captured.
              Example:
-                 'c:/projects/MyProject1/Assets/publish...' > '{root}/MyProject1/Assets...'
+                 'c:/projects/MyProject1/Assets/publish...' >
+                 '{root}/MyProject1/Assets...'
 
         Args:
                 anatomy: anatomy part from instance
                 path: path (absolute)
         Returns:
-                path: modified path if possible, or unmodified path + warning logged
+                path: modified path if possible, or unmodified path
+                + warning logged
         """
         success, rootless_path = (
             anatomy.find_root_template_from_path(path)
@@ -812,21 +823,25 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             path = rootless_path
         else:
             self.log.warning((
-                                 "Could not find root path for remapping \"{}\"."
-                                 " This may cause issues on farm."
+                                "Could not find root path for remapping \"{}\"."
+                                " This may cause issues on farm."
                              ).format(path))
         return path
 
     def get_files_info(self, instance, integrated_file_sizes):
         """ Prepare 'files' portion for attached resources and main asset.
-            Combining records from 'transfers' and 'hardlinks' parts from instance.
-            All attached resources should be added, currently without Context info.
+            Combining records from 'transfers' and 'hardlinks' parts from
+            instance.
+            All attached resources should be added, currently without
+            Context info.
 
         Arguments:
             instance: the current instance being published
-            integrated_file_sizes: dictionary of destination path (absolute) and its file size
+            integrated_file_sizes: dictionary of destination path (absolute)
+            and its file size
         Returns:
-            output_resources: array of dictionaries to be added to 'files' key in representation
+            output_resources: array of dictionaries to be added to 'files' key
+            in representation
         """
         resources = list(instance.data.get("transfers", []))
         resources.extend(list(instance.data.get("hardlinks", [])))
@@ -845,7 +860,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
     def get_dest_temp_url(self, dest):
         """ Enhance destination path with TMP_FILE_EXT to denote temporary file.
-            Temporary files will be renamed after successful registration into DB and full copy to destination
+            Temporary files will be renamed after successful registration
+            into DB and full copy to destination
 
         Arguments:
             dest: destination url of published file (absolute)
@@ -863,7 +879,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             path: destination url of published file (rootless)
             size(optional): size of file in bytes
             hash(optional): hash of file for synchronization validation
-            sites(optional): array of published locations, ['studio'] by default,
+            sites(optional): array of published locations, ['studio'] by default
                                 expected ['studio', 'site1', 'gdrive1']
         Returns:
             rec: dictionary with filled info
@@ -889,27 +905,35 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     def handle_destination_files(self, integrated_file_sizes, instance, mode):
         """ Clean destination files
             Called when error happened during integrating to DB or to disk
+            OR called to rename uploaded files from temporary name to final to
+            highlight publishing in progress/broken
             Used to clean unwanted files
 
         Arguments:
-            integrated_file_sizes: disctionary of uploaded files, file urls as keys
+            integrated_file_sizes: dictionary, file urls as keys, size as value
             instance: processed instance - for publish directories
-            mode: 'remove' - clean files,'finalize' - rename files, remove TMP_FILE_EXT suffix denoting temp file
+            mode: 'remove' - clean files,
+                  'finalize' - rename files,
+                               remove TMP_FILE_EXT suffix denoting temp file
         """
         if integrated_file_sizes:
             for file_url, file_size in integrated_file_sizes.items():
                 try:
                     if mode == 'remove':
                         self.log.debug("Removing file...{}".format(file_url))
-                        os.remove(file_url) # needs to be changed to Factory when sites implemented
+                        os.remove(file_url)
                     if mode == 'finalize':
                         self.log.debug("Renaming file...{}".format(file_url))
                         import re
-                        os.rename(file_url, re.sub('\.{}$'.format(self.TMP_FILE_EXT), '', file_url))  # needs to be changed to Factory when sites implemented
+                        os.rename(file_url,
+                                  re.sub('\.{}$'.format(self.TMP_FILE_EXT),
+                                         '',
+                                         file_url)
+                                  )
 
                 except FileNotFoundError:
-                    pass # file not there, nothing to delete
-                except OSError as e:
-                    self.log.critical("Cannot {} file {}".format(mode, file_url), exc_info=True)
-                    raise
-
+                    pass  # file not there, nothing to delete
+                except OSError:
+                    self.log.critical("Cannot {} file {}".format(mode, file_url)
+                                      , exc_info=True)
+                    raise
\ No newline at end of file

From a908e2867ec0ac0f1b3e11cbc8b21965cae2207b Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Fri, 12 Jun 2020 20:42:58 +0200
Subject: [PATCH 06/15] Performance testing script

Could be deleted later, no real functionality
---
 pype/tests/test_mongo_performance.py | 232 +++++++++++++++++++++++++++
 1 file changed, 232 insertions(+)
 create mode 100644 pype/tests/test_mongo_performance.py

diff --git a/pype/tests/test_mongo_performance.py b/pype/tests/test_mongo_performance.py
new file mode 100644
index 0000000000..6aa8e2ca43
--- /dev/null
+++ b/pype/tests/test_mongo_performance.py
@@ -0,0 +1,232 @@
+import pytest
+import logging
+from pprint import pprint
+import os
+import re
+import random
+import timeit
+
+import pymongo
+import bson
+
+class TestPerformance():
+    '''
+        Class for testing performance of representation and their 'files' parts.
+        Discussion is if embedded array:
+                            'files' : [ {'_id': '1111', 'path':'....},
+                                        {'_id'...}]
+                     OR documents:
+                            'files' : {
+                                            '1111': {'path':'....'},
+                                            '2222': {'path':'...'}
+                                        }
+                     is faster.
+
+        Current results: without additional partial index documents is 3x faster
+            With index is array 50x faster then document
+
+        Partial index something like:
+        db.getCollection('performance_test').createIndex
+            ({'files._id': 1},
+            {partialFilterExpresion: {'files': {'$exists': true}})
+        !DIDNT work for me, had to create manually in Compass
+
+    '''
+
+    MONGO_URL = 'mongodb://localhost:27017'
+    MONGO_DB = 'performance_test'
+    MONGO_COLLECTION = 'performance_test'
+
+    inserted_ids = []
+
+    def __init__(self, version='array'):
+        '''
+            It creates and fills collection, based on value of 'version'.
+
+        :param version: 'array' - files as embedded array,
+                        'doc' - as document
+        '''
+        self.client = pymongo.MongoClient(self.MONGO_URL)
+        self.db = self.client[self.MONGO_DB]
+        self.collection_name = self.MONGO_COLLECTION
+
+        self.version = version
+
+        if self.version != 'array':
+            self.collection_name = self.MONGO_COLLECTION + '_doc'
+
+        self.collection = self.db[self.collection_name]
+
+        self.ids = [] # for testing
+        self.inserted_ids = []
+
+    def prepare(self, no_of_records=100000):
+        '''
+            Produce 'no_of_records' of representations with 'files' segment.
+            It depends on 'version' value in constructor, 'arrray' or 'doc'
+        :return:
+        '''
+        print('Purging {} collection'.format(self.collection_name))
+        self.collection.delete_many({})
+
+        id = bson.objectid.ObjectId()
+
+        insert_recs = []
+        for i in range(no_of_records):
+            file_id =  bson.objectid.ObjectId()
+            file_id2 =  bson.objectid.ObjectId()
+            file_id3 =  bson.objectid.ObjectId()
+
+            self.inserted_ids.extend([file_id, file_id2, file_id3])
+
+            document = {"files": self.get_files(self.version, i,
+                                                file_id, file_id2, file_id3)
+                        ,
+                        "context": {
+                            "subset": "workfileLookdev",
+                            "username": "petrk",
+                            "task": "lookdev",
+                            "family": "workfile",
+                            "hierarchy": "Assets",
+                            "project": {"code": "test", "name": "Test"},
+                            "version": 1,
+                            "asset": "Cylinder",
+                            "representation": "mb",
+                            "root": "C:/projects"
+                        },
+                        "dependencies": [],
+                        "name": "mb",
+                        "parent": {"oid": '{}'.format(id)},
+                        "data": {
+                                    "path": "C:\\projects\\Test\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\v001\\test_Cylinder_workfileLookdev_v001.mb",
+                                    "template": "{root}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"
+                        },
+                        "type": "representation",
+                        "schema": "pype:representation-2.0"
+            }
+
+            insert_recs.append(document)
+
+        print('Prepared {} records in {} collection'.format(no_of_records, self.collection_name))
+        id = self.collection.insert_many(insert_recs)
+        # TODO refactore to produce real array and not needeing ugly regex
+        self.collection.insert_one({"inserted_id" : self.inserted_ids})
+        print('-' * 50)
+
+    def run(self, queries=1000, loops=3):
+        '''
+            Run X'queries' that are searching collection Y'loops' times
+        :param queries: how many times do ..find(...)
+        :param loops:  loop of testing X queries
+        :return: None
+        '''
+        print('Testing version {} on {}'.format(self.version, self.collection_name))
+
+        inserted_ids = list(self.collection.find({"inserted_id":{"$exists":True}}))
+        import re
+        self.ids = re.findall("'[0-9a-z]*'", str(inserted_ids))
+
+        import time
+
+        found_cnt = 0
+        for _ in range(loops):
+            start = time.time()
+            for i in range(queries):
+                val = random.choice(self.ids)
+                val = val.replace("'",'')
+                #print(val)
+                if (self.version == 'array'):
+                    # prepared for partial index, without 'files': exists
+                    # wont engage
+                    found = self.collection.find_one({'files': {"$exists": True},
+                                                      'files._id': "{}".format(val)})
+                else:
+                    key = "files.{}".format(val)
+                    found = self.collection.find_one({key: {"$exists": True}})
+                if found:
+                    found_cnt += 1
+
+            end = time.time()
+            print('duration per loop {}'.format(end - start))
+            print("found_cnt {}".format(found_cnt))
+
+    def get_files(self, mode, i, file_id, file_id2, file_id3):
+        '''
+            Wrapper to decide if 'array' or document version should be used
+        :param mode: 'array'|'doc'
+        :param i: step number
+        :param file_id: ObjectId of first dummy file
+        :param file_id2: ..
+        :param file_id3: ..
+        :return:
+        '''
+        if mode == 'array':
+            return self.get_files_array(i, file_id, file_id2, file_id3)
+        else:
+            return self.get_files_doc(i, file_id, file_id2, file_id3)
+
+    def get_files_array(self, i, file_id, file_id2, file_id3):
+        return [
+                                    {
+                                          "path":"c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderA_workfileLookdev_v{0:03}.mb".format(i),
+                                          "_id": '{}'.format(file_id),
+                                          "hash":"temphash",
+                                          "sites":["studio"],
+                                          "size":87236
+                                     },
+                                    {
+                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderB_workfileLookdev_v{0:03}.mb".format(
+                                            i),
+                                        "_id": '{}'.format(file_id2),
+                                        "hash": "temphash",
+                                        "sites": ["studio"],
+                                        "size": 87236
+                                    },
+                                    {
+                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderC_workfileLookdev_v{0:03}.mb".format(
+                                            i),
+                                        "_id": '{}'.format(file_id3),
+                                        "hash": "temphash",
+                                        "sites": ["studio"],
+                                        "size": 87236
+                                    }
+
+                                ]
+
+
+    def get_files_doc(self, i, file_id, file_id2, file_id3):
+        ret = {}
+        ret['{}'.format(file_id)] = {
+            "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderA_workfileLookdev_v{0:03}.mb".format(
+                i),
+            "hash": "temphash",
+            "sites": ["studio"],
+            "size": 87236
+        }
+
+
+        ret['{}'.format(file_id2)] = {
+                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderB_workfileLookdev_v{0:03}.mb".format(i),
+                                        "hash": "temphash",
+                                        "sites": ["studio"],
+                                        "size": 87236
+                                    }
+        ret['{}'.format(file_id3)] = {
+                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderC_workfileLookdev_v{0:03}.mb".format(i),
+                                        "hash": "temphash",
+                                        "sites": ["studio"],
+                                        "size": 87236
+                                    }
+
+        return ret
+
+if __name__ == '__main__':
+    tp = TestPerformance('array')
+    tp.prepare() # enable to prepare data
+    tp.run(1000, 3)
+
+    print('-'*50)
+
+    tp = TestPerformance('doc')
+    tp.prepare() # enable to prepare data
+    tp.run(1000, 3)

From b1383876564297e8c710dce89653f875cbbadef2 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 11:09:20 +0200
Subject: [PATCH 07/15] Added implementation of hash

Yanked source_hash function from extract_look into lib (and api)
---
 pype/api.py                                  | 21 ++++++--
 pype/lib.py                                  | 23 ++++++++-
 pype/plugins/global/publish/integrate_new.py | 54 ++++++++++++--------
 pype/plugins/maya/publish/extract_look.py    | 25 +--------
 4 files changed, 74 insertions(+), 49 deletions(-)

diff --git a/pype/api.py b/pype/api.py
index 2c227b5b4b..0cf2573298 100644
--- a/pype/api.py
+++ b/pype/api.py
@@ -1,5 +1,12 @@
-from .plugin import (
+from pypeapp import (
+    Logger,
+    Anatomy,
+    project_overrides_dir_path,
+    config,
+    execute
+)
 
+from .plugin import (
     Extractor,
 
     ValidatePipelineOrder,
@@ -16,8 +23,6 @@ from .action import (
     RepairContextAction
 )
 
-from pypeapp import Logger
-
 from .lib import (
     version_up,
     get_asset,
@@ -26,13 +31,20 @@ from .lib import (
     get_subsets,
     get_version_from_path,
     modified_environ,
-    add_tool_to_environment
+    add_tool_to_environment,
+    source_hash
 )
 
 # Special naming case for subprocess since its a built-in method.
 from .lib import _subprocess as subprocess
 
 __all__ = [
+    "Logger",
+    "Anatomy",
+    "project_overrides_dir_path",
+    "config",
+    "execute",
+
     # plugin classes
     "Extractor",
     # ordering
@@ -58,6 +70,7 @@ __all__ = [
     "get_version_from_path",
     "modified_environ",
     "add_tool_to_environment",
+    "source_hash",
 
     "subprocess"
 ]
diff --git a/pype/lib.py b/pype/lib.py
index 12d4706af8..006a396720 100644
--- a/pype/lib.py
+++ b/pype/lib.py
@@ -15,7 +15,7 @@ from abc import ABCMeta, abstractmethod
 from avalon import io, pipeline
 import six
 import avalon.api
-from pypeapp import config
+from .api import config
 
 log = logging.getLogger(__name__)
 
@@ -1349,3 +1349,24 @@ def ffprobe_streams(path_to_file):
     popen_output = popen.communicate()[0]
     log.debug("FFprobe output: {}".format(popen_output))
     return json.loads(popen_output)["streams"]
+
+
+def source_hash(filepath, *args):
+    """Generate simple identifier for a source file.
+    This is used to identify whether a source file has previously been
+    processe into the pipeline, e.g. a texture.
+    The hash is based on source filepath, modification time and file size.
+    This is only used to identify whether a specific source file was already
+    published before from the same location with the same modification date.
+    We opt to do it this way as opposed to Avalanch C4 hash as this is much
+    faster and predictable enough for all our production use cases.
+    Args:
+        filepath (str): The source file path.
+    You can specify additional arguments in the function
+    to allow for specific 'processing' values to be included.
+    """
+    # We replace dots with comma because . cannot be a key in a pymongo dict.
+    file_name = os.path.basename(filepath)
+    time = str(os.path.getmtime(filepath))
+    size = str(os.path.getsize(filepath))
+    return "|".join([file_name, time, size] + list(args)).replace(".", ",")
diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 0fa12f965d..7104611909 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -9,8 +9,9 @@ import six
 
 from pymongo import DeleteOne, InsertOne
 import pyblish.api
-from avalon import api, io
+from avalon import io
 from avalon.vendor import filelink
+import pype.api
 
 # this is needed until speedcopy for linux is fixed
 if sys.platform == "win32":
@@ -44,6 +45,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         "frameStart"
         "frameEnd"
         'fps'
+        "data": additional metadata for each representation.
     """
 
     label = "Integrate Asset New"
@@ -76,12 +78,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 "gizmo",
                 "source",
                 "matchmove",
-                "image"
+                "image",
                 "source",
                 "assembly",
                 "fbx",
                 "textures",
-                "action"
+                "action",
+                "harmony.template"
                 ]
     exclude_families = ["clip"]
     db_representation_context_keys = [
@@ -94,7 +97,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
     # file_url : file_size of all published and uploaded files
     integrated_file_sizes = {}
 
-    TMP_FILE_EXT = 'tmp' # suffix to denote temporary files, use without '.'
+    TMP_FILE_EXT = 'tmp'  # suffix to denote temporary files, use without '.'
 
     def process(self, instance):
         self.integrated_file_sizes = {}
@@ -107,12 +110,11 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             self.log.info("Integrated Asset in to the database ...")
             self.log.info("instance.data: {}".format(instance.data))
             self.handle_destination_files(self.integrated_file_sizes,
-                                          instance, 'finalize')
+                                          'finalize')
         except Exception as e:
             # clean destination
             self.log.critical("Error when registering",  exc_info=True)
-            self.handle_destination_files(self.integrated_file_sizes,
-                                          instance, 'remove')
+            self.handle_destination_files(self.integrated_file_sizes, 'remove')
             raise
 
     def register(self, instance):
@@ -394,9 +396,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                         index_frame_start += 1
 
                     dst = "{0}{1}{2}".format(
-                            dst_head,
-                            dst_padding,
-                            dst_tail).replace("..", ".")
+                        dst_head,
+                        dst_padding,
+                        dst_tail
+                    ).replace("..", ".")
 
                     self.log.debug("destination: `{}`".format(dst))
                     src = os.path.join(stagingdir, src_file_name)
@@ -469,13 +472,15 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             if repre_id is None:
                 repre_id = io.ObjectId()
 
+            data = repre.get("data") or {}
+            data.update({'path': dst, 'template': template})
             representation = {
                 "_id": repre_id,
                 "schema": "pype:representation-2.0",
                 "type": "representation",
                 "parent": version_id,
                 "name": repre['name'],
-                "data": {'path': dst, 'template': template},
+                "data": data,
                 "dependencies": instance.data.get("dependencies", "").split(),
 
                 # Imprint shortcut to context
@@ -500,11 +505,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 # so no rollback needed
                 self.log.debug("Integrating source files to destination ...")
                 self.integrated_file_sizes.update(self.integrate(instance))
-                self.log.debug("Integrated files {}".format(self.integrated_file_sizes))
+                self.log.debug("Integrated files {}".
+                               format(self.integrated_file_sizes))
 
             # get 'files' info for representation and all attached resources
             self.log.debug("Preparing files information ..")
-            representation["files"] = self.get_files_info(instance, self.integrated_file_sizes)
+            representation["files"] = self.get_files_info(
+                                                    instance,
+                                                    self.integrated_file_sizes)
 
             self.log.debug("__ representation: {}".format(representation))
             destination_list.append(dst)
@@ -800,7 +808,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
         return template_name
 
-
     def get_rootless_path(self, anatomy, path):
         """  Returns, if possible, path without absolute portion from host
              (eg. 'c:\' or '/opt/..')
@@ -846,15 +853,21 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         resources = list(instance.data.get("transfers", []))
         resources.extend(list(instance.data.get("hardlinks", [])))
 
-        self.log.debug("get_resource_files_info.resources: {}".format(resources))
+        self.log.debug("get_resource_files_info.resources:{}".format(resources))
 
         output_resources = []
         anatomy = instance.context.data["anatomy"]
         for src, dest in resources:
-            # TODO - hash or use self.integrated_file_size
             path = self.get_rootless_path(anatomy, dest)
             dest = self.get_dest_temp_url(dest)
-            output_resources.append(self.prepare_file_info(path, integrated_file_sizes[dest], 'temphash'))
+            hash = pype.api.source_hash(dest)
+            if self.TMP_FILE_EXT and ',{}'.format(self.TMP_FILE_EXT) in hash:
+                hash = hash.replace(',{}'.format(self.TMP_FILE_EXT), '')
+
+            file_info = self.prepare_file_info(path,
+                                               integrated_file_sizes[dest],
+                                               hash)
+            output_resources.append(file_info)
 
         return output_resources
 
@@ -872,7 +885,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             dest += '.{}'.format(self.TMP_FILE_EXT)
         return dest
 
-    def prepare_file_info(self, path, size = None, hash = None, sites = None):
+    def prepare_file_info(self, path, size=None, hash=None, sites=None):
         """ Prepare information for one file (asset or resource)
 
         Arguments:
@@ -902,7 +915,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
         return rec
 
-    def handle_destination_files(self, integrated_file_sizes, instance, mode):
+    def handle_destination_files(self, integrated_file_sizes, mode):
         """ Clean destination files
             Called when error happened during integrating to DB or to disk
             OR called to rename uploaded files from temporary name to final to
@@ -911,7 +924,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
         Arguments:
             integrated_file_sizes: dictionary, file urls as keys, size as value
-            instance: processed instance - for publish directories
             mode: 'remove' - clean files,
                   'finalize' - rename files,
                                remove TMP_FILE_EXT suffix denoting temp file
@@ -936,4 +948,4 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 except OSError:
                     self.log.critical("Cannot {} file {}".format(mode, file_url)
                                       , exc_info=True)
-                    raise
\ No newline at end of file
+                    raise
diff --git a/pype/plugins/maya/publish/extract_look.py b/pype/plugins/maya/publish/extract_look.py
index 58196433aa..6bd202093f 100644
--- a/pype/plugins/maya/publish/extract_look.py
+++ b/pype/plugins/maya/publish/extract_look.py
@@ -14,34 +14,13 @@ import avalon.maya
 from avalon import io, api
 
 import pype.api
-import pype.maya.lib as lib
+from pype.hosts.maya import lib
 
 # Modes for transfer
 COPY = 1
 HARDLINK = 2
 
 
-def source_hash(filepath, *args):
-    """Generate simple identifier for a source file.
-    This is used to identify whether a source file has previously been
-    processe into the pipeline, e.g. a texture.
-    The hash is based on source filepath, modification time and file size.
-    This is only used to identify whether a specific source file was already
-    published before from the same location with the same modification date.
-    We opt to do it this way as opposed to Avalanch C4 hash as this is much
-    faster and predictable enough for all our production use cases.
-    Args:
-        filepath (str): The source file path.
-    You can specify additional arguments in the function
-    to allow for specific 'processing' values to be included.
-    """
-    # We replace dots with comma because . cannot be a key in a pymongo dict.
-    file_name = os.path.basename(filepath)
-    time = str(os.path.getmtime(filepath))
-    size = str(os.path.getsize(filepath))
-    return "|".join([file_name, time, size] + list(args)).replace(".", ",")
-
-
 def find_paths_by_hash(texture_hash):
     # Find the texture hash key in the dictionary and all paths that
     # originate from it.
@@ -363,7 +342,7 @@ class ExtractLook(pype.api.Extractor):
         args = []
         if do_maketx:
             args.append("maketx")
-        texture_hash = source_hash(filepath, *args)
+        texture_hash = pype.api.source_hash(filepath, *args)
 
         # If source has been published before with the same settings,
         # then don't reprocess but hardlink from the original

From 2aaae63f80470957552571d27f0e99c719e8f82f Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 11:41:26 +0200
Subject: [PATCH 08/15] Fix - change level of logging

---
 pype/plugins/global/publish/integrate_new.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 7104611909..a7ff3e5748 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -946,6 +946,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 except FileNotFoundError:
                     pass  # file not there, nothing to delete
                 except OSError:
-                    self.log.critical("Cannot {} file {}".format(mode, file_url)
+                    self.log.error("Cannot {} file {}".format(mode, file_url)
                                       , exc_info=True)
                     raise

From eafa79af88428867881204f4ddfcf361ef300cef Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 11:42:20 +0200
Subject: [PATCH 09/15] Taming of Hound

---
 pype/plugins/global/publish/integrate_new.py |   2 +-
 pype/tests/test_mongo_performance.py         | 134 ++++++++++---------
 2 files changed, 70 insertions(+), 66 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index a7ff3e5748..c5ef2c3530 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -947,5 +947,5 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     pass  # file not there, nothing to delete
                 except OSError:
                     self.log.error("Cannot {} file {}".format(mode, file_url)
-                                      , exc_info=True)
+                                   , exc_info=True)
                     raise
diff --git a/pype/tests/test_mongo_performance.py b/pype/tests/test_mongo_performance.py
index 6aa8e2ca43..6b62f0fd1c 100644
--- a/pype/tests/test_mongo_performance.py
+++ b/pype/tests/test_mongo_performance.py
@@ -1,13 +1,7 @@
-import pytest
-import logging
-from pprint import pprint
-import os
-import re
-import random
-import timeit
-
 import pymongo
 import bson
+import random
+
 
 class TestPerformance():
     '''
@@ -57,7 +51,7 @@ class TestPerformance():
 
         self.collection = self.db[self.collection_name]
 
-        self.ids = [] # for testing
+        self.ids = []  # for testing
         self.inserted_ids = []
 
     def prepare(self, no_of_records=100000):
@@ -73,9 +67,9 @@ class TestPerformance():
 
         insert_recs = []
         for i in range(no_of_records):
-            file_id =  bson.objectid.ObjectId()
-            file_id2 =  bson.objectid.ObjectId()
-            file_id3 =  bson.objectid.ObjectId()
+            file_id = bson.objectid.ObjectId()
+            file_id2 = bson.objectid.ObjectId()
+            file_id3 = bson.objectid.ObjectId()
 
             self.inserted_ids.extend([file_id, file_id2, file_id3])
 
@@ -98,19 +92,21 @@ class TestPerformance():
                         "name": "mb",
                         "parent": {"oid": '{}'.format(id)},
                         "data": {
-                                    "path": "C:\\projects\\Test\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\v001\\test_Cylinder_workfileLookdev_v001.mb",
-                                    "template": "{root}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"
+                            "path": "C:\\projects\\Test\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\v001\\test_Cylinder_workfileLookdev_v001.mb",
+                            "template": "{root}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"
                         },
                         "type": "representation",
                         "schema": "pype:representation-2.0"
-            }
+                        }
 
             insert_recs.append(document)
 
-        print('Prepared {} records in {} collection'.format(no_of_records, self.collection_name))
-        id = self.collection.insert_many(insert_recs)
+        print('Prepared {} records in {} collection'.
+              format(no_of_records, self.collection_name))
+
+        self.collection.insert_many(insert_recs)
         # TODO refactore to produce real array and not needeing ugly regex
-        self.collection.insert_one({"inserted_id" : self.inserted_ids})
+        self.collection.insert_one({"inserted_id": self.inserted_ids})
         print('-' * 50)
 
     def run(self, queries=1000, loops=3):
@@ -120,9 +116,11 @@ class TestPerformance():
         :param loops:  loop of testing X queries
         :return: None
         '''
-        print('Testing version {} on {}'.format(self.version, self.collection_name))
+        print('Testing version {} on {}'.format(self.version,
+                                                self.collection_name))
 
-        inserted_ids = list(self.collection.find({"inserted_id":{"$exists":True}}))
+        inserted_ids = list(self.collection.
+                            find({"inserted_id": {"$exists": True}}))
         import re
         self.ids = re.findall("'[0-9a-z]*'", str(inserted_ids))
 
@@ -131,15 +129,16 @@ class TestPerformance():
         found_cnt = 0
         for _ in range(loops):
             start = time.time()
-            for i in range(queries):
+            for _ in range(queries):
                 val = random.choice(self.ids)
-                val = val.replace("'",'')
-                #print(val)
+                val = val.replace("'", '')
+
                 if (self.version == 'array'):
                     # prepared for partial index, without 'files': exists
                     # wont engage
-                    found = self.collection.find_one({'files': {"$exists": True},
-                                                      'files._id': "{}".format(val)})
+                    found = self.collection.\
+                        find_one({'files': {"$exists": True},
+                                  'files._id': "{}".format(val)})
                 else:
                     key = "files.{}".format(val)
                     found = self.collection.find_one({key: {"$exists": True}})
@@ -167,66 +166,71 @@ class TestPerformance():
 
     def get_files_array(self, i, file_id, file_id2, file_id3):
         return [
-                                    {
-                                          "path":"c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderA_workfileLookdev_v{0:03}.mb".format(i),
-                                          "_id": '{}'.format(file_id),
-                                          "hash":"temphash",
-                                          "sites":["studio"],
-                                          "size":87236
-                                     },
-                                    {
-                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderB_workfileLookdev_v{0:03}.mb".format(
-                                            i),
-                                        "_id": '{}'.format(file_id2),
-                                        "hash": "temphash",
-                                        "sites": ["studio"],
-                                        "size": 87236
-                                    },
-                                    {
-                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderC_workfileLookdev_v{0:03}.mb".format(
-                                            i),
-                                        "_id": '{}'.format(file_id3),
-                                        "hash": "temphash",
-                                        "sites": ["studio"],
-                                        "size": 87236
-                                    }
-
-                                ]
+            {
+                 "path": "c:/Test/Assets/Cylinder/publish/workfile/"
+                         "workfileLookdev/v001/"
+                         "test_CylinderA_workfileLookdev_v{0:03}.mb".format(i),
+                 "_id": '{}'.format(file_id),
+                 "hash": "temphash",
+                 "sites": ["studio"],
+                 "size":87236
+            },
+            {
+                "path": "c:/Test/Assets/Cylinder/publish/workfile/"
+                        "workfileLookdev/v001/"
+                        "test_CylinderB_workfileLookdev_v{0:03}.mb".format(i),
+                "_id": '{}'.format(file_id2),
+                "hash": "temphash",
+                "sites": ["studio"],
+                "size": 87236
+            },
+            {
+                "path": "c:/Test/Assets/Cylinder/publish/workfile/"
+                        "workfileLookdev/v001/"
+                        "test_CylinderC_workfileLookdev_v{0:03}.mb".format(i),
+                "_id": '{}'.format(file_id3),
+                "hash": "temphash",
+                "sites": ["studio"],
+                "size": 87236
+            }
 
+            ]
 
     def get_files_doc(self, i, file_id, file_id2, file_id3):
         ret = {}
         ret['{}'.format(file_id)] = {
-            "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderA_workfileLookdev_v{0:03}.mb".format(
-                i),
+            "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/"
+                    "v001/test_CylinderA_workfileLookdev_v{0:03}.mb".format(i),
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236
         }
 
-
         ret['{}'.format(file_id2)] = {
-                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderB_workfileLookdev_v{0:03}.mb".format(i),
-                                        "hash": "temphash",
-                                        "sites": ["studio"],
-                                        "size": 87236
-                                    }
+            "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/"
+                    "v001/test_CylinderB_workfileLookdev_v{0:03}.mb".format(i),
+            "hash": "temphash",
+            "sites": ["studio"],
+            "size": 87236
+        }
         ret['{}'.format(file_id3)] = {
-                                        "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/v001/test_CylinderC_workfileLookdev_v{0:03}.mb".format(i),
-                                        "hash": "temphash",
-                                        "sites": ["studio"],
-                                        "size": 87236
-                                    }
+            "path": "c:/Test/Assets/Cylinder/publish/workfile/workfileLookdev/"
+                    "v001/test_CylinderC_workfileLookdev_v{0:03}.mb".format(i),
+            "hash": "temphash",
+            "sites": ["studio"],
+            "size": 87236
+        }
 
         return ret
 
+
 if __name__ == '__main__':
     tp = TestPerformance('array')
-    tp.prepare() # enable to prepare data
+    tp.prepare()  # enable to prepare data
     tp.run(1000, 3)
 
     print('-'*50)
 
     tp = TestPerformance('doc')
-    tp.prepare() # enable to prepare data
+    tp.prepare()  # enable to prepare data
     tp.run(1000, 3)

From 42c77d0e13aead8cd0d1d27285f60ed07909b377 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 12:04:30 +0200
Subject: [PATCH 10/15] Fix - change level of logging on proper place

---
 pype/plugins/global/publish/integrate_new.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index c5ef2c3530..bb58479d64 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -600,8 +600,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         except OSError as e:
             if e.errno == errno.EEXIST:
                 pass
-            else:
-                self.log.critical("An unexpected error occurred.")
+            else:# clean destination
+                self.log.error("An unexpected error occurred.")
                 raise
 
         # copy file with speedcopy and check if size of files are simetrical

From fe9ed4c09e6cd04378a1b3f2887bf5a0e011c7a0 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 12:16:14 +0200
Subject: [PATCH 11/15] Fix - unwanted change of existing logging

---
 pype/plugins/global/publish/integrate_new.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index bb58479d64..c5ef2c3530 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -600,8 +600,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         except OSError as e:
             if e.errno == errno.EEXIST:
                 pass
-            else:# clean destination
-                self.log.error("An unexpected error occurred.")
+            else:
+                self.log.critical("An unexpected error occurred.")
                 raise
 
         # copy file with speedcopy and check if size of files are simetrical

From ca78a5ca2a28aa0c55163b2d5fcc34e595759443 Mon Sep 17 00:00:00 2001
From: "petr.kalis" <petr.kalis@gmail.com>
Date: Thu, 18 Jun 2020 12:40:57 +0200
Subject: [PATCH 12/15] Fix - switched from raise to six.reraise

Small formatting changes
---
 pype/plugins/global/publish/integrate_new.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index c5ef2c3530..5c48770f99 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -115,7 +115,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             # clean destination
             self.log.critical("Error when registering",  exc_info=True)
             self.handle_destination_files(self.integrated_file_sizes, 'remove')
-            raise
+            six.reraise(*sys.exc_info())
 
     def register(self, instance):
         # Required environment variables
@@ -509,7 +509,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                                format(self.integrated_file_sizes))
 
             # get 'files' info for representation and all attached resources
-            self.log.debug("Preparing files information ..")
+            self.log.debug("Preparing files information ...")
             representation["files"] = self.get_files_info(
                                                     instance,
                                                     self.integrated_file_sizes)
@@ -573,7 +573,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         hardlinks = instance.data.get("hardlinks", list())
         for src, dest in hardlinks:
             dest = self.get_dest_temp_url(dest)
-            self.log.debug("Hardlinking file .. {} -> {}".format(src, dest))
+            self.log.debug("Hardlinking file ... {} -> {}".format(src, dest))
             if not os.path.exists(dest):
                 self.hardlink_file(src, dest)
 
@@ -593,7 +593,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         """
         src = os.path.normpath(src)
         dst = os.path.normpath(dst)
-        self.log.debug("Copying file .. {} -> {}".format(src, dst))
+        self.log.debug("Copying file ... {} -> {}".format(src, dst))
         dirname = os.path.dirname(dst)
         try:
             os.makedirs(dirname)
@@ -602,7 +602,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 pass
             else:
                 self.log.critical("An unexpected error occurred.")
-                raise
+                six.reraise(*sys.exc_info())
 
         # copy file with speedcopy and check if size of files are simetrical
         while True:
@@ -625,7 +625,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 pass
             else:
                 self.log.critical("An unexpected error occurred.")
-                raise
+                six.reraise(*sys.exc_info())
 
         filelink.create(src, dst, filelink.HARDLINK)
 
@@ -638,7 +638,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         })
 
         if subset is None:
-            self.log.info("Subset '%s' not found, creating.." % subset_name)
+            self.log.info("Subset '%s' not found, creating ..." % subset_name)
             self.log.debug("families.  %s" % instance.data.get('families'))
             self.log.debug(
                 "families.  %s" % type(instance.data.get('families')))
@@ -932,10 +932,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             for file_url, file_size in integrated_file_sizes.items():
                 try:
                     if mode == 'remove':
-                        self.log.debug("Removing file...{}".format(file_url))
+                        self.log.debug("Removing file ...{}".format(file_url))
                         os.remove(file_url)
                     if mode == 'finalize':
-                        self.log.debug("Renaming file...{}".format(file_url))
+                        self.log.debug("Renaming file ...{}".format(file_url))
                         import re
                         os.rename(file_url,
                                   re.sub('\.{}$'.format(self.TMP_FILE_EXT),
@@ -948,4 +948,4 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 except OSError:
                     self.log.error("Cannot {} file {}".format(mode, file_url)
                                    , exc_info=True)
-                    raise
+                    six.reraise(*sys.exc_info())

From 96f7cb395979d293fb1fe514e329b56e3e16ccd4 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Wed, 12 Aug 2020 13:50:15 +0200
Subject: [PATCH 13/15] Fix conflicts from 'develop'

---
 pype/api.py                                  | 22 ++++-
 pype/lib.py                                  | 84 +++++++++++++++++---
 pype/plugins/global/publish/integrate_new.py | 46 ++++++-----
 3 files changed, 122 insertions(+), 30 deletions(-)

diff --git a/pype/api.py b/pype/api.py
index 0cf2573298..c722757a3c 100644
--- a/pype/api.py
+++ b/pype/api.py
@@ -6,6 +6,14 @@ from pypeapp import (
     execute
 )
 
+from pypeapp.lib.mongo import (
+    decompose_url,
+    compose_url,
+    get_default_components
+)
+
+from . import resources
+
 from .plugin import (
     Extractor,
 
@@ -30,9 +38,11 @@ from .lib import (
     get_hierarchy,
     get_subsets,
     get_version_from_path,
+    get_last_version_from_path,
     modified_environ,
     add_tool_to_environment,
-    source_hash
+    source_hash,
+    get_latest_version
 )
 
 # Special naming case for subprocess since its a built-in method.
@@ -44,6 +54,12 @@ __all__ = [
     "project_overrides_dir_path",
     "config",
     "execute",
+    "decompose_url",
+    "compose_url",
+    "get_default_components",
+
+    # Resources
+    "resources",
 
     # plugin classes
     "Extractor",
@@ -68,9 +84,11 @@ __all__ = [
     "get_asset",
     "get_subsets",
     "get_version_from_path",
+    "get_last_version_from_path",
     "modified_environ",
     "add_tool_to_environment",
     "source_hash",
 
-    "subprocess"
+    "subprocess",
+    "get_latest_version"
 ]
diff --git a/pype/lib.py b/pype/lib.py
index 006a396720..ec0fb84f84 100644
--- a/pype/lib.py
+++ b/pype/lib.py
@@ -469,6 +469,43 @@ def get_version_from_path(file):
         )
 
 
+def get_last_version_from_path(path_dir, filter):
+    """
+    Finds last version of given directory content
+
+    Args:
+        path_dir (string): directory path
+        filter (list): list of strings used as file name filter
+
+    Returns:
+        string: file name with last version
+
+    Example:
+        last_version_file = get_last_version_from_path(
+            "/project/shots/shot01/work", ["shot01", "compositing", "nk"])
+    """
+
+    assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory"
+    assert isinstance(filter, list) and (
+        len(filter) != 0), "`filter` argument needs to be list and not empty"
+
+    filtred_files = list()
+
+    # form regex for filtering
+    patern = r".*".join(filter)
+
+    for f in os.listdir(path_dir):
+        if not re.findall(patern, f):
+            continue
+        filtred_files.append(f)
+
+    if filtred_files:
+        sorted(filtred_files)
+        return filtred_files[-1]
+    else:
+        return None
+
+
 def get_avalon_database():
     if io._database is None:
         set_io_database()
@@ -482,14 +519,6 @@ def set_io_database():
     io.install()
 
 
-def get_all_avalon_projects():
-    db = get_avalon_database()
-    projects = []
-    for name in db.collection_names():
-        projects.append(db[name].find_one({'type': 'project'}))
-    return projects
-
-
 def filter_pyblish_plugins(plugins):
     """
     This servers as plugin filter / modifier for pyblish. It will load plugin
@@ -610,7 +639,7 @@ def get_subsets(asset_name,
 
         if len(repres_out) > 0:
             output_dict[subset["name"]] = {"version": version_sel,
-                                           "representaions": repres_out}
+                                           "representations": repres_out}
 
     return output_dict
 
@@ -1350,7 +1379,6 @@ def ffprobe_streams(path_to_file):
     log.debug("FFprobe output: {}".format(popen_output))
     return json.loads(popen_output)["streams"]
 
-
 def source_hash(filepath, *args):
     """Generate simple identifier for a source file.
     This is used to identify whether a source file has previously been
@@ -1370,3 +1398,39 @@ def source_hash(filepath, *args):
     time = str(os.path.getmtime(filepath))
     size = str(os.path.getsize(filepath))
     return "|".join([file_name, time, size] + list(args)).replace(".", ",")
+
+def get_latest_version(asset_name, subset_name):
+    """Retrieve latest version from `asset_name`, and `subset_name`.
+
+    Args:
+        asset_name (str): Name of asset.
+        subset_name (str): Name of subset.
+    """
+    # Get asset
+    asset_name = io.find_one(
+        {"type": "asset", "name": asset_name}, projection={"name": True}
+    )
+
+    subset = io.find_one(
+        {"type": "subset", "name": subset_name, "parent": asset_name["_id"]},
+        projection={"_id": True, "name": True},
+    )
+
+    # Check if subsets actually exists.
+    assert subset, "No subsets found."
+
+    # Get version
+    version_projection = {
+        "name": True,
+        "parent": True,
+    }
+
+    version = io.find_one(
+        {"type": "version", "parent": subset["_id"]},
+        projection=version_projection,
+        sort=[("name", -1)],
+    )
+
+    assert version, "No version found, this is a bug"
+
+    return version
diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 5c48770f99..2cf09474b8 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -84,7 +84,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 "fbx",
                 "textures",
                 "action",
-                "harmony.template"
+                "harmony.template",
+                "harmony.palette",
+                "editorial"
                 ]
     exclude_families = ["clip"]
     db_representation_context_keys = [
@@ -111,7 +113,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             self.log.info("instance.data: {}".format(instance.data))
             self.handle_destination_files(self.integrated_file_sizes,
                                           'finalize')
-        except Exception as e:
+        except Exception:
             # clean destination
             self.log.critical("Error when registering",  exc_info=True)
             self.handle_destination_files(self.integrated_file_sizes, 'remove')
@@ -155,6 +157,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if task_name:
             anatomy_data["task"] = task_name
 
+        anatomy_data["family"] = instance.data.get("family")
+
         stagingdir = instance.data.get("stagingDir")
         if not stagingdir:
             self.log.info((
@@ -398,8 +402,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     dst = "{0}{1}{2}".format(
                         dst_head,
                         dst_padding,
-                        dst_tail
-                    ).replace("..", ".")
+                        dst_tail).replace("..", ".")
 
                     self.log.debug("destination: `{}`".format(dst))
                     src = os.path.join(stagingdir, src_file_name)
@@ -606,12 +609,19 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
 
         # copy file with speedcopy and check if size of files are simetrical
         while True:
+            import shutil
             try:
                 copyfile(src, dst)
-            except OSError as e:
-                self.log.critical("Cannot copy {} to {}".format(src, dst))
-                self.log.critical(e)
-                six.reraise(*sys.exc_info())
+            except shutil.SameFileError as sfe:
+                self.log.critical("files are the same {} to {}".format(src, dst))
+                os.remove(dst)
+                try:
+                    shutil.copyfile(src, dst)
+                    self.log.debug("Copying files with shutil...")
+                except (OSError) as e:
+                    self.log.critical("Cannot copy {} to {}".format(src, dst))
+                    self.log.critical(e)
+                    six.reraise(*sys.exc_info())
             if str(getsize(src)) in str(getsize(dst)):
                 break
 
@@ -648,7 +658,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 "type": "subset",
                 "name": subset_name,
                 "data": {
-                    "families": instance.data.get('families')
+                    "families": instance.data.get("families", [])
                 },
                 "parent": asset["_id"]
             }).inserted_id
@@ -761,7 +771,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         task_name = io.Session.get("AVALON_TASK")
         family = self.main_family_from_instance(instance)
 
-        matching_profiles = None
+        matching_profiles = {}
         highest_value = -1
         self.log.info(self.template_name_profiles)
         for name, filters in self.template_name_profiles.items():
@@ -860,13 +870,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         for src, dest in resources:
             path = self.get_rootless_path(anatomy, dest)
             dest = self.get_dest_temp_url(dest)
-            hash = pype.api.source_hash(dest)
-            if self.TMP_FILE_EXT and ',{}'.format(self.TMP_FILE_EXT) in hash:
-                hash = hash.replace(',{}'.format(self.TMP_FILE_EXT), '')
+            file_hash = pype.api.source_hash(dest)
+            if self.TMP_FILE_EXT and ',{}'.format(self.TMP_FILE_EXT) in file_hash:
+                file_hash = file_hash.replace(',{}'.format(self.TMP_FILE_EXT), '')
 
             file_info = self.prepare_file_info(path,
                                                integrated_file_sizes[dest],
-                                               hash)
+                                               file_hash)
             output_resources.append(file_info)
 
         return output_resources
@@ -885,13 +895,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             dest += '.{}'.format(self.TMP_FILE_EXT)
         return dest
 
-    def prepare_file_info(self, path, size=None, hash=None, sites=None):
+    def prepare_file_info(self, path, size=None, file_hash=None, sites=None):
         """ Prepare information for one file (asset or resource)
 
         Arguments:
             path: destination url of published file (rootless)
             size(optional): size of file in bytes
-            hash(optional): hash of file for synchronization validation
+            file_hash(optional): hash of file for synchronization validation
             sites(optional): array of published locations, ['studio'] by default
                                 expected ['studio', 'site1', 'gdrive1']
         Returns:
@@ -905,8 +915,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if size:
             rec["size"] = size
 
-        if hash:
-            rec["hash"] = hash
+        if file_hash:
+            rec["hash"] = file_hash
 
         if sites:
             rec["sites"] = sites

From d37eac4a226444fa5b49798f6e8b39d8b6303219 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Wed, 12 Aug 2020 13:51:28 +0200
Subject: [PATCH 14/15] Modified 'sites' element to object of objects for
 better syncing

---
 pype/plugins/global/publish/integrate_new.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index 2cf09474b8..fb1588f7d8 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -12,6 +12,7 @@ import pyblish.api
 from avalon import io
 from avalon.vendor import filelink
 import pype.api
+from datetime import datetime
 
 # this is needed until speedcopy for linux is fixed
 if sys.platform == "win32":
@@ -902,13 +903,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             path: destination url of published file (rootless)
             size(optional): size of file in bytes
             file_hash(optional): hash of file for synchronization validation
-            sites(optional): array of published locations, ['studio'] by default
-                                expected ['studio', 'site1', 'gdrive1']
+            sites(optional): array of published locations, ['studio': {'created_dt':date}] by default
+                                keys expected ['studio', 'site1', 'gdrive1']
         Returns:
             rec: dictionary with filled info
         """
 
-        rec = {  # TODO update collection step to extend with necessary values
+        rec = {
             "_id": io.ObjectId(),
             "path": path
         }
@@ -921,7 +922,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if sites:
             rec["sites"] = sites
         else:
-            rec["sites"] = ["studio"]
+            meta = {"created_dt": datetime.now()}
+            rec["sites"] = {"studio": meta}
 
         return rec
 

From b7148c08947ca8acd20c6658f34e6562fa7a56e0 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Wed, 12 Aug 2020 14:21:26 +0200
Subject: [PATCH 15/15] Hound

---
 pype/lib.py                                  |  2 +
 pype/plugins/global/publish/integrate_new.py | 42 +++++++++++---------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/pype/lib.py b/pype/lib.py
index ec0fb84f84..1dd5dd9de5 100644
--- a/pype/lib.py
+++ b/pype/lib.py
@@ -1379,6 +1379,7 @@ def ffprobe_streams(path_to_file):
     log.debug("FFprobe output: {}".format(popen_output))
     return json.loads(popen_output)["streams"]
 
+
 def source_hash(filepath, *args):
     """Generate simple identifier for a source file.
     This is used to identify whether a source file has previously been
@@ -1399,6 +1400,7 @@ def source_hash(filepath, *args):
     size = str(os.path.getsize(filepath))
     return "|".join([file_name, time, size] + list(args)).replace(".", ",")
 
+
 def get_latest_version(asset_name, subset_name):
     """Retrieve latest version from `asset_name`, and `subset_name`.
 
diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py
index fb1588f7d8..88267e1b0a 100644
--- a/pype/plugins/global/publish/integrate_new.py
+++ b/pype/plugins/global/publish/integrate_new.py
@@ -116,7 +116,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                                           'finalize')
         except Exception:
             # clean destination
-            self.log.critical("Error when registering",  exc_info=True)
+            self.log.critical("Error when registering", exc_info=True)
             self.handle_destination_files(self.integrated_file_sizes, 'remove')
             six.reraise(*sys.exc_info())
 
@@ -515,8 +515,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             # get 'files' info for representation and all attached resources
             self.log.debug("Preparing files information ...")
             representation["files"] = self.get_files_info(
-                                                    instance,
-                                                    self.integrated_file_sizes)
+                                           instance,
+                                           self.integrated_file_sizes)
 
             self.log.debug("__ representation: {}".format(representation))
             destination_list.append(dst)
@@ -613,13 +613,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             import shutil
             try:
                 copyfile(src, dst)
-            except shutil.SameFileError as sfe:
-                self.log.critical("files are the same {} to {}".format(src, dst))
+            except shutil.SameFileError:
+                self.log.critical("files are the same {} to {}".format(src,
+                                                                       dst))
                 os.remove(dst)
                 try:
                     shutil.copyfile(src, dst)
                     self.log.debug("Copying files with shutil...")
-                except (OSError) as e:
+                except OSError as e:
                     self.log.critical("Cannot copy {} to {}".format(src, dst))
                     self.log.critical(e)
                     six.reraise(*sys.exc_info())
@@ -841,9 +842,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             path = rootless_path
         else:
             self.log.warning((
-                                "Could not find root path for remapping \"{}\"."
-                                " This may cause issues on farm."
-                             ).format(path))
+                              "Could not find root path for remapping \"{}\"."
+                              " This may cause issues on farm."
+                              ).format(path))
         return path
 
     def get_files_info(self, instance, integrated_file_sizes):
@@ -864,16 +865,19 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         resources = list(instance.data.get("transfers", []))
         resources.extend(list(instance.data.get("hardlinks", [])))
 
-        self.log.debug("get_resource_files_info.resources:{}".format(resources))
+        self.log.debug("get_resource_files_info.resources:{}".
+                       format(resources))
 
         output_resources = []
         anatomy = instance.context.data["anatomy"]
-        for src, dest in resources:
+        for _src, dest in resources:
             path = self.get_rootless_path(anatomy, dest)
             dest = self.get_dest_temp_url(dest)
             file_hash = pype.api.source_hash(dest)
-            if self.TMP_FILE_EXT and ',{}'.format(self.TMP_FILE_EXT) in file_hash:
-                file_hash = file_hash.replace(',{}'.format(self.TMP_FILE_EXT), '')
+            if self.TMP_FILE_EXT and \
+               ',{}'.format(self.TMP_FILE_EXT) in file_hash:
+                file_hash = file_hash.replace(',{}'.format(self.TMP_FILE_EXT),
+                                              '')
 
             file_info = self.prepare_file_info(path,
                                                integrated_file_sizes[dest],
@@ -883,7 +887,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         return output_resources
 
     def get_dest_temp_url(self, dest):
-        """ Enhance destination path with TMP_FILE_EXT to denote temporary file.
+        """ Enhance destination path with TMP_FILE_EXT to denote temporary
+            file.
             Temporary files will be renamed after successful registration
             into DB and full copy to destination
 
@@ -903,7 +908,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             path: destination url of published file (rootless)
             size(optional): size of file in bytes
             file_hash(optional): hash of file for synchronization validation
-            sites(optional): array of published locations, ['studio': {'created_dt':date}] by default
+            sites(optional): array of published locations,
+                            ['studio': {'created_dt':date}] by default
                                 keys expected ['studio', 'site1', 'gdrive1']
         Returns:
             rec: dictionary with filled info
@@ -941,7 +947,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                                remove TMP_FILE_EXT suffix denoting temp file
         """
         if integrated_file_sizes:
-            for file_url, file_size in integrated_file_sizes.items():
+            for file_url, _file_size in integrated_file_sizes.items():
                 try:
                     if mode == 'remove':
                         self.log.debug("Removing file ...{}".format(file_url))
@@ -958,6 +964,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 except FileNotFoundError:
                     pass  # file not there, nothing to delete
                 except OSError:
-                    self.log.error("Cannot {} file {}".format(mode, file_url)
-                                   , exc_info=True)
+                    self.log.error("Cannot {} file {}".format(mode, file_url),
+                                   exc_info=True)
                     six.reraise(*sys.exc_info())