From c43a58efa9394bcf4d62575161fec9eedcd45889 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Thu, 5 Nov 2020 19:27:56 +0100
Subject: [PATCH 01/10] feat(SP): wip editorial expansion to image sequences

---
 .../publish/collect_editorial.py              |  67 +++++++++---
 .../publish/collect_hierarchy.py              |  58 +++++++---
 .../publish/collect_instance_data.py          |   2 +-
 .../publish/collect_instance_resources.py     |  57 ++++++++++
 ...clip_instances.py => collect_instances.py} |  31 +++---
 .../publish/extract_shot_data.py              |  92 ----------------
 .../publish/extract_trim_video_audio.py       | 101 ++++++++++++++++++
 .../publish/validate_editorial_resources.py   |   2 +-
 8 files changed, 277 insertions(+), 133 deletions(-)
 create mode 100644 pype/plugins/standalonepublisher/publish/collect_instance_resources.py
 rename pype/plugins/standalonepublisher/publish/{collect_clip_instances.py => collect_instances.py} (89%)
 delete mode 100644 pype/plugins/standalonepublisher/publish/extract_shot_data.py
 create mode 100644 pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py

diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py
index 5e6fd106e4..7e532c3741 100644
--- a/pype/plugins/standalonepublisher/publish/collect_editorial.py
+++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py
@@ -1,3 +1,19 @@
+"""
+Optional:
+    presets     -> extensions (
+        example of use:
+            [".mov", ".mp4"]
+    )
+    presets     -> source_dir (
+        example of use:
+            "C:/pathToFolder"
+            "{root}/{project[name]}/inputs"
+            "{root[work]}/{project[name]}/inputs"
+            "./input"
+            "../input"
+    )
+"""
+
 import os
 import opentimelineio as otio
 import pyblish.api
@@ -33,8 +49,10 @@ class CollectEditorial(pyblish.api.InstancePlugin):
 
     # presets
     extensions = [".mov", ".mp4"]
+    source_dir = None
 
     def process(self, instance):
+        root_dir = None
         # remove context test attribute
         if instance.context.data.get("subsetNamesCheck"):
             instance.context.data.pop("subsetNamesCheck")
@@ -53,19 +71,42 @@ class CollectEditorial(pyblish.api.InstancePlugin):
             # get video file path
             video_path = None
             basename = os.path.splitext(os.path.basename(file_path))[0]
-            for f in os.listdir(staging_dir):
-                self.log.debug(f"__ test file: `{f}`")
-                # filter out by not sharing the same name
-                if os.path.splitext(f)[0] not in basename:
-                    continue
-                # filter out by respected extensions
-                if os.path.splitext(f)[1] not in self.extensions:
-                    continue
-                video_path = os.path.join(
-                    staging_dir, f
-                )
-                self.log.debug(f"__ video_path: `{video_path}`")
-            instance.data["editorialVideoPath"] = video_path
+
+            if self.source_dir:
+                source_dir = self.source_dir.replace("\\", "/")
+                if ("./" in source_dir) or ("../" in source_dir):
+                    # get current working dir
+                    cwd = os.getcwd()
+                    # set cwd to staging dir for absolute path solving
+                    os.chdir(staging_dir)
+                    root_dir = os.path.abspath(source_dir)
+                    # set back original cwd
+                    os.chdir(cwd)
+                elif "{" in source_dir:
+                    root_dir = source_dir
+                else:
+                    root_dir = os.path.normpath(source_dir)
+
+            if root_dir:
+                # search for source data will need to be done
+                instance.data["editorialSourceRoot"] = root_dir
+                instance.data["editorialSourcePath"] = None
+            else:
+                # source data are already found
+                for f in os.listdir(staging_dir):
+                    # filter out by not sharing the same name
+                    if os.path.splitext(f)[0] not in basename:
+                        continue
+                    # filter out by respected extensions
+                    if os.path.splitext(f)[1] not in self.extensions:
+                        continue
+                    video_path = os.path.join(
+                        staging_dir, f
+                    )
+                    self.log.debug(f"__ video_path: `{video_path}`")
+                instance.data["editorialSourceRoot"] = staging_dir
+                instance.data["editorialSourcePath"] = video_path
+
             instance.data["stagingDir"] = staging_dir
 
             # get editorial sequence file into otio timeline object
diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
index ac7413706a..6ce6232943 100644
--- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
+++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
@@ -2,7 +2,7 @@ import pyblish.api
 import re
 import os
 from avalon import io
-
+from copy import deepcopy
 
 class CollectHierarchyInstance(pyblish.api.ContextPlugin):
     """Collecting hierarchy context from `parents` and `hierarchy` data
@@ -60,7 +60,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
 
     def create_hierarchy(self, instance):
         parents = list()
-        hierarchy = ""
+        hierarchy = list()
         visual_hierarchy = [instance.context.data["assetEntity"]]
         while True:
             visual_parent = io.find_one(
@@ -81,22 +81,51 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
             })
 
         if self.shot_add_hierarchy:
+            parent_template_patern = re.compile(r"\{([a-z]*?)\}")
             # fill the parents parts from presets
             shot_add_hierarchy = self.shot_add_hierarchy.copy()
             hierarchy_parents = shot_add_hierarchy["parents"].copy()
-            for parent in hierarchy_parents:
-                hierarchy_parents[parent] = hierarchy_parents[parent].format(
-                    **instance.data["anatomyData"])
+
+            # fill parent keys data template from anatomy data
+            for parent_key in hierarchy_parents:
+                hierarchy_parents[parent_key] = hierarchy_parents[
+                    parent_key].format(**instance.data["anatomyData"])
+
+            for _index, _parent in enumerate(
+                    shot_add_hierarchy["parents_path"].split("/")):
+                parent_filled = _parent.format(**hierarchy_parents)
+                parent_key = parent_template_patern.findall(_parent).pop()
+
+                # in case SP context is set to the same folder
+                if (_index == 0) and ("folder" in parent_key) \
+                        and (parents[-1]["entityName"] == parent_filled):
+                    self.log.debug(f" skiping : {parent_filled}")
+                    continue
+
+                # in case first parent is project then start parents from start
+                if (_index == 0) and ("project" in parent_key):
+                    self.log.debug("rebuilding parents from scratch")
+                    project_parent = parents[0]
+                    parents = [project_parent]
+                    self.log.debug(f"project_parent: {project_parent}")
+                    self.log.debug(f"parents: {parents}")
+                    continue
+
                 prnt = self.convert_to_entity(
-                    parent, hierarchy_parents[parent])
+                    parent_key, parent_filled)
                 parents.append(prnt)
+                hierarchy.append(parent_filled)
 
-            hierarchy = shot_add_hierarchy[
-                "parents_path"].format(**hierarchy_parents)
+        # convert hierarchy to string
+        hierarchy = "/".join(hierarchy)
 
+        # assing to instance data
         instance.data["hierarchy"] = hierarchy
         instance.data["parents"] = parents
+
+        # print
         self.log.debug(f"Hierarchy: {hierarchy}")
+        self.log.debug(f"parents: {parents}")
 
         if self.shot_add_tasks:
             instance.data["tasks"] = self.shot_add_tasks
@@ -117,7 +146,8 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
     def processing_instance(self, instance):
         self.log.info(f"_ instance: {instance}")
         # adding anatomyData for burnins
-        instance.data["anatomyData"] = instance.context.data["anatomyData"]
+        instance.data["anatomyData"] = deepcopy(
+            instance.context.data["anatomyData"])
 
         asset = instance.data["asset"]
         assets_shared = instance.context.data.get("assetsShared")
@@ -133,9 +163,6 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
         shot_name = instance.data["asset"]
         self.log.debug(f"Shot Name: {shot_name}")
 
-        if instance.data["hierarchy"] not in shot_name:
-            self.log.warning("wrong parent")
-
         label = f"{shot_name} ({frame_start}-{frame_end})"
         instance.data["label"] = label
 
@@ -150,7 +177,8 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
             "asset": instance.data["asset"],
             "hierarchy": instance.data["hierarchy"],
             "parents": instance.data["parents"],
-            "tasks": instance.data["tasks"]
+            "tasks": instance.data["tasks"],
+            "anatomyData": instance.data["anatomyData"]
         })
 
 
@@ -194,6 +222,7 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin):
                 instance.data["parents"] = s_asset_data["parents"]
                 instance.data["hierarchy"] = s_asset_data["hierarchy"]
                 instance.data["tasks"] = s_asset_data["tasks"]
+                instance.data["anatomyData"] = s_asset_data["anatomyData"]
 
             # generate hierarchy data only on shot instances
             if 'shot' not in instance.data.get('family', ''):
@@ -224,7 +253,9 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin):
 
             in_info['tasks'] = instance.data['tasks']
 
+            from pprint import pformat
             parents = instance.data.get('parents', [])
+            self.log.debug(f"parents: {pformat(parents)}")
 
             actual = {name: in_info}
 
@@ -240,4 +271,5 @@ class CollectHierarchyContext(pyblish.api.ContextPlugin):
 
         # adding hierarchy context to instance
         context.data["hierarchyContext"] = final_context
+        self.log.debug(f"hierarchyContext: {pformat(final_context)}")
         self.log.info("Hierarchy instance collected")
diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_data.py b/pype/plugins/standalonepublisher/publish/collect_instance_data.py
index 1b32ea9144..58b81324f5 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instance_data.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_data.py
@@ -22,7 +22,7 @@ class CollectInstanceData(pyblish.api.InstancePlugin):
     hosts = ["standalonepublisher"]
 
     def process(self, instance):
-        fps = instance.data["assetEntity"]["data"]["fps"]
+        fps = instance.context.data["assetEntity"]["data"]["fps"]
         instance.data.update({
             "fps": fps
         })
diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
new file mode 100644
index 0000000000..63b98f2721
--- /dev/null
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
@@ -0,0 +1,57 @@
+import os
+import tempfile
+import pyblish.api
+from copy import deepcopy
+
+class CollectInstanceResources(pyblish.api.InstancePlugin):
+    """Collect instance's resources"""
+
+    # must be after `CollectInstances`
+    order = pyblish.api.CollectorOrder + 0.011
+    label = "Collect Instance Resources"
+    hosts = ["standalonepublisher"]
+    families = ["clip"]
+
+    def process(self, instance):
+        anatomy = instance.context.data["anatomy"]
+        anatomy_data = deepcopy(instance.context.data["anatomyData"])
+        anatomy_data.update({"root": anatomy.roots})
+
+        subset = instance.data["subset"]
+        clip_name = instance.data["clipName"]
+
+        editorial_source_root = instance.data["editorialSourceRoot"]
+        editorial_source_path = instance.data["editorialSourcePath"]
+
+        if editorial_source_path:
+            # add family if mov or mp4 found which is longer for
+            # cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin
+            staging_dir = os.path.normpath(
+                tempfile.mkdtemp(prefix="pyblish_tmp_")
+            )
+            instance.data["stagingDir"] = staging_dir
+            instance.data["families"] += ["trimming"]
+            return
+
+        if "{" in editorial_source_root:
+            editorial_source_root = editorial_source_root.format(
+                **anatomy_data)
+
+        self.log.debug(f"root: {editorial_source_root}")
+
+        for root, dirs, files in os.walk(editorial_source_root):
+            if subset in root and clip_name in root:
+                staging_dir = root
+
+        self.log.debug(f"staging_dir: {staging_dir}")
+
+
+        # add `editorialSourceRoot`  as staging dir
+
+        # if `editorialSourcePath` is none then loop
+        # trough `editorialSourceRoot`
+
+        # if image sequence then create representation > match
+        # with subset name in dict
+
+        # idenfify as image sequence `isSequence` on instance data
diff --git a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py
similarity index 89%
rename from pype/plugins/standalonepublisher/publish/collect_clip_instances.py
rename to pype/plugins/standalonepublisher/publish/collect_instances.py
index def0c13a78..9cd8d9f36c 100644
--- a/pype/plugins/standalonepublisher/publish/collect_clip_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instances.py
@@ -1,15 +1,14 @@
 import os
 import opentimelineio as otio
-import tempfile
 import pyblish.api
 from pype import lib as plib
 
 
-class CollectClipInstances(pyblish.api.InstancePlugin):
-    """Collect Clips instances from editorial's OTIO sequence"""
+class CollectInstances(pyblish.api.InstancePlugin):
+    """Collect instances from editorial's OTIO sequence"""
 
     order = pyblish.api.CollectorOrder + 0.01
-    label = "Collect Clips"
+    label = "Collect Instances"
     hosts = ["standalonepublisher"]
     families = ["editorial"]
 
@@ -19,13 +18,13 @@ class CollectClipInstances(pyblish.api.InstancePlugin):
             "family": "review",
             "families": ["clip", "ftrack"],
             # "ftrackFamily": "review",
-            "extension": ".mp4"
+            "extensions": [".mp4"]
         },
         "audioMain": {
             "family": "audio",
             "families": ["clip", "ftrack"],
             # "ftrackFamily": "audio",
-            "extension": ".wav",
+            "extensions": [".wav"],
             # "version": 1
         },
         "shotMain": {
@@ -37,12 +36,14 @@ class CollectClipInstances(pyblish.api.InstancePlugin):
     custom_start_frame = None
 
     def process(self, instance):
-        staging_dir = os.path.normpath(
-            tempfile.mkdtemp(prefix="pyblish_tmp_")
-        )
         # get context
         context = instance.context
 
+        instance_data_filter = [
+            "editorialSourceRoot",
+            "editorialSourcePath"
+        ]
+
         # attribute for checking duplicity during creation
         if not context.data.get("assetNameCheck"):
             context.data["assetNameCheck"] = list()
@@ -103,7 +104,10 @@ class CollectClipInstances(pyblish.api.InstancePlugin):
 
                 # frame ranges data
                 clip_in = clip.range_in_parent().start_time.value
+                clip_in += track_start_frame
                 clip_out = clip.range_in_parent().end_time_inclusive().value
+                clip_out += track_start_frame
+                self.log.info(f"clip_in: {clip_in} | clip_out: {clip_out}")
 
                 # add offset in case there is any
                 if self.timeline_frame_offset:
@@ -131,14 +135,11 @@ class CollectClipInstances(pyblish.api.InstancePlugin):
 
                 # create shared new instance data
                 instance_data = {
-                    "stagingDir": staging_dir,
-
                     # shared attributes
                     "asset": name,
                     "assetShareName": name,
-                    "editorialVideoPath": instance.data[
-                        "editorialVideoPath"],
                     "item": clip,
+                    "clipName": clip_name,
 
                     # parent time properities
                     "trackStartFrame": track_start_frame,
@@ -167,6 +168,10 @@ class CollectClipInstances(pyblish.api.InstancePlugin):
                     "frameEndH": frame_end + handle_end
                 }
 
+                for data_key in instance_data_filter:
+                    instance_data.update({
+                        data_key: instance.data.get(data_key)})
+
                 # adding subsets to context as instances
                 for subset, properities in self.subsets.items():
                     # adding Review-able instance
diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py
deleted file mode 100644
index d5af7638ee..0000000000
--- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py
+++ /dev/null
@@ -1,92 +0,0 @@
-import os
-import clique
-import pype.api
-
-from pprint import pformat
-
-
-class ExtractShotData(pype.api.Extractor):
-    """Extract shot "mov" and "wav" files."""
-
-    label = "Extract Shot Data"
-    hosts = ["standalonepublisher"]
-    families = ["clip"]
-
-    # presets
-
-    def process(self, instance):
-        representation = instance.data.get("representations")
-        self.log.debug(f"_ representation: {representation}")
-
-        if not representation:
-            instance.data["representations"] = list()
-
-        # get ffmpet path
-        ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg")
-
-        # get staging dir
-        staging_dir = self.staging_dir(instance)
-        self.log.info("Staging dir set to: `{}`".format(staging_dir))
-
-        # Generate mov file.
-        fps = instance.data["fps"]
-        video_file_path = instance.data["editorialVideoPath"]
-        ext = instance.data.get("extension", ".mov")
-
-        clip_trimed_path = os.path.join(
-            staging_dir, instance.data["name"] + ext)
-        #
-        # # check video file metadata
-        # input_data = plib.ffprobe_streams(video_file_path)[0]
-        # self.log.debug(f"__ input_data: `{input_data}`")
-
-        start = float(instance.data["clipInH"])
-        dur = float(instance.data["clipDurationH"])
-
-        if ext in ".wav":
-            start += 0.5
-
-        args = [
-            ffmpeg_path,
-            "-ss", str(start / fps),
-            "-i", f"\"{video_file_path}\"",
-            "-t", str(dur / fps)
-        ]
-        if ext in [".mov", ".mp4"]:
-            args.extend([
-                "-crf", "18",
-                "-pix_fmt", "yuv420p"])
-        elif ext in ".wav":
-            args.extend([
-                "-vn -acodec pcm_s16le",
-                "-ar 48000 -ac 2"
-            ])
-
-        # add output path
-        args.append(f"\"{clip_trimed_path}\"")
-
-        self.log.info(f"Processing: {args}")
-        ffmpeg_args = " ".join(args)
-        output = pype.api.subprocess(ffmpeg_args, shell=True)
-        self.log.info(output)
-
-        repr = {
-            "name": ext[1:],
-            "ext": ext[1:],
-            "files": os.path.basename(clip_trimed_path),
-            "stagingDir": staging_dir,
-            "frameStart": int(instance.data["frameStart"]),
-            "frameEnd": int(instance.data["frameEnd"]),
-            "frameStartFtrack": int(instance.data["frameStartH"]),
-            "frameEndFtrack": int(instance.data["frameEndH"]),
-            "fps": fps,
-        }
-
-        if ext[1:] in ["mov", "mp4"]:
-            repr.update({
-                "thumbnail": True,
-                "tags": ["review", "ftrackreview", "delete"]})
-
-        instance.data["representations"].append(repr)
-
-        self.log.debug(f"Instance data: {pformat(instance.data)}")
diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py
new file mode 100644
index 0000000000..c955275b4f
--- /dev/null
+++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py
@@ -0,0 +1,101 @@
+import os
+import pyblish.api
+import pype.api
+
+from pprint import pformat
+
+
+class ExtractTrimVideoAudio(pype.api.Extractor):
+    """Trim with ffmpeg "mov" and "wav" files."""
+
+    label = "Extract Trim Video/Audio"
+    hosts = ["standalonepublisher"]
+    families = ["clip", "trimming"]
+
+    # make sure it is enabled only if at least both families are available
+    match = pyblish.api.Subset
+
+    # presets
+
+    def process(self, instance):
+        representation = instance.data.get("representations")
+        self.log.debug(f"_ representation: {representation}")
+
+        if not representation:
+            instance.data["representations"] = list()
+
+        # get ffmpet path
+        ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg")
+
+        # get staging dir
+        staging_dir = self.staging_dir(instance)
+        self.log.info("Staging dir set to: `{}`".format(staging_dir))
+
+        # Generate mov file.
+        fps = instance.data["fps"]
+        video_file_path = instance.data["editorialSourcePath"]
+        extensions = instance.data.get("extensions", [".mov"])
+
+        for ext in extensions:
+            clip_trimed_path = os.path.join(
+                staging_dir, instance.data["name"] + ext)
+            # # check video file metadata
+            # input_data = plib.ffprobe_streams(video_file_path)[0]
+            # self.log.debug(f"__ input_data: `{input_data}`")
+
+            start = float(instance.data["clipInH"])
+            dur = float(instance.data["clipDurationH"])
+
+            if ext in ".wav":
+                # offset time as ffmpeg is having bug
+                start += 0.5
+                # remove "review" from families
+                instance.data["families"] = [
+                    fml for fml in instance.data["families"]
+                    if "trimming" not in fml
+                ]
+
+            args = [
+                ffmpeg_path,
+                "-ss", str(start / fps),
+                "-i", f"\"{video_file_path}\"",
+                "-t", str(dur / fps)
+            ]
+            if ext in [".mov", ".mp4"]:
+                args.extend([
+                    "-crf", "18",
+                    "-pix_fmt", "yuv420p"])
+            elif ext in ".wav":
+                args.extend([
+                    "-vn -acodec pcm_s16le",
+                    "-ar 48000 -ac 2"
+                ])
+
+            # add output path
+            args.append(f"\"{clip_trimed_path}\"")
+
+            self.log.info(f"Processing: {args}")
+            ffmpeg_args = " ".join(args)
+            output = pype.api.subprocess(ffmpeg_args, shell=True)
+            self.log.info(output)
+
+            repr = {
+                "name": ext[1:],
+                "ext": ext[1:],
+                "files": os.path.basename(clip_trimed_path),
+                "stagingDir": staging_dir,
+                "frameStart": int(instance.data["frameStart"]),
+                "frameEnd": int(instance.data["frameEnd"]),
+                "frameStartFtrack": int(instance.data["frameStartH"]),
+                "frameEndFtrack": int(instance.data["frameEndH"]),
+                "fps": fps,
+            }
+
+            if ext[1:] in ["mov", "mp4"]:
+                repr.update({
+                    "thumbnail": True,
+                    "tags": ["review", "ftrackreview", "delete"]})
+
+            instance.data["representations"].append(repr)
+
+            self.log.debug(f"Instance data: {pformat(instance.data)}")
diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
index 7e1694fbd1..65581a6cdc 100644
--- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
+++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
@@ -15,6 +15,6 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin):
         self.log.debug(
             f"Instance: {instance}, Families: "
             f"{[instance.data['family']] + instance.data['families']}")
-        check_file = instance.data["editorialVideoPath"]
+        check_file = instance.data["editorialSourcePath"]
         msg = f"Missing \"{check_file}\"."
         assert check_file, msg

From 2d375f092c1df316f633bcb3f68e8f4cab401e6b Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Fri, 6 Nov 2020 15:20:59 +0100
Subject: [PATCH 02/10] feat(SP): wip publishing editorial with dir of subsets
 and files

---
 .../publish/collect_clear_instances.py        |  20 +++
 .../publish/collect_instance_resources.py     | 151 +++++++++++++++++-
 .../publish/collect_instances.py              |  10 +-
 .../publish/validate_editorial_resources.py   |   5 +-
 4 files changed, 173 insertions(+), 13 deletions(-)
 create mode 100644 pype/plugins/standalonepublisher/publish/collect_clear_instances.py

diff --git a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py
new file mode 100644
index 0000000000..0d64c57d3a
--- /dev/null
+++ b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py
@@ -0,0 +1,20 @@
+"""
+Optional:
+    instance.data["remove"]     -> mareker for removing
+"""
+import pyblish.api
+
+
+class CollectClearInstances(pyblish.api.ContextPlugin):
+    """Clear all marked instances"""
+
+    order = pyblish.api.CollectorOrder + 0.4999
+    label = "Clear Instances"
+    hosts = ["standalonepublisher"]
+
+    def process(self, context):
+
+        for instance in context:
+            if instance.data.get("remove"):
+                self.log.info(f"Removing: {instance}")
+                context.remove(instance)
diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
index 63b98f2721..25bdffd422 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
@@ -2,6 +2,8 @@ import os
 import tempfile
 import pyblish.api
 from copy import deepcopy
+import clique
+
 
 class CollectInstanceResources(pyblish.api.InstancePlugin):
     """Collect instance's resources"""
@@ -13,6 +15,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
     families = ["clip"]
 
     def process(self, instance):
+        context = instance.context
+        self.log.info(f"Processing instance: {instance}")
+        subset_files = dict()
+        subset_dirs = list()
         anatomy = instance.context.data["anatomy"]
         anatomy_data = deepcopy(instance.context.data["anatomyData"])
         anatomy_data.update({"root": anatomy.roots})
@@ -23,6 +29,7 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
         editorial_source_root = instance.data["editorialSourceRoot"]
         editorial_source_path = instance.data["editorialSourcePath"]
 
+        # if `editorial_source_path` then loop trough
         if editorial_source_path:
             # add family if mov or mp4 found which is longer for
             # cutting `trimming` to enable `ExtractTrimmingVideoAudio` plugin
@@ -33,23 +40,153 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
             instance.data["families"] += ["trimming"]
             return
 
+        # if template patern in path then fill it with `anatomy_data`
         if "{" in editorial_source_root:
             editorial_source_root = editorial_source_root.format(
                 **anatomy_data)
 
         self.log.debug(f"root: {editorial_source_root}")
-
+        # loop `editorial_source_root` and find clip name in folders
+        # and look for any subset name alternatives
         for root, dirs, files in os.walk(editorial_source_root):
-            if subset in root and clip_name in root:
-                staging_dir = root
+            correct_clip_dir = None
+            for d in dirs:
+                # avoid all non clip dirs
+                if d not in clip_name:
+                    continue
+                # found correct dir for clip
+                correct_clip_dir = d
 
-        self.log.debug(f"staging_dir: {staging_dir}")
+            # continue if clip dir was not found
+            if not correct_clip_dir:
+                continue
 
+            clip_dir_path = os.path.join(root, correct_clip_dir)
+            subset_files_items = list()
+            # list content of clip dir and search for subset items
+            for subset_item in os.listdir(clip_dir_path):
+                # avoid all items which are not defined as subsets by name
+                if subset not in subset_item:
+                    continue
 
-        # add `editorialSourceRoot`  as staging dir
+                subset_item_path = os.path.join(
+                    clip_dir_path, subset_item)
+                # if it is dir store it to `subset_dirs` list
+                if os.path.isdir(subset_item_path):
+                    subset_dirs.append(subset_item_path)
 
-        # if `editorialSourcePath` is none then loop
-        # trough `editorialSourceRoot`
+                # if it is file then store it to `subset_files` list
+                if os.path.isfile(subset_item_path):
+                    subset_files_items.append(subset_item_path)
+
+            if subset_files_items:
+                subset_files.update({clip_dir_path: subset_files_items})
+            if correct_clip_dir:
+                break
+
+        if subset_dirs:
+            # look all dirs and check for subset name alternatives
+            copy_instance_data = deepcopy(
+                {_k: _v for _k, _v in instance.data.items()})
+
+            # find next available precise subset name with comprahantion
+            subset_dir_found = next(
+                (d for d in subset_dirs
+                 if os.path.basename(d) in subset),
+                None)
+
+            if not subset_dir_found:
+                instance.data["remove"] = True
+
+            for _dir in subset_dirs:
+                sub_dir = os.path.basename(_dir)
+                instance_data = instance.data
+                # if subset name is only alternative then create new instance
+                if sub_dir != subset:
+                    new_instance_data = dict()
+                    for _key, _value in copy_instance_data.items():
+                        new_instance_data[_key] = _value
+                        if not isinstance(_value, str):
+                            continue
+                        if subset in _value:
+                            new_instance_data[_key] = _value.replace(
+                                subset, sub_dir)
+                    new_instance = context.create_instance(
+                        new_instance_data["name"])
+                    new_instance.data.update(new_instance_data)
+                    self.log.info(f"Creating new instance: {new_instance}")
+                    instance_data = new_instance.data
+
+                staging_dir = _dir
+                files = os.listdir(_dir)
+                collections, remainder = clique.assemble(files)
+                # self.log.debug(f"collections: {collections}")
+                # self.log.debug(f"remainder: {remainder}")
+                # self.log.debug(f"staging_dir: {staging_dir}")
+
+                # add staging_dir to instance_data
+                instance_data["stagingDir"] = staging_dir
+                # add representations to instance_data
+                instance_data["representations"] = list()
+
+                # loop trough collections and create representations
+                for _collection in collections:
+                    ext = _collection.tail
+                    repre_data = {
+                        "name": ext[1:],
+                        "ext": ext[1:],
+                        "files": [item for item in _collection],
+                        "stagingDir": staging_dir
+                    }
+                    instance_data["representations"].append(repre_data)
+
+                # loop trough reminders and create representations
+                for _reminding_file in remainder:
+                    ext = os.path.splitext(_reminding_file)[-1]
+                    if ext not in instance_data["extensions"]:
+                        continue
+
+                    repre_data = {
+                        "name": ext[1:],
+                        "ext": ext[1:],
+                        "files": _reminding_file,
+                        "stagingDir": staging_dir
+                    }
+
+                    # exception for thumbnail
+                    if "thumb" in _reminding_file:
+                        repre_data.update({
+                            'name': "thumbnail",
+                            'thumbnail': True
+                        })
+
+                    # exception for mp4 preview
+                    if ".mp4" in _reminding_file:
+                        frame_start = instance_data["frameStart"]
+                        frame_end = instance_data["frameEnd"]
+                        instance_data["families"].append("review")
+                        repre_data.update({
+                            "frameStart": 0,
+                            "frameEnd": (frame_end - frame_start) + 1,
+                            "frameStartFtrack": 0,
+                            "frameEndFtrack": (frame_end - frame_start) + 1,
+                            "step": 1,
+                            "fps": context.data.get("fps"),
+                            "name": "review",
+                            "tags": ["review", "ftrackreview"],
+                        })
+
+                    instance_data["representations"].append(repre_data)
+
+                representations = instance_data["representations"]
+                self.log.debug(f">>>_<<< representations: {representations}")
+
+        if subset_files:
+            staging_dir = list(subset_files.keys()).pop()
+            collections, remainder = clique.assemble(subset_files[staging_dir])
+            # self.log.debug(f"collections: {collections}")
+            # self.log.debug(f"remainder: {remainder}")
+            # self.log.debug(f"staging_dir: {staging_dir}")
 
         # if image sequence then create representation > match
         # with subset name in dict
diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py
index 9cd8d9f36c..3d577c1527 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instances.py
@@ -69,7 +69,6 @@ class CollectInstances(pyblish.api.InstancePlugin):
         handle_start = int(asset_data["handleStart"])
         handle_end = int(asset_data["handleEnd"])
 
-        instances = []
         for track in tracks:
             try:
                 track_start_frame = (
@@ -179,12 +178,13 @@ class CollectInstances(pyblish.api.InstancePlugin):
                     subset_instance_data.update(properities)
                     subset_instance_data.update({
                         # unique attributes
-                        "name": f"{subset}_{name}",
-                        "label": f"{subset} {name} ({clip_in}-{clip_out})",
+                        "name": f"{name}_{subset}",
+                        "label": f"{name} {subset} ({clip_in}-{clip_out})",
                         "subset": subset
                     })
-                    instances.append(instance.context.create_instance(
-                        **subset_instance_data))
+                    # create new instance
+                    instance.context.create_instance(
+                        **subset_instance_data)
 
                 context.data["assetsShared"][name] = {
                     "_clipIn": clip_in,
diff --git a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
index 65581a6cdc..0dfca92f66 100644
--- a/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
+++ b/pype/plugins/standalonepublisher/publish/validate_editorial_resources.py
@@ -7,7 +7,10 @@ class ValidateEditorialResources(pyblish.api.InstancePlugin):
 
     label = "Validate Editorial Resources"
     hosts = ["standalonepublisher"]
-    families = ["clip"]
+    families = ["clip", "trimming"]
+
+    # make sure it is enabled only if at least both families are available
+    match = pyblish.api.Subset
 
     order = pype.api.ValidateContentsOrder
 

From e68ce589b5e076c351a81cf41ac5b74597ee405a Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Fri, 6 Nov 2020 18:41:36 +0100
Subject: [PATCH 03/10] feat(SP): final publishing editorial with image
 sequence and other

---
 .../publish/collect_hierarchy.py              |   9 +-
 .../publish/collect_instance_resources.py     | 241 ++++++++++--------
 2 files changed, 147 insertions(+), 103 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
index 6ce6232943..3d9465cb1a 100644
--- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
+++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
@@ -128,7 +128,14 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
         self.log.debug(f"parents: {parents}")
 
         if self.shot_add_tasks:
-            instance.data["tasks"] = self.shot_add_tasks
+            tasks_to_add = dict()
+            project_tasks = io.find_one({"type": "project"})["config"]["tasks"]
+            for task in self.shot_add_tasks:
+                for task_type in project_tasks.keys():
+                    if task_type.lower() in task.lower():
+                        tasks_to_add.update({task: {"type": task_type}})
+
+            instance.data["tasks"] = tasks_to_add
         else:
             instance.data["tasks"] = list()
 
diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
index 25bdffd422..6807e82193 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
@@ -1,4 +1,5 @@
 import os
+import re
 import tempfile
 import pyblish.api
 from copy import deepcopy
@@ -15,12 +16,13 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
     families = ["clip"]
 
     def process(self, instance):
-        context = instance.context
+        self.context = instance.context
+        instance_data = instance.data
         self.log.info(f"Processing instance: {instance}")
         subset_files = dict()
         subset_dirs = list()
-        anatomy = instance.context.data["anatomy"]
-        anatomy_data = deepcopy(instance.context.data["anatomyData"])
+        anatomy = self.context.data["anatomy"]
+        anatomy_data = deepcopy(self.context.data["anatomyData"])
         anatomy_data.update({"root": anatomy.roots})
 
         subset = instance.data["subset"]
@@ -36,8 +38,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
             staging_dir = os.path.normpath(
                 tempfile.mkdtemp(prefix="pyblish_tmp_")
             )
-            instance.data["stagingDir"] = staging_dir
-            instance.data["families"] += ["trimming"]
+            instance_data["stagingDir"] = staging_dir
+            instance_data["families"] += ["trimming"]
             return
 
         # if template patern in path then fill it with `anatomy_data`
@@ -86,109 +88,144 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
 
         if subset_dirs:
             # look all dirs and check for subset name alternatives
-            copy_instance_data = deepcopy(
-                {_k: _v for _k, _v in instance.data.items()})
-
-            # find next available precise subset name with comprahantion
-            subset_dir_found = next(
-                (d for d in subset_dirs
-                 if os.path.basename(d) in subset),
-                None)
-
-            if not subset_dir_found:
-                instance.data["remove"] = True
-
             for _dir in subset_dirs:
                 sub_dir = os.path.basename(_dir)
-                instance_data = instance.data
                 # if subset name is only alternative then create new instance
                 if sub_dir != subset:
-                    new_instance_data = dict()
-                    for _key, _value in copy_instance_data.items():
-                        new_instance_data[_key] = _value
-                        if not isinstance(_value, str):
-                            continue
-                        if subset in _value:
-                            new_instance_data[_key] = _value.replace(
-                                subset, sub_dir)
-                    new_instance = context.create_instance(
-                        new_instance_data["name"])
-                    new_instance.data.update(new_instance_data)
-                    self.log.info(f"Creating new instance: {new_instance}")
-                    instance_data = new_instance.data
+                    instance_data = self.duplicate_instance(
+                        instance.data, subset, sub_dir)
 
-                staging_dir = _dir
-                files = os.listdir(_dir)
-                collections, remainder = clique.assemble(files)
-                # self.log.debug(f"collections: {collections}")
-                # self.log.debug(f"remainder: {remainder}")
-                # self.log.debug(f"staging_dir: {staging_dir}")
-
-                # add staging_dir to instance_data
-                instance_data["stagingDir"] = staging_dir
-                # add representations to instance_data
-                instance_data["representations"] = list()
-
-                # loop trough collections and create representations
-                for _collection in collections:
-                    ext = _collection.tail
-                    repre_data = {
-                        "name": ext[1:],
-                        "ext": ext[1:],
-                        "files": [item for item in _collection],
-                        "stagingDir": staging_dir
-                    }
-                    instance_data["representations"].append(repre_data)
-
-                # loop trough reminders and create representations
-                for _reminding_file in remainder:
-                    ext = os.path.splitext(_reminding_file)[-1]
-                    if ext not in instance_data["extensions"]:
-                        continue
-
-                    repre_data = {
-                        "name": ext[1:],
-                        "ext": ext[1:],
-                        "files": _reminding_file,
-                        "stagingDir": staging_dir
-                    }
-
-                    # exception for thumbnail
-                    if "thumb" in _reminding_file:
-                        repre_data.update({
-                            'name': "thumbnail",
-                            'thumbnail': True
-                        })
-
-                    # exception for mp4 preview
-                    if ".mp4" in _reminding_file:
-                        frame_start = instance_data["frameStart"]
-                        frame_end = instance_data["frameEnd"]
-                        instance_data["families"].append("review")
-                        repre_data.update({
-                            "frameStart": 0,
-                            "frameEnd": (frame_end - frame_start) + 1,
-                            "frameStartFtrack": 0,
-                            "frameEndFtrack": (frame_end - frame_start) + 1,
-                            "step": 1,
-                            "fps": context.data.get("fps"),
-                            "name": "review",
-                            "tags": ["review", "ftrackreview"],
-                        })
-
-                    instance_data["representations"].append(repre_data)
-
-                representations = instance_data["representations"]
-                self.log.debug(f">>>_<<< representations: {representations}")
+                # create all representations
+                self.create_representations(
+                    os.listdir(_dir), instance_data, _dir)
 
         if subset_files:
-            staging_dir = list(subset_files.keys()).pop()
-            collections, remainder = clique.assemble(subset_files[staging_dir])
-            # self.log.debug(f"collections: {collections}")
-            # self.log.debug(f"remainder: {remainder}")
-            # self.log.debug(f"staging_dir: {staging_dir}")
+            unique_subset_names = list()
+            root_dir = list(subset_files.keys()).pop()
+            files_list = subset_files[root_dir]
+            search_patern = f"({subset}[A-Za-z0-9]+)(?=[\\._\\s])"
+            for _file in files_list:
+                patern = re.compile(search_patern)
+                match = patern.findall(_file)
+                if not match:
+                    continue
+                match_subset = match.pop()
+                if match_subset in unique_subset_names:
+                    continue
+                unique_subset_names.append(match_subset)
 
-        # if image sequence then create representation > match
-        # with subset name in dict
+            self.log.debug(f"unique_subset_names: {unique_subset_names}")
 
-        # idenfify as image sequence `isSequence` on instance data
+            for _un_subs in unique_subset_names:
+                instance_data = self.duplicate_instance(
+                    instance.data, subset, _un_subs)
+
+            # create all representations
+            self.create_representations(
+                [os.path.basename(f) for f in files_list],
+                instance_data, root_dir)
+
+        # if the original subset name was not found in input folders
+        # then representations were nod added and it can be removed
+        if not instance.data.get("representations"):
+            instance.data["remove"] = True
+
+    def duplicate_instance(self, instance_data, subset, new_subset):
+
+        new_instance_data = dict()
+        for _key, _value in instance_data.items():
+            new_instance_data[_key] = _value
+            if not isinstance(_value, str):
+                continue
+            if subset in _value:
+                new_instance_data[_key] = _value.replace(
+                    subset, new_subset)
+        new_instance = self.context.create_instance(
+            new_instance_data["name"])
+        new_instance.data.update(new_instance_data)
+        self.log.info(f"Creating new instance: {new_instance}")
+        return new_instance.data
+
+    def create_representations(
+            self, files_list, instance_data, staging_dir):
+        """ Create representations from Collection object
+        """
+        # collecting frames for later frame start/end reset
+        frames = list()
+        # break down Collection object to collections and reminders
+        collections, remainder = clique.assemble(files_list)
+        # add staging_dir to instance_data
+        instance_data["stagingDir"] = staging_dir
+        # add representations to instance_data
+        instance_data["representations"] = list()
+
+        # loop trough collections and create representations
+        for _collection in collections:
+            ext = _collection.tail
+            frame_start = list(_collection.indexes)[0]
+            frame_end = list(_collection.indexes)[-1]
+            repre_data = {
+                "frameStart": frame_start,
+                "frameEnd": frame_end,
+                "name": ext[1:],
+                "ext": ext[1:],
+                "files": [item for item in _collection],
+                "stagingDir": staging_dir
+            }
+            instance_data["representations"].append(repre_data)
+
+            # add to frames for frame range reset
+            frames.append(frame_start)
+            frames.append(frame_end)
+
+        # loop trough reminders and create representations
+        for _reminding_file in remainder:
+            ext = os.path.splitext(_reminding_file)[-1]
+            if ext not in instance_data["extensions"]:
+                continue
+
+            frame_start = 1
+            frame_end = 1
+
+            repre_data = {
+                "name": ext[1:],
+                "ext": ext[1:],
+                "files": _reminding_file,
+                "stagingDir": staging_dir
+            }
+
+            # exception for thumbnail
+            if "thumb" in _reminding_file:
+                repre_data.update({
+                    'name': "thumbnail",
+                    'thumbnail': True
+                })
+
+            # exception for mp4 preview
+            if ".mp4" in _reminding_file:
+                frame_start = 0
+                frame_end = (
+                    (instance_data["frameEnd"] - instance_data["frameStart"])
+                    + 1)
+                instance_data["families"] += ["review", "ftrack"]
+                repre_data.update({
+                    "frameStart": frame_start,
+                    "frameEnd": frame_end,
+                    "frameStartFtrack": frame_start,
+                    "frameEndFtrack": frame_end,
+                    "step": 1,
+                    "fps": self.context.data.get("fps"),
+                    "name": "review",
+                    "tags": ["review", "ftrackreview", "delete"],
+                })
+
+            # add to frames for frame range reset only if no collection
+            if not collections:
+                frames.append(frame_start)
+                frames.append(frame_end)
+
+            instance_data["representations"].append(repre_data)
+
+        # reset frame start / end
+        instance_data["frameStart"] = min(frames)
+        instance_data["frameEnd"] = max(frames)

From e7b8717c594986984208bdf9df3ff4ae996216d1 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Fri, 6 Nov 2020 19:23:31 +0100
Subject: [PATCH 04/10] feat(PS): fixing original publishing with trimming

---
 .../standalonepublisher/publish/collect_hierarchy.py      | 2 +-
 .../standalonepublisher/publish/collect_instances.py      | 8 +++-----
 .../publish/extract_trim_video_audio.py                   | 8 ++++++--
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
index 3d9465cb1a..45d2fb4160 100644
--- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
+++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
@@ -137,7 +137,7 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
 
             instance.data["tasks"] = tasks_to_add
         else:
-            instance.data["tasks"] = list()
+            instance.data["tasks"] = dict()
 
         # updating hierarchy data
         instance.data["anatomyData"].update({
diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py
index 3d577c1527..6e8bbea2e8 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instances.py
@@ -17,22 +17,20 @@ class CollectInstances(pyblish.api.InstancePlugin):
         "referenceMain": {
             "family": "review",
             "families": ["clip", "ftrack"],
-            # "ftrackFamily": "review",
             "extensions": [".mp4"]
         },
         "audioMain": {
             "family": "audio",
             "families": ["clip", "ftrack"],
-            # "ftrackFamily": "audio",
             "extensions": [".wav"],
-            # "version": 1
         },
         "shotMain": {
             "family": "shot",
             "families": []
         }
     }
-    timeline_frame_offset = None  # if 900000 for edl default then -900000
+    timeline_frame_start = 900000  # starndart edl default (01:00:00:00)
+    timeline_frame_offset = None
     custom_start_frame = None
 
     def process(self, instance):
@@ -73,7 +71,7 @@ class CollectInstances(pyblish.api.InstancePlugin):
             try:
                 track_start_frame = (
                     abs(track.source_range.start_time.value)
-                )
+                ) - self.timeline_frame_start
             except AttributeError:
                 track_start_frame = 0
 
diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py
index c955275b4f..193902a9f6 100644
--- a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py
+++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py
@@ -8,6 +8,8 @@ from pprint import pformat
 class ExtractTrimVideoAudio(pype.api.Extractor):
     """Trim with ffmpeg "mov" and "wav" files."""
 
+    # must be before `ExtractThumbnailSP`
+    order = pyblish.api.ExtractorOrder - 0.01
     label = "Extract Trim Video/Audio"
     hosts = ["standalonepublisher"]
     families = ["clip", "trimming"]
@@ -37,6 +39,8 @@ class ExtractTrimVideoAudio(pype.api.Extractor):
         extensions = instance.data.get("extensions", [".mov"])
 
         for ext in extensions:
+            self.log.info("Processing ext: `{}`".format(ext))
+
             clip_trimed_path = os.path.join(
                 staging_dir, instance.data["name"] + ext)
             # # check video file metadata
@@ -46,7 +50,7 @@ class ExtractTrimVideoAudio(pype.api.Extractor):
             start = float(instance.data["clipInH"])
             dur = float(instance.data["clipDurationH"])
 
-            if ext in ".wav":
+            if ext == ".wav":
                 # offset time as ffmpeg is having bug
                 start += 0.5
                 # remove "review" from families
@@ -91,7 +95,7 @@ class ExtractTrimVideoAudio(pype.api.Extractor):
                 "fps": fps,
             }
 
-            if ext[1:] in ["mov", "mp4"]:
+            if ext in [".mov", ".mp4"]:
                 repr.update({
                     "thumbnail": True,
                     "tags": ["review", "ftrackreview", "delete"]})

From 0f3f479bca4202a30463e880f7afa1b7015010d5 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Fri, 6 Nov 2020 20:03:05 +0100
Subject: [PATCH 05/10] feat(SP): default timeline frame start

---
 .../standalonepublisher/publish/collect_instances.py     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py
index 6e8bbea2e8..e99be5df38 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instances.py
@@ -29,7 +29,7 @@ class CollectInstances(pyblish.api.InstancePlugin):
             "families": []
         }
     }
-    timeline_frame_start = 900000  # starndart edl default (01:00:00:00)
+    timeline_frame_start = 900000  # starndard edl default (10:00:00:00)
     timeline_frame_offset = None
     custom_start_frame = None
 
@@ -68,13 +68,18 @@ class CollectInstances(pyblish.api.InstancePlugin):
         handle_end = int(asset_data["handleEnd"])
 
         for track in tracks:
+            self.log.debug(f"track.name: {track.name}")
             try:
                 track_start_frame = (
                     abs(track.source_range.start_time.value)
-                ) - self.timeline_frame_start
+                )
+                self.log.debug(f"track_start_frame: {track_start_frame}")
+                track_start_frame -= self.timeline_frame_start
             except AttributeError:
                 track_start_frame = 0
 
+            self.log.debug(f"track_start_frame: {track_start_frame}")
+
             for clip in track.each_child():
                 if clip.name is None:
                     continue

From 5bb609c512292fb803c0bf8168b5bbf37dc0eb13 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Tue, 10 Nov 2020 11:15:39 +0100
Subject: [PATCH 06/10] fix(SP): creating duplicity in instance data

---
 .../publish/collect_clear_instances.py        | 15 ++--
 .../publish/collect_instance_resources.py     | 79 +++++++++++++------
 .../publish/collect_instances.py              |  7 +-
 .../publish/extract_thumbnail.py              |  7 +-
 4 files changed, 76 insertions(+), 32 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py
index 0d64c57d3a..097e730251 100644
--- a/pype/plugins/standalonepublisher/publish/collect_clear_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_clear_instances.py
@@ -5,16 +5,17 @@ Optional:
 import pyblish.api
 
 
-class CollectClearInstances(pyblish.api.ContextPlugin):
+class CollectClearInstances(pyblish.api.InstancePlugin):
     """Clear all marked instances"""
 
     order = pyblish.api.CollectorOrder + 0.4999
     label = "Clear Instances"
     hosts = ["standalonepublisher"]
 
-    def process(self, context):
-
-        for instance in context:
-            if instance.data.get("remove"):
-                self.log.info(f"Removing: {instance}")
-                context.remove(instance)
+    def process(self, instance):
+        self.log.debug(
+            f"Instance: `{instance}` | "
+            f"families: `{instance.data['families']}`")
+        if instance.data.get("remove"):
+            self.log.info(f"Removing: {instance}")
+            instance.context.remove(instance)
diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
index 6807e82193..3b2f121608 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
@@ -17,8 +17,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
 
     def process(self, instance):
         self.context = instance.context
-        instance_data = instance.data
         self.log.info(f"Processing instance: {instance}")
+        self.new_instances = []
         subset_files = dict()
         subset_dirs = list()
         anatomy = self.context.data["anatomy"]
@@ -38,8 +38,8 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
             staging_dir = os.path.normpath(
                 tempfile.mkdtemp(prefix="pyblish_tmp_")
             )
-            instance_data["stagingDir"] = staging_dir
-            instance_data["families"] += ["trimming"]
+            instance.data["stagingDir"] = staging_dir
+            instance.data["families"] += ["trimming"]
             return
 
         # if template patern in path then fill it with `anatomy_data`
@@ -51,13 +51,14 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
         # loop `editorial_source_root` and find clip name in folders
         # and look for any subset name alternatives
         for root, dirs, files in os.walk(editorial_source_root):
+            # search only for directories related to clip name
             correct_clip_dir = None
-            for d in dirs:
+            for _d_search in dirs:
                 # avoid all non clip dirs
-                if d not in clip_name:
+                if _d_search not in clip_name:
                     continue
                 # found correct dir for clip
-                correct_clip_dir = d
+                correct_clip_dir = _d_search
 
             # continue if clip dir was not found
             if not correct_clip_dir:
@@ -83,22 +84,31 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
 
             if subset_files_items:
                 subset_files.update({clip_dir_path: subset_files_items})
+
+            # break the loop if correct_clip_dir was captured
+            # no need to cary on if corect folder was found
             if correct_clip_dir:
                 break
 
         if subset_dirs:
             # look all dirs and check for subset name alternatives
             for _dir in subset_dirs:
+                instance_data = deepcopy(
+                    {k: v for k, v in instance.data.items()})
                 sub_dir = os.path.basename(_dir)
                 # if subset name is only alternative then create new instance
                 if sub_dir != subset:
                     instance_data = self.duplicate_instance(
-                        instance.data, subset, sub_dir)
+                        instance_data, subset, sub_dir)
 
                 # create all representations
                 self.create_representations(
                     os.listdir(_dir), instance_data, _dir)
 
+                if sub_dir == subset:
+                    self.new_instances.append(instance_data)
+                    # instance.data.update(instance_data)
+
         if subset_files:
             unique_subset_names = list()
             root_dir = list(subset_files.keys()).pop()
@@ -120,15 +130,21 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
                 instance_data = self.duplicate_instance(
                     instance.data, subset, _un_subs)
 
-            # create all representations
-            self.create_representations(
-                [os.path.basename(f) for f in files_list],
-                instance_data, root_dir)
+                # create all representations
+                self.create_representations(
+                    [os.path.basename(f) for f in files_list
+                     if _un_subs in f],
+                    instance_data, root_dir)
 
-        # if the original subset name was not found in input folders
-        # then representations were nod added and it can be removed
-        if not instance.data.get("representations"):
-            instance.data["remove"] = True
+        # remove the original instance as it had been used only
+        # as template and is duplicated
+        self.context.remove(instance)
+
+        # create all instances in self.new_instances into context
+        for new_instance in self.new_instances:
+            _new_instance = self.context.create_instance(
+                new_instance["name"])
+            _new_instance.data.update(new_instance)
 
     def duplicate_instance(self, instance_data, subset, new_subset):
 
@@ -140,11 +156,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
             if subset in _value:
                 new_instance_data[_key] = _value.replace(
                     subset, new_subset)
-        new_instance = self.context.create_instance(
-            new_instance_data["name"])
-        new_instance.data.update(new_instance_data)
-        self.log.info(f"Creating new instance: {new_instance}")
-        return new_instance.data
+
+        self.log.info(f"Creating new instance: {new_instance_data['name']}")
+        self.new_instances.append(new_instance_data)
+        return new_instance_data
 
     def create_representations(
             self, files_list, instance_data, staging_dir):
@@ -159,9 +174,11 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
         # add representations to instance_data
         instance_data["representations"] = list()
 
+        collection_head_name = None
         # loop trough collections and create representations
         for _collection in collections:
             ext = _collection.tail
+            collection_head_name = _collection.head
             frame_start = list(_collection.indexes)[0]
             frame_end = list(_collection.indexes)[-1]
             repre_data = {
@@ -172,6 +189,17 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
                 "files": [item for item in _collection],
                 "stagingDir": staging_dir
             }
+
+            if "review" in instance_data["families"]:
+                repre_data.update({
+                    "thumbnail": True,
+                    "frameStartFtrack": frame_start,
+                    "frameEndFtrack": frame_end,
+                    "step": 1,
+                    "fps": self.context.data.get("fps"),
+                    "name": "review",
+                    "tags": ["review", "ftrackreview", "delete"],
+                })
             instance_data["representations"].append(repre_data)
 
             # add to frames for frame range reset
@@ -183,7 +211,11 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
             ext = os.path.splitext(_reminding_file)[-1]
             if ext not in instance_data["extensions"]:
                 continue
-
+            if collection_head_name and (
+                (collection_head_name + ext[1:]) not in _reminding_file
+            ) and (ext in [".mp4", ".mov"]):
+                self.log.info(f"Skipping file: {_reminding_file}")
+                continue
             frame_start = 1
             frame_end = 1
 
@@ -207,7 +239,10 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
                 frame_end = (
                     (instance_data["frameEnd"] - instance_data["frameStart"])
                     + 1)
-                instance_data["families"] += ["review", "ftrack"]
+                # add review ftrack family into families
+                for _family in ["review", "ftrack"]:
+                    if _family not in instance_data["families"]:
+                        instance_data["families"].append(_family)
                 repre_data.update({
                     "frameStart": frame_start,
                     "frameEnd": frame_end,
diff --git a/pype/plugins/standalonepublisher/publish/collect_instances.py b/pype/plugins/standalonepublisher/publish/collect_instances.py
index e99be5df38..090ffe2cbb 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instances.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instances.py
@@ -186,10 +186,15 @@ class CollectInstances(pyblish.api.InstancePlugin):
                         "subset": subset
                     })
                     # create new instance
-                    instance.context.create_instance(
+                    _instance = instance.context.create_instance(
                         **subset_instance_data)
+                    self.log.debug(
+                        f"Instance: `{_instance}` | "
+                        f"families: `{subset_instance_data['families']}`")
 
                 context.data["assetsShared"][name] = {
                     "_clipIn": clip_in,
                     "_clipOut": clip_out
                 }
+
+                self.log.debug("Instance: `{}` | families: `{}`")
diff --git a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py
index fca4039d0e..f9932db695 100644
--- a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py
+++ b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py
@@ -46,6 +46,7 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin):
             files_len = 1
             file = files
 
+        staging_dir = None
         is_jpeg = False
         if file.endswith(".jpeg") or file.endswith(".jpg"):
             is_jpeg = True
@@ -56,7 +57,8 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin):
 
         elif is_jpeg:
             # use first frame as thumbnail if is sequence of jpegs
-            full_thumbnail_path = file
+            staging_dir = thumbnail_repre.get("stagingDir")
+            full_thumbnail_path = os.path.join(staging_dir, file)
             self.log.info(
                 "For thumbnail is used file: {}".format(full_thumbnail_path)
             )
@@ -104,7 +106,7 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin):
         thumbnail_repre.pop("thumbnail")
 
         filename = os.path.basename(full_thumbnail_path)
-        staging_dir = os.path.dirname(full_thumbnail_path)
+        staging_dir = staging_dir or os.path.dirname(full_thumbnail_path)
 
         # create new thumbnail representation
         representation = {
@@ -119,4 +121,5 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin):
         if not is_jpeg:
             representation["tags"].append("delete")
 
+        self.log.info(f"New representation {representation}")
         instance.data["representations"].append(representation)

From 3c29bfb5bfad7acc2326ec88c1731e17965a5776 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Tue, 10 Nov 2020 11:51:44 +0100
Subject: [PATCH 07/10] hound(SP): suggestions

---
 .../standalonepublisher/publish/collect_instance_resources.py   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
index 3b2f121608..565d066fd8 100644
--- a/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
+++ b/pype/plugins/standalonepublisher/publish/collect_instance_resources.py
@@ -50,7 +50,7 @@ class CollectInstanceResources(pyblish.api.InstancePlugin):
         self.log.debug(f"root: {editorial_source_root}")
         # loop `editorial_source_root` and find clip name in folders
         # and look for any subset name alternatives
-        for root, dirs, files in os.walk(editorial_source_root):
+        for root, dirs, _files in os.walk(editorial_source_root):
             # search only for directories related to clip name
             correct_clip_dir = None
             for _d_search in dirs:

From 4118093e2c36885876b21e79b99334253ad32f95 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Tue, 10 Nov 2020 13:01:36 +0100
Subject: [PATCH 08/10] fix(SP): after merge 2xDev balast

---
 .../publish/extract_shot_data.py              | 92 -------------------
 1 file changed, 92 deletions(-)
 delete mode 100644 pype/plugins/standalonepublisher/publish/extract_shot_data.py

diff --git a/pype/plugins/standalonepublisher/publish/extract_shot_data.py b/pype/plugins/standalonepublisher/publish/extract_shot_data.py
deleted file mode 100644
index e4eb813bae..0000000000
--- a/pype/plugins/standalonepublisher/publish/extract_shot_data.py
+++ /dev/null
@@ -1,92 +0,0 @@
-import os
-import clique
-import pype.api
-
-from pprint import pformat
-
-
-class ExtractShotData(pype.api.Extractor):
-    """Extract shot "mov" and "wav" files."""
-
-    label = "Extract Shot Data"
-    hosts = ["standalonepublisher"]
-    families = ["clip"]
-
-    # presets
-
-    def process(self, instance):
-        representation = instance.data.get("representations")
-        self.log.debug(f"_ representation: {representation}")
-
-        if not representation:
-            instance.data["representations"] = list()
-
-        # get ffmpet path
-        ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg")
-
-        # get staging dir
-        staging_dir = self.staging_dir(instance)
-        self.log.info("Staging dir set to: `{}`".format(staging_dir))
-
-        # Generate mov file.
-        fps = instance.data["fps"]
-        video_file_path = instance.data["editorialVideoPath"]
-        ext = instance.data.get("extension", ".mov")
-
-        clip_trimed_path = os.path.join(
-            staging_dir, instance.data["name"] + ext)
-        #
-        # # check video file metadata
-        # input_data = plib.ffprobe_streams(video_file_path)[0]
-        # self.log.debug(f"__ input_data: `{input_data}`")
-
-        start = float(instance.data["clipInH"])
-        dur = float(instance.data["clipDurationH"])
-
-        if ext in ".wav":
-            start += 0.5
-
-        args = [
-            "\"{}\"".format(ffmpeg_path),
-            "-ss", str(start / fps),
-            "-i", f"\"{video_file_path}\"",
-            "-t", str(dur / fps)
-        ]
-        if ext in [".mov", ".mp4"]:
-            args.extend([
-                "-crf", "18",
-                "-pix_fmt", "yuv420p"])
-        elif ext in ".wav":
-            args.extend([
-                "-vn -acodec pcm_s16le",
-                "-ar 48000 -ac 2"
-            ])
-
-        # add output path
-        args.append(f"\"{clip_trimed_path}\"")
-
-        self.log.info(f"Processing: {args}")
-        ffmpeg_args = " ".join(args)
-        output = pype.api.subprocess(ffmpeg_args, shell=True)
-        self.log.info(output)
-
-        repr = {
-            "name": ext[1:],
-            "ext": ext[1:],
-            "files": os.path.basename(clip_trimed_path),
-            "stagingDir": staging_dir,
-            "frameStart": int(instance.data["frameStart"]),
-            "frameEnd": int(instance.data["frameEnd"]),
-            "frameStartFtrack": int(instance.data["frameStartH"]),
-            "frameEndFtrack": int(instance.data["frameEndH"]),
-            "fps": fps,
-        }
-
-        if ext[1:] in ["mov", "mp4"]:
-            repr.update({
-                "thumbnail": True,
-                "tags": ["review", "ftrackreview", "delete"]})
-
-        instance.data["representations"].append(repr)
-
-        self.log.debug(f"Instance data: {pformat(instance.data)}")

From dbeed281cb1ddcea1923965cd51409b3e934bee4 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Wed, 18 Nov 2020 12:13:25 +0100
Subject: [PATCH 09/10] feat(SP): dictionary way of adding tasks from preset

---
 .../publish/collect_hierarchy.py                    | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
index 45d2fb4160..b2d71084e4 100644
--- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
+++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
@@ -130,10 +130,17 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
         if self.shot_add_tasks:
             tasks_to_add = dict()
             project_tasks = io.find_one({"type": "project"})["config"]["tasks"]
-            for task in self.shot_add_tasks:
+            for task_name, task_data in self.shot_add_tasks.items():
                 for task_type in project_tasks.keys():
-                    if task_type.lower() in task.lower():
-                        tasks_to_add.update({task: {"type": task_type}})
+                    try:
+                        if task_type in task_data["type"]:
+                            tasks_to_add.update({task_name: task_data})
+                    except KeyError as error:
+                        self.log.error(
+                            "Wrong presets: `{}` \n"
+                            "example: {\"task_name\": {\"type\":"
+                            " \"FtrackTaskType\"}}".format(error)
+                        )
 
             instance.data["tasks"] = tasks_to_add
         else:

From 03d11f368884262f0029bbad93bed549c09f5d50 Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Wed, 18 Nov 2020 12:28:18 +0100
Subject: [PATCH 10/10] fix(SP): improving exception for nonexistent task type

---
 .../publish/collect_hierarchy.py              | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
index b2d71084e4..be36f30f4b 100644
--- a/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
+++ b/pype/plugins/standalonepublisher/publish/collect_hierarchy.py
@@ -131,16 +131,20 @@ class CollectHierarchyInstance(pyblish.api.ContextPlugin):
             tasks_to_add = dict()
             project_tasks = io.find_one({"type": "project"})["config"]["tasks"]
             for task_name, task_data in self.shot_add_tasks.items():
-                for task_type in project_tasks.keys():
-                    try:
-                        if task_type in task_data["type"]:
-                            tasks_to_add.update({task_name: task_data})
-                    except KeyError as error:
-                        self.log.error(
-                            "Wrong presets: `{}` \n"
-                            "example: {\"task_name\": {\"type\":"
-                            " \"FtrackTaskType\"}}".format(error)
-                        )
+                try:
+                    if task_data["type"] in project_tasks.keys():
+                        tasks_to_add.update({task_name: task_data})
+                    else:
+                        raise KeyError(
+                            "Wrong FtrackTaskType `{}` for `{}` is not"
+                            " existing in `{}``".format(
+                                task_data["type"],
+                                task_name,
+                                list(project_tasks.keys())))
+                except KeyError as error:
+                    raise KeyError(
+                        "Wrong presets: `{0}`".format(error)
+                    )
 
             instance.data["tasks"] = tasks_to_add
         else: