From 52feeabb447f4fd0c79ff324fd83e3caafd65678 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Tue, 8 Dec 2020 17:47:13 +0100
Subject: [PATCH 1/9] DWAA support

Redo of PR on 'master'
---
 pype/lib/__init__.py                          | 10 +-
 pype/lib/plugin_tools.py                      | 97 +++++++++++++++++++
 pype/plugins/global/publish/extract_burnin.py | 48 +++++++--
 pype/plugins/global/publish/extract_jpeg.py   | 29 ++++--
 pype/plugins/global/publish/extract_review.py | 42 ++++++--
 5 files changed, 200 insertions(+), 26 deletions(-)

diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py
index 78fd69da98..03cab2aad2 100644
--- a/pype/lib/__init__.py
+++ b/pype/lib/__init__.py
@@ -29,7 +29,11 @@ from .plugin_tools import (
     filter_pyblish_plugins,
     source_hash,
     get_unique_layer_name,
-    get_background_layers
+    get_background_layers,
+    oiio_supported,
+    decompress,
+    get_decompress_dir,
+    should_decompress
 )
 
 from .path_tools import (
@@ -64,6 +68,10 @@ __all__ = [
     "filter_pyblish_plugins",
     "get_unique_layer_name",
     "get_background_layers",
+    "oiio_supported",
+    "decompress",
+    "get_decompress_dir",
+    "should_decompress",
 
     "version_up",
     "get_version_from_path",
diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py
index f5eb354ca3..2fc99bcf16 100644
--- a/pype/lib/plugin_tools.py
+++ b/pype/lib/plugin_tools.py
@@ -5,6 +5,8 @@ import inspect
 import logging
 import re
 import json
+import pype.api
+import tempfile
 
 from ..api import config
 
@@ -134,3 +136,98 @@ def get_background_layers(file_url):
                                                layer.get("filename")).
                                   replace("\\", "/"))
     return layers
+
+
+def oiio_supported():
+    """
+        Checks if oiiotool is configured for this platform.
+
+        'should_decompress' will throw exception if configured,
+        but not present or working.
+    """
+    return os.getenv("PYPE_OIIO_PATH", "") != ""
+
+
+def decompress(target_dir, file_url,
+               input_frame_start=None, input_frame_end=None, log=None):
+    """
+        Decompresses DWAA 'file_url' .exr to 'target_dir'.
+
+        Creates uncompressed files in 'target_dir', they need to be cleaned.
+
+        File url could be for single file or for a sequence, in that case
+        %0Xd will be as a placeholder for frame number AND input_frame* will
+        be filled.
+        In that case single oiio command with '--frames' will be triggered for
+        all frames, this should be faster then looping and running sequentially
+
+        Args:
+            target_dir (str): extended from stagingDir
+            file_url (str): full urls to source file (with or without %0Xd)
+            input_frame_start (int) (optional): first frame
+            input_frame_end (int) (optional): last frame
+            log (Logger) (optional): pype logger
+    """
+    is_sequence = input_frame_start is not None and \
+        input_frame_end is not None and \
+        (int(input_frame_end) > int(input_frame_start))
+
+    oiio_cmd = []
+    oiio_cmd.append(os.getenv("PYPE_OIIO_PATH"))
+
+    oiio_cmd.append("--compression none")
+
+    base_file_name = os.path.basename(file_url)
+    oiio_cmd.append(file_url)
+
+    if is_sequence:
+        oiio_cmd.append("--frames {}-{}".format(input_frame_start,
+                                                input_frame_end))
+
+    oiio_cmd.append("-o")
+    oiio_cmd.append(os.path.join(target_dir, base_file_name))
+
+    subprocess_exr = " ".join(oiio_cmd)
+
+    if not log:
+        log = logging.getLogger(__name__)
+
+    log.debug("Decompressing {}".format(subprocess_exr))
+    pype.api.subprocess(
+        subprocess_exr, shell=True, logger=log
+    )
+
+
+def get_decompress_dir():
+    """
+        Creates temporary folder for decompressing.
+        Its local, in case of farm it is 'local' to the farm machine.
+
+        Should be much faster, needs to be cleaned up later.
+    """
+    return os.path.normpath(
+        tempfile.mkdtemp(prefix="pyblish_tmp_")
+    )
+
+
+def should_decompress(file_url):
+    """
+        Tests that 'file_url' is compressed with DWAA.
+
+        Uses 'oiio_supported' to check that OIIO tool is available for this
+        platform
+
+        Args:
+            file_url (str): path to rendered file (in sequence it would be
+                first file, if that compressed it is expected that whole seq
+                will be too)
+        Returns:
+            (bool): 'file_url' is DWAA compressed and should be decompressed
+    """
+    if oiio_supported():
+        output = pype.api.subprocess([os.getenv("PYPE_OIIO_PATH"),
+                                      "--info", "-v", file_url])
+        return "compression: \"dwaa\"" in output or \
+                "compression: \"dwab\"" in output
+
+    return False
diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py
index 5be5060590..79b02ed01c 100644
--- a/pype/plugins/global/publish/extract_burnin.py
+++ b/pype/plugins/global/publish/extract_burnin.py
@@ -6,6 +6,9 @@ import tempfile
 
 import pype.api
 import pyblish
+from pype.lib import oiio_supported, should_decompress, \
+    get_decompress_dir, decompress
+import shutil
 
 
 class ExtractBurnin(pype.api.Extractor):
@@ -28,7 +31,8 @@ class ExtractBurnin(pype.api.Extractor):
         "premiere",
         "standalonepublisher",
         "harmony",
-        "fusion"
+        "fusion",
+        "aftereffects"
     ]
     optional = True
 
@@ -54,15 +58,16 @@ class ExtractBurnin(pype.api.Extractor):
     def process(self, instance):
         # ffmpeg doesn't support multipart exrs
         if instance.data.get("multipartExr") is True:
-            instance_label = (
-                getattr(instance, "label", None)
-                or instance.data.get("label")
-                or instance.data.get("name")
-            )
-            self.log.info((
-                "Instance \"{}\" contain \"multipartExr\". Skipped."
-            ).format(instance_label))
-            return
+            if not oiio_supported():
+                instance_label = (
+                    getattr(instance, "label", None)
+                    or instance.data.get("label")
+                    or instance.data.get("name")
+                )
+                self.log.info((
+                    "Instance \"{}\" contain \"multipartExr\". Skipped."
+                ).format(instance_label))
+                return
 
         # QUESTION what is this for and should we raise an exception?
         if "representations" not in instance.data:
@@ -212,6 +217,26 @@ class ExtractBurnin(pype.api.Extractor):
                 # Prepare paths and files for process.
                 self.input_output_paths(new_repre, temp_data, filename_suffix)
 
+                decompressed_dir = ''
+                full_input_path = temp_data["full_input_path"]
+                do_decompress = should_decompress(full_input_path)
+                if do_decompress:
+                    decompressed_dir = get_decompress_dir()
+
+                    decompress(
+                        decompressed_dir,
+                        full_input_path,
+                        temp_data["frame_start"],
+                        temp_data["frame_end"],
+                        self.log
+                    )
+
+                    # input path changed, 'decompressed' added
+                    input_file = os.path.basename(full_input_path)
+                    temp_data["full_input_path"] = os.path.join(
+                        decompressed_dir,
+                        input_file)
+
                 # Data for burnin script
                 script_data = {
                     "input": temp_data["full_input_path"],
@@ -271,6 +296,9 @@ class ExtractBurnin(pype.api.Extractor):
                     os.remove(filepath)
                     self.log.debug("Removed: \"{}\"".format(filepath))
 
+            if do_decompress and os.path.exists(decompressed_dir):
+                shutil.rmtree(decompressed_dir)
+
     def prepare_basic_data(self, instance):
         """Pick data from instance for processing and for burnin strings.
 
diff --git a/pype/plugins/global/publish/extract_jpeg.py b/pype/plugins/global/publish/extract_jpeg.py
index 551e57796a..85bc60ddfc 100644
--- a/pype/plugins/global/publish/extract_jpeg.py
+++ b/pype/plugins/global/publish/extract_jpeg.py
@@ -3,6 +3,9 @@ import os
 import pyblish.api
 import pype.api
 import pype.lib
+from pype.lib import oiio_supported, should_decompress, \
+    get_decompress_dir, decompress
+import shutil
 
 
 class ExtractJpegEXR(pyblish.api.InstancePlugin):
@@ -22,9 +25,11 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
         if 'crypto' in instance.data['subset']:
             return
 
-        # ffmpeg doesn't support multipart exrs
+        do_decompress = False
+        # ffmpeg doesn't support multipart exrs, use oiiotool if available
         if instance.data.get("multipartExr") is True:
-            return
+            if not oiio_supported():
+                return
 
         # Skip review when requested.
         if not instance.data.get("review", True):
@@ -36,10 +41,6 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
         # filter out mov and img sequences
         representations_new = representations[:]
 
-        if instance.data.get("multipartExr"):
-            # ffmpeg doesn't support multipart exrs
-            return
-
         for repre in representations:
             tags = repre.get("tags", [])
             self.log.debug(repre)
@@ -60,6 +61,19 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
             full_input_path = os.path.join(stagingdir, input_file)
             self.log.info("input {}".format(full_input_path))
 
+            decompressed_dir = ''
+            do_decompress = should_decompress(full_input_path)
+            if do_decompress:
+                decompressed_dir = get_decompress_dir()
+
+                decompress(
+                    decompressed_dir,
+                    full_input_path)
+                # input path changed, 'decompressed' added
+                full_input_path = os.path.join(
+                    decompressed_dir,
+                    input_file)
+
             filename = os.path.splitext(input_file)[0]
             if not filename.endswith('.'):
                 filename += "."
@@ -111,4 +125,7 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
             self.log.debug("Adding: {}".format(representation))
             representations_new.append(representation)
 
+            if do_decompress and os.path.exists(decompressed_dir):
+                shutil.rmtree(decompressed_dir)
+
         instance.data["representations"] = representations_new
diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py
index aa8d8accb5..a40a943559 100644
--- a/pype/plugins/global/publish/extract_review.py
+++ b/pype/plugins/global/publish/extract_review.py
@@ -6,6 +6,8 @@ import pyblish.api
 import clique
 import pype.api
 import pype.lib
+from pype.lib import oiio_supported, should_decompress, \
+    get_decompress_dir, decompress
 
 
 class ExtractReview(pyblish.api.InstancePlugin):
@@ -14,7 +16,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
     Compulsory attribute of representation is tags list with "review",
     otherwise the representation is ignored.
 
-    All new represetnations are created and encoded by ffmpeg following
+    All new representations are created and encoded by ffmpeg following
     presets found in `pype-config/presets/plugins/global/
     publish.json:ExtractReview:outputs`.
     """
@@ -58,7 +60,9 @@ class ExtractReview(pyblish.api.InstancePlugin):
             return
 
         # ffmpeg doesn't support multipart exrs
-        if instance.data.get("multipartExr") is True:
+        if instance.data.get("multipartExr") is True \
+                and not oiio_supported():
+
             instance_label = (
                 getattr(instance, "label", None)
                 or instance.data.get("label")
@@ -318,9 +322,9 @@ class ExtractReview(pyblish.api.InstancePlugin):
         Args:
             output_def (dict): Currently processed output definition.
             instance (Instance): Currently processed instance.
-            new_repre (dict): Reprensetation representing output of this
+            new_repre (dict): Representation representing output of this
                 process.
-            temp_data (dict): Base data for successfull process.
+            temp_data (dict): Base data for successful process.
         """
 
         # Get FFmpeg arguments from profile presets
@@ -331,9 +335,29 @@ class ExtractReview(pyblish.api.InstancePlugin):
         ffmpeg_video_filters = out_def_ffmpeg_args.get("video_filters") or []
         ffmpeg_audio_filters = out_def_ffmpeg_args.get("audio_filters") or []
 
+        input_files_urls = [os.path.join(new_repre["stagingDir"], f) for f
+                            in new_repre['files']]
+        do_decompress = should_decompress(input_files_urls[0])
+        if do_decompress:
+            # change stagingDir, decompress first
+            # calculate all paths with modified directory, used on too many
+            # places
+            # will be purged by cleanup.py automatically
+            orig_staging_dir = new_repre["stagingDir"]
+            new_repre["stagingDir"] = get_decompress_dir()
+
         # Prepare input and output filepaths
         self.input_output_paths(new_repre, output_def, temp_data)
 
+        if do_decompress:
+            input_file = temp_data["full_input_path"].\
+                replace(new_repre["stagingDir"], orig_staging_dir)
+
+            decompress(new_repre["stagingDir"], input_file,
+                       temp_data["frame_start"],
+                       temp_data["frame_end"],
+                       self.log)
+
         # Set output frames len to 1 when ouput is single image
         if (
             temp_data["output_ext_is_image"]
@@ -930,7 +954,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
         return regexes
 
     def validate_value_by_regexes(self, value, in_list):
-        """Validates in any regexe from list match entered value.
+        """Validates in any regex from list match entered value.
 
         Args:
             in_list (list): List with regexes.
@@ -955,9 +979,9 @@ class ExtractReview(pyblish.api.InstancePlugin):
     def profile_exclusion(self, matching_profiles):
         """Find out most matching profile byt host, task and family match.
 
-        Profiles are selectivelly filtered. Each profile should have
+        Profiles are selectively filtered. Each profile should have
         "__value__" key with list of booleans. Each boolean represents
-        existence of filter for specific key (host, taks, family).
+        existence of filter for specific key (host, tasks, family).
         Profiles are looped in sequence. In each sequence are split into
         true_list and false_list. For next sequence loop are used profiles in
         true_list if there are any profiles else false_list is used.
@@ -1036,7 +1060,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
 
         highest_profile_points = -1
         # Each profile get 1 point for each matching filter. Profile with most
-        # points is returnd. For cases when more than one profile will match
+        # points is returned. For cases when more than one profile will match
         # are also stored ordered lists of matching values.
         for profile in self.profiles:
             profile_points = 0
@@ -1648,7 +1672,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
 
     def add_video_filter_args(self, args, inserting_arg):
         """
-        Fixing video filter argumets to be one long string
+        Fixing video filter arguments to be one long string
 
         Args:
             args (list): list of string arguments

From b74dc3a7a79bdf6067f8c7ea7d11e0088487cf5b Mon Sep 17 00:00:00 2001
From: Jakub Jezek <jakub@orbi.tools>
Date: Wed, 9 Dec 2020 10:07:18 +0100
Subject: [PATCH 2/9] fix(global): two types on repre["files"] support and
 better exception

oiio_supported didn't test path existence
---
 pype/lib/plugin_tools.py                      |  6 +++++-
 pype/plugins/global/publish/extract_review.py | 12 +++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py
index 2fc99bcf16..460665935a 100644
--- a/pype/lib/plugin_tools.py
+++ b/pype/lib/plugin_tools.py
@@ -145,7 +145,11 @@ def oiio_supported():
         'should_decompress' will throw exception if configured,
         but not present or working.
     """
-    return os.getenv("PYPE_OIIO_PATH", "") != ""
+    oiio_path = os.getenv("PYPE_OIIO_PATH", "")
+    if not os.path.exists(oiio_path) or not oiio_path:
+        raise IOError("Files do not exists in `{}`".format(oiio_path))
+
+    return True
 
 
 def decompress(target_dir, file_url,
diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py
index a40a943559..26e60fbd48 100644
--- a/pype/plugins/global/publish/extract_review.py
+++ b/pype/plugins/global/publish/extract_review.py
@@ -335,9 +335,15 @@ class ExtractReview(pyblish.api.InstancePlugin):
         ffmpeg_video_filters = out_def_ffmpeg_args.get("video_filters") or []
         ffmpeg_audio_filters = out_def_ffmpeg_args.get("audio_filters") or []
 
-        input_files_urls = [os.path.join(new_repre["stagingDir"], f) for f
-                            in new_repre['files']]
-        do_decompress = should_decompress(input_files_urls[0])
+        if isinstance(new_repre['files'], list):
+            input_files_urls = [os.path.join(new_repre["stagingDir"], f) for f
+                                in new_repre['files']]
+            do_decompress = should_decompress(input_files_urls[0])
+        else:
+            test_path = os.path.join(
+                new_repre["stagingDir"], new_repre['files'])
+            do_decompress = should_decompress(test_path)
+
         if do_decompress:
             # change stagingDir, decompress first
             # calculate all paths with modified directory, used on too many

From 5ca4f470970bebbc4aea7695a1d7bb7a639b6531 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Wed, 9 Dec 2020 11:30:24 +0100
Subject: [PATCH 3/9] Changed order of conditions

---
 pype/lib/plugin_tools.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py
index 460665935a..57019e6a72 100644
--- a/pype/lib/plugin_tools.py
+++ b/pype/lib/plugin_tools.py
@@ -146,7 +146,7 @@ def oiio_supported():
         but not present or working.
     """
     oiio_path = os.getenv("PYPE_OIIO_PATH", "")
-    if not os.path.exists(oiio_path) or not oiio_path:
+    if not oiio_path or not os.path.exists(oiio_path):
         raise IOError("Files do not exists in `{}`".format(oiio_path))
 
     return True

From 7d9a7c4625023ddd4082aea139b5b9d5065c1300 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Fri, 11 Dec 2020 21:59:57 +0100
Subject: [PATCH 4/9] Rework oiio_supported - return only boolean

Removed multipartExr part, not implemented yet, nothing to do with DWAA
Try catch added temporarily to maximalize chance to finish publish,
possible failures beause of DWAA and no oiio results in log and empty
result, not an exception and hard fail.
---
 pype/lib/plugin_tools.py                      | 18 ++++++++++++---
 pype/plugins/global/publish/extract_burnin.py | 19 ++++++++--------
 pype/plugins/global/publish/extract_jpeg.py   | 12 +++++++---
 pype/plugins/global/publish/extract_review.py | 22 ++++++++++++-------
 4 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py
index 57019e6a72..c2b938c9bb 100644
--- a/pype/lib/plugin_tools.py
+++ b/pype/lib/plugin_tools.py
@@ -142,12 +142,18 @@ def oiio_supported():
     """
         Checks if oiiotool is configured for this platform.
 
+        Expects full path to executable.
+
         'should_decompress' will throw exception if configured,
-        but not present or working.
+        but not present or not working.
+        Returns:
+            (bool)
     """
     oiio_path = os.getenv("PYPE_OIIO_PATH", "")
     if not oiio_path or not os.path.exists(oiio_path):
-        raise IOError("Files do not exists in `{}`".format(oiio_path))
+        log.debug("OIIOTool is not configured or not present at {}".
+                  format(oiio_path))
+        return False
 
     return True
 
@@ -219,7 +225,12 @@ def should_decompress(file_url):
         Tests that 'file_url' is compressed with DWAA.
 
         Uses 'oiio_supported' to check that OIIO tool is available for this
-        platform
+        platform.
+
+        Shouldn't throw exception as oiiotool is guarded by check function.
+        Currently implemented this way as there is no support for Mac and Linux
+        In the future, it should be more strict and throws exception on
+        misconfiguration.
 
         Args:
             file_url (str): path to rendered file (in sequence it would be
@@ -227,6 +238,7 @@ def should_decompress(file_url):
                 will be too)
         Returns:
             (bool): 'file_url' is DWAA compressed and should be decompressed
+                and we can decompress (oiiotool supported)
     """
     if oiio_supported():
         output = pype.api.subprocess([os.getenv("PYPE_OIIO_PATH"),
diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py
index 79b02ed01c..d9b12a5dba 100644
--- a/pype/plugins/global/publish/extract_burnin.py
+++ b/pype/plugins/global/publish/extract_burnin.py
@@ -58,16 +58,15 @@ class ExtractBurnin(pype.api.Extractor):
     def process(self, instance):
         # ffmpeg doesn't support multipart exrs
         if instance.data.get("multipartExr") is True:
-            if not oiio_supported():
-                instance_label = (
-                    getattr(instance, "label", None)
-                    or instance.data.get("label")
-                    or instance.data.get("name")
-                )
-                self.log.info((
-                    "Instance \"{}\" contain \"multipartExr\". Skipped."
-                ).format(instance_label))
-                return
+            instance_label = (
+                getattr(instance, "label", None)
+                or instance.data.get("label")
+                or instance.data.get("name")
+            )
+            self.log.info((
+                "Instance \"{}\" contain \"multipartExr\". Skipped."
+            ).format(instance_label))
+            return
 
         # QUESTION what is this for and should we raise an exception?
         if "representations" not in instance.data:
diff --git a/pype/plugins/global/publish/extract_jpeg.py b/pype/plugins/global/publish/extract_jpeg.py
index 85bc60ddfc..f667382665 100644
--- a/pype/plugins/global/publish/extract_jpeg.py
+++ b/pype/plugins/global/publish/extract_jpeg.py
@@ -28,8 +28,7 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
         do_decompress = False
         # ffmpeg doesn't support multipart exrs, use oiiotool if available
         if instance.data.get("multipartExr") is True:
-            if not oiio_supported():
-                return
+            return
 
         # Skip review when requested.
         if not instance.data.get("review", True):
@@ -107,7 +106,14 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
 
             # run subprocess
             self.log.debug("{}".format(subprocess_jpeg))
-            pype.api.subprocess(subprocess_jpeg, shell=True)
+            try:  # temporary until oiiotool is supported cross platform
+                pype.api.subprocess(subprocess_jpeg, shell=True)
+            except RuntimeError as exp:
+                if "Compression" in str(exp):
+                    self.log.debug("Unsupported compression on input files. " +
+                                   "Skipping!!!")
+                    return
+                raise
 
             if "representations" not in instance.data:
                 instance.data["representations"] = []
diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py
index 26e60fbd48..e0caba1a20 100644
--- a/pype/plugins/global/publish/extract_review.py
+++ b/pype/plugins/global/publish/extract_review.py
@@ -60,9 +60,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
             return
 
         # ffmpeg doesn't support multipart exrs
-        if instance.data.get("multipartExr") is True \
-                and not oiio_supported():
-
+        if instance.data.get("multipartExr") is True:
             instance_label = (
                 getattr(instance, "label", None)
                 or instance.data.get("label")
@@ -192,9 +190,17 @@ class ExtractReview(pyblish.api.InstancePlugin):
 
                 temp_data = self.prepare_temp_data(instance, repre, output_def)
 
-                ffmpeg_args = self._ffmpeg_arguments(
-                    output_def, instance, new_repre, temp_data
-                )
+                try:  # temporary until oiiotool is supported cross platform
+                    ffmpeg_args = self._ffmpeg_arguments(
+                        output_def, instance, new_repre, temp_data
+                    )
+                except ZeroDivisionError:
+                    if 'exr' in temp_data["origin_repre"]["ext"]:
+                        self.log.debug("Unsupported compression on input " +
+                                       "files. Skipping!!!")
+                        return
+                    raise
+
                 subprcs_cmd = " ".join(ffmpeg_args)
 
                 # run subprocess
@@ -338,11 +344,11 @@ class ExtractReview(pyblish.api.InstancePlugin):
         if isinstance(new_repre['files'], list):
             input_files_urls = [os.path.join(new_repre["stagingDir"], f) for f
                                 in new_repre['files']]
-            do_decompress = should_decompress(input_files_urls[0])
+            test_path = input_files_urls[0]
         else:
             test_path = os.path.join(
                 new_repre["stagingDir"], new_repre['files'])
-            do_decompress = should_decompress(test_path)
+        do_decompress = should_decompress(test_path)
 
         if do_decompress:
             # change stagingDir, decompress first

From 2ca672b7b78ec457b9a4f480ba127078738d74b0 Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Fri, 11 Dec 2020 22:06:53 +0100
Subject: [PATCH 5/9] Hound

---
 pype/lib/plugin_tools.py                      | 7 ++++---
 pype/plugins/global/publish/extract_burnin.py | 2 +-
 pype/plugins/global/publish/extract_jpeg.py   | 2 +-
 pype/plugins/global/publish/extract_review.py | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py
index c2b938c9bb..d310ac2b8d 100644
--- a/pype/lib/plugin_tools.py
+++ b/pype/lib/plugin_tools.py
@@ -241,9 +241,10 @@ def should_decompress(file_url):
                 and we can decompress (oiiotool supported)
     """
     if oiio_supported():
-        output = pype.api.subprocess([os.getenv("PYPE_OIIO_PATH"),
-                                      "--info", "-v", file_url])
+        output = pype.api.subprocess([
+            os.getenv("PYPE_OIIO_PATH"),
+            "--info", "-v", file_url])
         return "compression: \"dwaa\"" in output or \
-                "compression: \"dwab\"" in output
+            "compression: \"dwab\"" in output
 
     return False
diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py
index d9b12a5dba..f776846eab 100644
--- a/pype/plugins/global/publish/extract_burnin.py
+++ b/pype/plugins/global/publish/extract_burnin.py
@@ -6,7 +6,7 @@ import tempfile
 
 import pype.api
 import pyblish
-from pype.lib import oiio_supported, should_decompress, \
+from pype.lib import should_decompress, \
     get_decompress_dir, decompress
 import shutil
 
diff --git a/pype/plugins/global/publish/extract_jpeg.py b/pype/plugins/global/publish/extract_jpeg.py
index f667382665..af90d4366d 100644
--- a/pype/plugins/global/publish/extract_jpeg.py
+++ b/pype/plugins/global/publish/extract_jpeg.py
@@ -3,7 +3,7 @@ import os
 import pyblish.api
 import pype.api
 import pype.lib
-from pype.lib import oiio_supported, should_decompress, \
+from pype.lib import should_decompress, \
     get_decompress_dir, decompress
 import shutil
 
diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py
index e0caba1a20..37fe83bf10 100644
--- a/pype/plugins/global/publish/extract_review.py
+++ b/pype/plugins/global/publish/extract_review.py
@@ -6,7 +6,7 @@ import pyblish.api
 import clique
 import pype.api
 import pype.lib
-from pype.lib import oiio_supported, should_decompress, \
+from pype.lib import should_decompress, \
     get_decompress_dir, decompress
 
 

From 86ff90dad71e55f66ebc0d54b0c20a279c9f9080 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <ondrej.samohel@pype.club>
Date: Tue, 15 Dec 2020 18:58:52 +0100
Subject: [PATCH 6/9] handle referenced AOVs

---
 pype/hosts/maya/expected_files.py             | 67 ++++++++++++-------
 pype/plugins/maya/create/create_render.py     |  1 +
 pype/plugins/maya/publish/collect_render.py   |  5 +-
 .../publish/validate_vray_referenced_aovs.py  | 50 ++++++++++++++
 4 files changed, 98 insertions(+), 25 deletions(-)
 create mode 100644 pype/plugins/maya/publish/validate_vray_referenced_aovs.py

diff --git a/pype/hosts/maya/expected_files.py b/pype/hosts/maya/expected_files.py
index a2ddec1640..174876db4e 100644
--- a/pype/hosts/maya/expected_files.py
+++ b/pype/hosts/maya/expected_files.py
@@ -32,6 +32,9 @@ Attributes:
     ImagePrefixes (dict): Mapping between renderers and their respective
         image prefix atrribute names.
 
+Todo:
+    Determine `multipart` from render instance.
+
 """
 
 import types
@@ -94,6 +97,10 @@ class ExpectedFiles:
 
     multipart = False
 
+    def __init__(self, render_instance):
+        """Constructor."""
+        self._render_instance = render_instance
+
     def get(self, renderer, layer):
         """Get expected files for given renderer and render layer.
 
@@ -114,15 +121,20 @@ class ExpectedFiles:
         renderSetup.instance().switchToLayerUsingLegacyName(layer)
 
         if renderer.lower() == "arnold":
-            return self._get_files(ExpectedFilesArnold(layer))
+            return self._get_files(ExpectedFilesArnold(layer,
+                                                       self._render_instance))
         elif renderer.lower() == "vray":
-            return self._get_files(ExpectedFilesVray(layer))
+            return self._get_files(ExpectedFilesVray(
+                layer, self._render_instance))
         elif renderer.lower() == "redshift":
-            return self._get_files(ExpectedFilesRedshift(layer))
+            return self._get_files(ExpectedFilesRedshift(
+                layer, self._render_instance))
         elif renderer.lower() == "mentalray":
-            return self._get_files(ExpectedFilesMentalray(layer))
+            return self._get_files(ExpectedFilesMentalray(
+                layer, self._render_instance))
         elif renderer.lower() == "renderman":
-            return self._get_files(ExpectedFilesRenderman(layer))
+            return self._get_files(ExpectedFilesRenderman(
+                layer, self._render_instance))
         else:
             raise UnsupportedRendererException(
                 "unsupported {}".format(renderer)
@@ -149,9 +161,10 @@ class AExpectedFiles:
     layer = None
     multipart = False
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Constructor."""
         self.layer = layer
+        self.render_instance = render_instance
 
     @abstractmethod
     def get_aovs(self):
@@ -460,9 +473,9 @@ class ExpectedFilesArnold(AExpectedFiles):
         "maya": "",
     }
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Constructor."""
-        super(ExpectedFilesArnold, self).__init__(layer)
+        super(ExpectedFilesArnold, self).__init__(layer, render_instance)
         self.renderer = "arnold"
 
     def get_aovs(self):
@@ -531,9 +544,9 @@ class ExpectedFilesArnold(AExpectedFiles):
 class ExpectedFilesVray(AExpectedFiles):
     """Expected files for V-Ray renderer."""
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Constructor."""
-        super(ExpectedFilesVray, self).__init__(layer)
+        super(ExpectedFilesVray, self).__init__(layer, render_instance)
         self.renderer = "vray"
 
     def get_renderer_prefix(self):
@@ -615,14 +628,22 @@ class ExpectedFilesVray(AExpectedFiles):
             default_ext = "exr"
 
         # filter all namespace prefixed AOVs - they are pulled in from
-        # references and are not rendered.
-        vr_aovs = [
-            n
-            for n in cmds.ls(
-                type=["VRayRenderElement", "VRayRenderElementSet"]
-            )
-            if len(n.split(":")) == 1
-        ]
+        # references. Or leave them alone, based on preferences on render
+        # instance.
+        ref_aovs = self.render_instance.data.get(
+            "vrayUseReferencedAovs", False) or False
+
+        if ref_aovs:
+            vr_aovs = cmds.ls(
+                type=["VRayRenderElement", "VRayRenderElementSet"])
+        else:
+            vr_aovs = [
+                n
+                for n in cmds.ls(
+                    type=["VRayRenderElement", "VRayRenderElementSet"]
+                )
+                if len(n.split(":")) == 1
+            ]
 
         for aov in vr_aovs:
             enabled = self.maya_is_true(cmds.getAttr("{}.enabled".format(aov)))
@@ -696,9 +717,9 @@ class ExpectedFilesRedshift(AExpectedFiles):
 
     ext_mapping = ["iff", "exr", "tif", "png", "tga", "jpg"]
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Construtor."""
-        super(ExpectedFilesRedshift, self).__init__(layer)
+        super(ExpectedFilesRedshift, self).__init__(layer, render_instance)
         self.renderer = "redshift"
 
     def get_renderer_prefix(self):
@@ -815,9 +836,9 @@ class ExpectedFilesRenderman(AExpectedFiles):
         This is very rudimentary and needs more love and testing.
     """
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Constructor."""
-        super(ExpectedFilesRenderman, self).__init__(layer)
+        super(ExpectedFilesRenderman, self).__init__(layer, render_instance)
         self.renderer = "renderman"
 
     def get_aovs(self):
@@ -880,7 +901,7 @@ class ExpectedFilesRenderman(AExpectedFiles):
 class ExpectedFilesMentalray(AExpectedFiles):
     """Skeleton unimplemented class for Mentalray renderer."""
 
-    def __init__(self, layer):
+    def __init__(self, layer, render_instance):
         """Constructor.
 
         Raises:
diff --git a/pype/plugins/maya/create/create_render.py b/pype/plugins/maya/create/create_render.py
index fa0e269126..5a4f8f9dcb 100644
--- a/pype/plugins/maya/create/create_render.py
+++ b/pype/plugins/maya/create/create_render.py
@@ -189,6 +189,7 @@ class CreateRender(avalon.maya.Creator):
         self.data["tilesX"] = 2
         self.data["tilesY"] = 2
         self.data["convertToScanline"] = False
+        self.data["vrayUseReferencedAovs"] = False
         # Disable for now as this feature is not working yet
         # self.data["assScene"] = False
 
diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py
index 3dde3b1592..0853473120 100644
--- a/pype/plugins/maya/publish/collect_render.py
+++ b/pype/plugins/maya/publish/collect_render.py
@@ -149,7 +149,7 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
 
             # return all expected files for all cameras and aovs in given
             # frame range
-            ef = ExpectedFiles()
+            ef = ExpectedFiles(render_instance)
             exp_files = ef.get(renderer, layer_name)
             self.log.info("multipart: {}".format(ef.multipart))
             assert exp_files, "no file names were generated, this is bug"
@@ -248,7 +248,8 @@ class CollectMayaRender(pyblish.api.ContextPlugin):
                 "tilesX": render_instance.data.get("tilesX") or 2,
                 "tilesY": render_instance.data.get("tilesY") or 2,
                 "priority": render_instance.data.get("priority"),
-                "convertToScanline": render_instance.data.get("convertToScanline") or False  # noqa: E501
+                "convertToScanline": render_instance.data.get("convertToScanline") or False,  # noqa: E501
+                "vrayUseReferencedAovs": render_instance.data.get("vrayUseReferencedAovs") or False  # noqa: E501
             }
 
             if self.sync_workfile_version:
diff --git a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
new file mode 100644
index 0000000000..923cb06263
--- /dev/null
+++ b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+"""Validate if there are AOVs pulled from references."""
+import pyblish.api
+import pype.api
+
+from maya import cmds
+
+import pype.hosts.maya.action
+
+
+class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
+    """Validate whether the V-Ray Render Elements (AOVs) include references.
+
+    This will check if there are AOVs pulled from references. If
+    `Vray Use Referenced Aovs` is checked on render instance, u must add those
+    manually to Render Elements as Pype will expect them to be rendered.
+
+    """
+
+    order = pyblish.api.ValidatorOrder
+    label = 'VRay Referenced AOVs'
+    hosts = ['maya']
+    families = ['renderlayer']
+    actions = [pype.hosts.maya.action.SelectInvalidAction]
+
+    def process(self, instance):
+        """Plugin main entry point."""
+        if instance.data.get("renderer") != "vray":
+            # If not V-Ray ignore..
+            return
+
+        if not instance.data.get("vrayUseReferencedAovs"):
+            self.get_invalid(instance)
+
+    @classmethod
+    def get_invalid(cls, instance):
+        """Find referenced AOVs in scene."""
+        # those aovs with namespace prefix are coming from references
+        ref_aovs = [
+            n for n in
+            cmds.ls(type=["VRayRenderElement", "VRayRenderElementSet"])
+            if len(n.split(":")) > 1
+        ]
+
+        if ref_aovs:
+            cls.log.warning(
+                "Scene contain referenced AOVs: {}".format(ref_aovs))
+
+            # Return the instance itself
+            return ref_aovs

From 5e8aca606ca36eefe0cadb65309de4f8ed85a89f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <ondrej.samohel@pype.club>
Date: Tue, 15 Dec 2020 19:09:05 +0100
Subject: [PATCH 7/9] better check for referenced AOVs

---
 pype/hosts/maya/expected_files.py                    | 12 ++++--------
 .../maya/publish/validate_vray_referenced_aovs.py    | 11 +++++------
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/pype/hosts/maya/expected_files.py b/pype/hosts/maya/expected_files.py
index 174876db4e..9dd10e573e 100644
--- a/pype/hosts/maya/expected_files.py
+++ b/pype/hosts/maya/expected_files.py
@@ -635,15 +635,11 @@ class ExpectedFilesVray(AExpectedFiles):
 
         if ref_aovs:
             vr_aovs = cmds.ls(
-                type=["VRayRenderElement", "VRayRenderElementSet"])
+                type=["VRayRenderElement", "VRayRenderElementSet"]) or []
         else:
-            vr_aovs = [
-                n
-                for n in cmds.ls(
-                    type=["VRayRenderElement", "VRayRenderElementSet"]
-                )
-                if len(n.split(":")) == 1
-            ]
+            vr_aovs = cmds.ls(
+                type=["VRayRenderElement", "VRayRenderElementSet"],
+                referencedNodes=False) or []
 
         for aov in vr_aovs:
             enabled = self.maya_is_true(cmds.getAttr("{}.enabled".format(aov)))
diff --git a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
index 923cb06263..0c1a5f552a 100644
--- a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
+++ b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
@@ -35,12 +35,11 @@ class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
     @classmethod
     def get_invalid(cls, instance):
         """Find referenced AOVs in scene."""
-        # those aovs with namespace prefix are coming from references
-        ref_aovs = [
-            n for n in
-            cmds.ls(type=["VRayRenderElement", "VRayRenderElementSet"])
-            if len(n.split(":")) > 1
-        ]
+
+        if cmds.getAttr("vraySettings.relements_usereferenced") == 0:
+            ref_aovs = cmds.ls(
+                type=["VRayRenderElement", "VRayRenderElementSet"],
+                referencedNodes=True) or []
 
         if ref_aovs:
             cls.log.warning(

From 6fb3cfafdd92b8c63187cd64912bd87f8ef19e54 Mon Sep 17 00:00:00 2001
From: Ondrej Samohel <ondrej.samohel@pype.club>
Date: Wed, 16 Dec 2020 18:44:04 +0100
Subject: [PATCH 8/9] repair action for validator, skip unwanted referenced
 aovs

---
 pype/hosts/maya/expected_files.py             | 21 +++---
 .../publish/validate_vray_referenced_aovs.py  | 73 +++++++++++++++----
 2 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/pype/hosts/maya/expected_files.py b/pype/hosts/maya/expected_files.py
index 9dd10e573e..07b3f94aa0 100644
--- a/pype/hosts/maya/expected_files.py
+++ b/pype/hosts/maya/expected_files.py
@@ -627,19 +627,20 @@ class ExpectedFilesVray(AExpectedFiles):
         if default_ext == "exr (multichannel)" or default_ext == "exr (deep)":
             default_ext = "exr"
 
-        # filter all namespace prefixed AOVs - they are pulled in from
-        # references. Or leave them alone, based on preferences on render
-        # instance.
-        ref_aovs = self.render_instance.data.get(
+        # handle aovs from references
+        use_ref_aovs = self.render_instance.data.get(
             "vrayUseReferencedAovs", False) or False
 
-        if ref_aovs:
-            vr_aovs = cmds.ls(
-                type=["VRayRenderElement", "VRayRenderElementSet"]) or []
-        else:
-            vr_aovs = cmds.ls(
+        # this will have list of all aovs no matter if they are coming from
+        # reference or not.
+        vr_aovs = cmds.ls(
+            type=["VRayRenderElement", "VRayRenderElementSet"]) or []
+        if not use_ref_aovs:
+            ref_aovs = cmds.ls(
                 type=["VRayRenderElement", "VRayRenderElementSet"],
-                referencedNodes=False) or []
+                referencedNodes=True) or []
+            # get difference
+            vr_aovs = list(set(vr_aovs) - set(ref_aovs))
 
         for aov in vr_aovs:
             enabled = self.maya_is_true(cmds.getAttr("{}.enabled".format(aov)))
diff --git a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
index 0c1a5f552a..67d5ed558c 100644
--- a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
+++ b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 """Validate if there are AOVs pulled from references."""
 import pyblish.api
-import pype.api
-
+import types
 from maya import cmds
 
 import pype.hosts.maya.action
@@ -21,7 +20,7 @@ class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
     label = 'VRay Referenced AOVs'
     hosts = ['maya']
     families = ['renderlayer']
-    actions = [pype.hosts.maya.action.SelectInvalidAction]
+    actions = [pype.api.RepairContextAction]
 
     def process(self, instance):
         """Plugin main entry point."""
@@ -29,21 +28,65 @@ class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
             # If not V-Ray ignore..
             return
 
+        ref_aovs = cmds.ls(
+            type=["VRayRenderElement", "VRayRenderElementSet"],
+            referencedNodes=True)
+        ref_aovs_enabled = ValidateVrayReferencedAOVs.maya_is_true(
+            cmds.getAttr("vraySettings.relements_usereferenced"))
+
         if not instance.data.get("vrayUseReferencedAovs"):
-            self.get_invalid(instance)
+            if ref_aovs_enabled and ref_aovs:
+                self.log.warning((
+                    "Referenced AOVs are enabled in Vray "
+                    "Render Settings and are detected in scene, but "
+                    "Pype render instance option for referenced AOVs is "
+                    "disabled. Those AOVs will be rendered but not published "
+                    "by Pype."
+                ))
+                self.log.warning(", ".join(ref_aovs))
+        else:
+            if not ref_aovs:
+                self.log.warning((
+                    "Use of referenced AOVs enabled but there are none "
+                    "in the scene."
+                ))
+            if not ref_aovs_enabled:
+                self.log.error((
+                    "'Use referenced' not enabled in Vray Render Settings."
+                ))
+                raise AssertionError("Invalid render settings")
 
     @classmethod
-    def get_invalid(cls, instance):
-        """Find referenced AOVs in scene."""
+    def repair(cls, context):
 
-        if cmds.getAttr("vraySettings.relements_usereferenced") == 0:
-            ref_aovs = cmds.ls(
-                type=["VRayRenderElement", "VRayRenderElementSet"],
-                referencedNodes=True) or []
+        vray_settings = cmds.ls(type="VRaySettingsNode")
+        if not vray_settings:
+            node = cmds.createNode("VRaySettingsNode")
+        else:
+            node = vray_settings[0]
 
-        if ref_aovs:
-            cls.log.warning(
-                "Scene contain referenced AOVs: {}".format(ref_aovs))
+        cmds.setAttr("{}.relements_usereferenced".format(node), True)
 
-            # Return the instance itself
-            return ref_aovs
+
+
+    @staticmethod
+    def maya_is_true(attr_val):
+        """Whether a Maya attr evaluates to True.
+
+        When querying an attribute value from an ambiguous object the
+        Maya API will return a list of values, which need to be properly
+        handled to evaluate properly.
+
+        Args:
+            attr_val (mixed): Maya attribute to be evaluated as bool.
+
+        Returns:
+            bool: cast Maya attribute to Pythons boolean value.
+
+        """
+        if isinstance(attr_val, types.BooleanType):
+            return attr_val
+        elif isinstance(attr_val, (types.ListType, types.GeneratorType)):
+            return any(attr_val)
+        else:
+            return bool(attr_val)

From 183058a6c4e836b5601399e89e0dd98198a12d55 Mon Sep 17 00:00:00 2001
From: Ondrej Samohel <ondrej.samohel@pype.club>
Date: Wed, 16 Dec 2020 18:46:41 +0100
Subject: [PATCH 9/9] shut the hound up

---
 pype/plugins/maya/publish/validate_vray_referenced_aovs.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
index 67d5ed558c..120677021d 100644
--- a/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
+++ b/pype/plugins/maya/publish/validate_vray_referenced_aovs.py
@@ -58,7 +58,7 @@ class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
 
     @classmethod
     def repair(cls, context):
-
+        """Repair action."""
         vray_settings = cmds.ls(type="VRaySettingsNode")
         if not vray_settings:
             node = cmds.createNode("VRaySettingsNode")
@@ -67,8 +67,6 @@ class ValidateVrayReferencedAOVs(pyblish.api.InstancePlugin):
 
         cmds.setAttr("{}.relements_usereferenced".format(node), True)
 
-
-
     @staticmethod
     def maya_is_true(attr_val):
         """Whether a Maya attr evaluates to True.