Merge branch 'develop' into feature/PYPE-81-nuke-write-render-workflow

# Conflicts:
#	pype/nuke/lib.py

pype/nuke/lib.py

@@ -223,7 +223,8 @@ def create_write_node(name, data, prenodes=None):
     fpath = data["fpath_template"].format(
         work=fpath, version=data["version"], subset=data["subset"],
         frame=data["frame"],
-        ext=data["nuke_dataflow_writes"]["file_type"])
+        ext=data["nuke_dataflow_writes"]["file_type"]
+    )
 
     # create directory
     if not os.path.isdir(os.path.dirname(fpath)):

pype/plugins/global/publish/cleanup.py

@@ -3,11 +3,33 @@ import shutil
 import pyblish.api
 
 
+def clean_renders(instance):
+    transfers = instance.data.get("transfers", list())
+
+    current_families = instance.data.get("families", list())
+    instance_family = instance.data.get("family", None)
+    dirnames = []
+
+    for src, dest in transfers:
+        if os.path.normpath(src) != os.path.normpath(dest):
+            if instance_family == 'render' or 'render' in current_families:
+                os.remove(src)
+                dirnames.append(os.path.dirname(src))
+
+    # make unique set
+    cleanup_dirs = set(dirnames)
+    for dir in cleanup_dirs:
+        try:
+            os.rmdir(dir)
+        except OSError:
+            # directory is not empty, skipping
+            continue
+
+
 class CleanUp(pyblish.api.InstancePlugin):
     """Cleans up the staging directory after a successful publish.
 
-    The removal will only happen for staging directories which are inside the
+    This will also clean published renders and delete their parent directories.
-    temporary folder, otherwise the folder is ignored.
 
     """
 
@@ -36,3 +58,5 @@ class CleanUp(pyblish.api.InstancePlugin):
 
         self.log.info("Removing temporary folder ...")
         shutil.rmtree(staging_dir)
+        self.log.info("Cleaning renders ...")
+        clean_renders(instance)

pype/plugins/global/publish/collect_filesequences.py

@@ -160,10 +160,13 @@ class CollectRenderedFrames(pyblish.api.ContextPlugin):
 
             # Get family from the data
             families = data.get("families", ["render"])
-            assert isinstance(families, (list, tuple)), "Must be iterable"
+            if "render" not in families:
-            assert families, "Must have at least a single family"
+                families.append("render")
-            families.append("ftrack")
+            if "ftrack" not in families:
-            families.append("review")
+                families.append("ftrack")
+            if "review" not in families:
+                families.append("review")
+
             for collection in collections:
                 instance = context.create_instance(str(collection))
                 self.log.info("Collection: %s" % list(collection))

pype/plugins/global/publish/extract_burnin.py

@@ -61,19 +61,45 @@ class ExtractBurnin(pype.api.Extractor):
             self.log.debug("__ burnin_data2: {}".format(burnin_data))
 
             json_data = json.dumps(burnin_data)
-            scriptpath = os.path.normpath(os.path.join(os.environ['PYPE_MODULE_ROOT'],
+
-                                      "pype",
+            # Get script path.
-                                      "scripts",
+            module_path = os.environ['PYPE_MODULE_ROOT']
-                                      "otio_burnin.py"))
+
+            # There can be multiple paths in PYPE_MODULE_ROOT, in which case
+            # we just take first one.
+            if os.pathsep in module_path:
+                module_path = module_path.split(os.pathsep)[0]
+
+            scriptpath = os.path.normpath(
+                os.path.join(
+                    module_path,
+                    "pype",
+                    "scripts",
+                    "otio_burnin.py"
+                )
+            )
 
             self.log.debug("__ scriptpath: {}".format(scriptpath))
-            self.log.debug("__ EXE: {}".format(os.getenv("PYPE_PYTHON_EXE")))
+
+            # Get executable.
+            executable = os.getenv("PYPE_PYTHON_EXE")
+
+            # There can be multiple paths in PYPE_PYTHON_EXE, in which case
+            # we just take first one.
+            if os.pathsep in executable:
+                executable = executable.split(os.pathsep)[0]
+
+            self.log.debug("__ EXE: {}".format(executable))
 
             try:
-                p = subprocess.Popen(
+                args = [executable, scriptpath, json_data]
-                    [os.getenv("PYPE_PYTHON_EXE"), scriptpath, json_data]
+                self.log.debug("Executing: {}".format(args))
-                )
+
+                # Explicitly passing the environment, because there are cases
+                # where enviroment is not inherited.
+                p = subprocess.Popen(args, env=os.environ)
                 p.wait()
+
                 if not os.path.isfile(full_burnin_path):
                     raise RuntimeError("File not existing: {}".format(full_burnin_path))
             except Exception as e:

pype/plugins/global/publish/integrate_new.py

@@ -403,20 +403,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         self.log.info("Registered {} items".format(len(representations)))
 
     def integrate(self, instance):
-        """Move the files
+        """ Move the files.
 
-        Through `instance.data["transfers"]`
+            Through `instance.data["transfers"]`
 
-        Args:
+            Args:
-            instance: the instance to integrate
+                instance: the instance to integrate
         """
 
-        transfers = instance.data.get("transfers", list())
-
-        for src, dest in transfers:
-            if os.path.normpath(src) != os.path.normpath(dest):
-                self.copy_file(src, dest)
-
         # Produce hardlinked copies
         # Note: hardlink can only be produced between two files on the same
         # server/disk and editing one of the two will edit both files at once.

pype/plugins/maya/publish/submit_maya_deadline.py

@@ -51,7 +51,7 @@ def get_renderer_variables(renderlayer=None):
         # returns an index number.
         filename_base = os.path.basename(filename_0)
         extension = os.path.splitext(filename_base)[-1].strip(".")
-        filename_prefix = "<Scene>/<RenderLayer>/<RenderLayer>"
+        filename_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix")
 
     return {"ext": extension,
             "filename_prefix": filename_prefix,
@@ -77,8 +77,19 @@ def preview_fname(folder, scene, layer, padding, ext):
 
     """
 
-    # Following hardcoded "<Scene>/<Scene>_<Layer>/<Layer>"
+    fileprefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix")
-    output = "{scene}/{layer}/{layer}.{number}.{ext}".format(
+    output = fileprefix + ".{number}.{ext}"
+    # RenderPass is currently hardcoded to "beauty" because its not important
+    # for the deadline submission, but we will need something to replace
+    # "<RenderPass>".
+    mapping = {
+        "<Scene>": "{scene}",
+        "<RenderLayer>": "{layer}",
+        "RenderPass": "beauty"
+    }
+    for key, value in mapping.items():
+        output = output.replace(key, value)
+    output = output.format(
         scene=scene,
         layer=layer,
         number="#" * padding,

pype/plugins/maya/publish/submit_maya_muster.py

@@ -250,8 +250,15 @@ class MayaSubmitMuster(pyblish.api.InstancePlugin):
         render publish job and submit job to farm.
         """
         # setup muster environment
-        self.MUSTER_REST_URL = os.environ.get("MUSTER_REST_URL",
+        self.MUSTER_REST_URL = os.environ.get("MUSTER_REST_URL")
-                                              "https://localhost:9891")
+
+        if self.MUSTER_REST_URL is None:
+            self.log.debug(
+                "\"MUSTER_REST_URL\" is not found. Skipping "
+                "\"{}\".".format(instance)
+            )
+            return
+
         self._load_credentials()
         self._authenticate()
         self._get_templates()

pype/plugins/maya/publish/validate_rendersettings.py

@@ -1,4 +1,7 @@
-import maya.cmds as cmds
+import os
+
+from maya import cmds, mel
+import pymel.core as pm
 
 import pyblish.api
 import pype.api
@@ -9,9 +12,9 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
     """Validates the global render settings
 
     * File Name Prefix must be as followed:
-        * vray: <Scene>/<Layer>/<Layer>
+        * vray: maya/<Layer>/<Layer>
-        * arnold: <Scene>/<RenderLayer>/<RenderLayer>
+        * arnold: maya/<RenderLayer>/<RenderLayer>
-        * default: <Scene>/<RenderLayer>/<RenderLayer>
+        * default: maya/<RenderLayer>/<RenderLayer>
 
     * Frame Padding must be:
         * default: 4
@@ -34,8 +37,8 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
     actions = [pype.api.RepairAction]
 
     DEFAULT_PADDING = 4
-    RENDERER_PREFIX = {"vray": "<Scene>/<Layer>/<Layer>"}
+    RENDERER_PREFIX = {"vray": "maya/<Layer>/<Layer>"}
-    DEFAULT_PREFIX = "<Scene>/<RenderLayer>/<RenderLayer>_<RenderPass>"
+    DEFAULT_PREFIX = "maya/<RenderLayer>/<RenderLayer>_<RenderPass>"
 
     def process(self, instance):
 
@@ -66,8 +69,8 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
             cls.log.error("Animation needs to be enabled. Use the same "
                           "frame for start and end to render single frame")
 
-        fname_prefix = cls.RENDERER_PREFIX.get(renderer,
+        fname_prefix = cls.get_prefix(renderer)
-                                               cls.DEFAULT_PREFIX)
+
         if prefix != fname_prefix:
             invalid = True
             cls.log.error("Wrong file name prefix: %s (expected: %s)"
@@ -80,6 +83,21 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
 
         return invalid
 
+    @classmethod
+    def get_prefix(cls, renderer):
+        prefix = cls.RENDERER_PREFIX.get(renderer, cls.DEFAULT_PREFIX)
+        # maya.cmds and pymel.core return only default project directory and
+        # not the current one but only default.
+        output_path = os.path.join(
+            mel.eval("workspace -q -rd;"), pm.workspace.fileRules["images"]
+        )
+        # Workfile paths can be configured to have host name in file path.
+        # In this case we want to avoid duplicate folder names.
+        if "maya" in output_path.lower():
+            prefix = prefix.replace("maya/", "")
+
+        return prefix
+
     @classmethod
     def repair(cls, instance):
 
@@ -94,7 +112,7 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin):
             node = render_attrs["node"]
             prefix_attr = render_attrs["prefix"]
 
-            fname_prefix = cls.RENDERER_PREFIX.get(renderer, cls.DEFAULT_PREFIX)
+            fname_prefix = cls.get_prefix(renderer)
             cmds.setAttr("{}.{}".format(node, prefix_attr),
                          fname_prefix, type="string")
 

pype/plugins/nuke/create/create_write.py

@@ -68,7 +68,7 @@ class CreateWriteRender(avalon.nuke.Creator):
             else:
                 self.log.info("Adding template path from plugin")
                 write_data.update({
-                    "fpath_template": "{work}/renders/v{version}/{subset}.{frame}.{ext}"})
+                    "fpath_template": "{work}/renders/nuke/{subset}/{subset}.{frame}.{ext}"})
 
             create_write_node(self.data["subset"], write_data)
 

pype/plugins/nuke/load/load_mov.py

@@ -182,7 +182,6 @@ class LoadMov(api.Loader):
         """
 
         from avalon.nuke import (
-            ls_img_sequence,
             update_container
         )
 
@@ -190,8 +189,7 @@ class LoadMov(api.Loader):
         # TODO: prepare also for other Read img/geo/camera
         assert node.Class() == "Read", "Must be Read"
 
-        root = api.get_representation_path(representation)
+        file = api.get_representation_path(representation)
-        file = ls_img_sequence(os.path.dirname(root), one=True)
 
         # Get start frame from version data
         version = io.find_one({
@@ -238,7 +236,7 @@ class LoadMov(api.Loader):
         # Update the loader's path whilst preserving some values
         with preserve_trim(node):
             node["file"].setValue(file["path"])
-            log.info("__ node['file']: {}".format(node["file"]))
+            log.info("__ node['file']: {}".format(node["file"].value()))
 
         # Set the global in to the start frame of the sequence
         loader_shift(node, first, relative=True)

pype/plugins/nuke/load/load_sequence.py

@@ -179,8 +179,8 @@ class LoadSequence(api.Loader):
         # TODO: prepare also for other Read img/geo/camera
         assert node.Class() == "Read", "Must be Read"
 
-        root = api.get_representation_path(representation)
+        path = api.get_representation_path(representation)
-        file = ls_img_sequence(os.path.dirname(root), one=True)
+        file = ls_img_sequence(path)
 
         # Get start frame from version data
         version = io.find_one({
@@ -222,7 +222,7 @@ class LoadSequence(api.Loader):
         # Update the loader's path whilst preserving some values
         with preserve_trim(node):
             node["file"].setValue(file["path"])
-            log.info("__ node['file']: {}".format(node["file"]))
+            log.info("__ node['file']: {}".format(node["file"].value()))
 
         # Set the global in to the start frame of the sequence
         loader_shift(node, first, relative=True)

pype/scripts/publish_filesequence.py

@@ -58,7 +58,9 @@ def __main__():
     ]
 
     print("Pype command: {}".format(" ".join(args)))
-    exit_code = subprocess.call(args, shell=True)
+    # Forcing forwaring the environment because environment inheritance does
+    # not always work.
+    exit_code = subprocess.call(args, env=os.environ)
     if exit_code != 0:
         raise ValueError("Publishing failed.")