Merge pull request #243 from pypeclub/feature/102-_draft_Celaction_quick_integration

Feature/102  draft celaction quick integration

pype/api.py

@@ -30,6 +30,7 @@ from .lib import (
     get_hierarchy,
     get_subsets,
     get_version_from_path,
+    get_last_version_from_path,
     modified_environ,
     add_tool_to_environment
 )
@@ -67,6 +68,7 @@ __all__ = [
     "get_asset",
     "get_subsets",
     "get_version_from_path",
+    "get_last_version_from_path",
     "modified_environ",
     "add_tool_to_environment",
 

pype/hooks/celaction/prelaunch.py

@@ -0,0 +1,208 @@
+import logging
+import os
+import winreg
+import shutil
+from pype.lib import PypeHook
+from pype.api import (
+    Anatomy,
+    Logger,
+    get_last_version_from_path
+)
+
+from avalon import io, api, lib
+
+log = logging.getLogger(__name__)
+
+
+class CelactionPrelaunchHook(PypeHook):
+    """
+    This hook will check if current workfile path has Unreal
+    project inside. IF not, it initialize it and finally it pass
+    path to the project by environment variable to Unreal launcher
+    shell script.
+    """
+    workfile_ext = "scn"
+
+    def __init__(self, logger=None):
+        if not logger:
+            self.log = Logger().get_logger(self.__class__.__name__)
+        else:
+            self.log = logger
+
+        self.signature = "( {} )".format(self.__class__.__name__)
+
+    def execute(self, *args, env: dict = None) -> bool:
+        if not env:
+            env = os.environ
+
+        # initialize
+        self._S = api.Session
+
+        # get publish version of celaction
+        app = "celaction_publish"
+
+        # get context variables
+        project = self._S["AVALON_PROJECT"] = env["AVALON_PROJECT"]
+        asset = self._S["AVALON_ASSET"] = env["AVALON_ASSET"]
+        task = self._S["AVALON_TASK"] = env["AVALON_TASK"]
+        workdir = self._S["AVALON_WORKDIR"] = env["AVALON_WORKDIR"]
+
+        # get workfile path
+        anatomy_filled = self.get_anatomy_filled()
+        workfile = anatomy_filled["work"]["file"]
+        version = anatomy_filled["version"]
+
+        # create workdir if doesn't exist
+        os.makedirs(workdir, exist_ok=True)
+        self.log.info(f"Work dir is: `{workdir}`")
+
+        # get last version of workfile
+        workfile_last = get_last_version_from_path(
+            workdir, workfile.split(version))
+
+        if workfile_last:
+            workfile = workfile_last
+
+        workfile_path = os.path.join(workdir, workfile)
+
+        # copy workfile from template if doesnt exist any on path
+        if not os.path.isfile(workfile_path):
+            # try to get path from environment or use default
+            # from `pype.celation` dir
+            template_path = env.get("CELACTION_TEMPLATE") or os.path.join(
+                env.get("PYPE_MODULE_ROOT"),
+                "pype/hosts/celaction/celaction_template_scene.scn"
+            )
+            self.log.info(
+                f"Creating workfile from template: `{template_path}`")
+            shutil.copy2(
+                os.path.normpath(template_path),
+                os.path.normpath(workfile_path)
+            )
+
+        self.log.info(f"Workfile to open: `{workfile_path}`")
+
+        # adding compulsory environment var for openting file
+        env["PYPE_CELACTION_PROJECT_FILE"] = workfile_path
+
+        # setting output parameters
+        path = r"Software\CelAction\CelAction2D\User Settings"
+        winreg.CreateKey(winreg.HKEY_CURRENT_USER, path)
+        hKey = winreg.OpenKey(
+            winreg.HKEY_CURRENT_USER,
+            "Software\\CelAction\\CelAction2D\\User Settings", 0,
+            winreg.KEY_ALL_ACCESS)
+
+        # TODO: change to root path and pyblish standalone to premiere way
+        pype_root_path = os.getenv("PYPE_SETUP_PATH")
+        path = os.path.join(pype_root_path,
+                            "pype.bat")
+
+        winreg.SetValueEx(hKey, "SubmitAppTitle", 0, winreg.REG_SZ, path)
+
+        parameters = [
+            "launch",
+            f"--app {app}",
+            f"--project {project}",
+            f"--asset {asset}",
+            f"--task {task}",
+            "--currentFile \"*SCENE*\"",
+            "--chunk *CHUNK*",
+            "--frameStart *START*",
+            "--frameEnd *END*",
+            "--resolutionWidth *X*",
+            "--resolutionHeight *Y*",
+            # "--programDir \"'*PROGPATH*'\""
+        ]
+        winreg.SetValueEx(hKey, "SubmitParametersTitle", 0, winreg.REG_SZ,
+                          " ".join(parameters))
+
+        # setting resolution parameters
+        path = r"Software\CelAction\CelAction2D\User Settings\Dialogs"
+        path += r"\SubmitOutput"
+        winreg.CreateKey(winreg.HKEY_CURRENT_USER, path)
+        hKey = winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0,
+                              winreg.KEY_ALL_ACCESS)
+        winreg.SetValueEx(hKey, "SaveScene", 0, winreg.REG_DWORD, 1)
+        winreg.SetValueEx(hKey, "CustomX", 0, winreg.REG_DWORD, 1920)
+        winreg.SetValueEx(hKey, "CustomY", 0, winreg.REG_DWORD, 1080)
+
+        # making sure message dialogs don't appear when overwriting
+        path = r"Software\CelAction\CelAction2D\User Settings\Messages"
+        path += r"\OverwriteScene"
+        winreg.CreateKey(winreg.HKEY_CURRENT_USER, path)
+        hKey = winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0,
+                              winreg.KEY_ALL_ACCESS)
+        winreg.SetValueEx(hKey, "Result", 0, winreg.REG_DWORD, 6)
+        winreg.SetValueEx(hKey, "Valid", 0, winreg.REG_DWORD, 1)
+
+        path = r"Software\CelAction\CelAction2D\User Settings\Messages"
+        path += r"\SceneSaved"
+        winreg.CreateKey(winreg.HKEY_CURRENT_USER, path)
+        hKey = winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0,
+                              winreg.KEY_ALL_ACCESS)
+        winreg.SetValueEx(hKey, "Result", 0, winreg.REG_DWORD, 1)
+        winreg.SetValueEx(hKey, "Valid", 0, winreg.REG_DWORD, 1)
+
+        return True
+
+    def get_anatomy_filled(self):
+        root_path = api.registered_root()
+        project_name = self._S["AVALON_PROJECT"]
+        asset_name = self._S["AVALON_ASSET"]
+
+        io.install()
+        project_entity = io.find_one({
+            "type": "project",
+            "name": project_name
+        })
+        assert project_entity, (
+            "Project '{0}' was not found."
+        ).format(project_name)
+        log.debug("Collected Project \"{}\"".format(project_entity))
+
+        asset_entity = io.find_one({
+            "type": "asset",
+            "name": asset_name,
+            "parent": project_entity["_id"]
+        })
+        assert asset_entity, (
+            "No asset found by the name '{0}' in project '{1}'"
+        ).format(asset_name, project_name)
+
+        project_name = project_entity["name"]
+
+        log.info(
+            "Anatomy object collected for project \"{}\".".format(project_name)
+        )
+
+        hierarchy_items = asset_entity["data"]["parents"]
+        hierarchy = ""
+        if hierarchy_items:
+            hierarchy = os.path.join(*hierarchy_items)
+
+        template_data = {
+            "root": root_path,
+            "project": {
+                "name": project_name,
+                "code": project_entity["data"].get("code")
+            },
+            "asset": asset_entity["name"],
+            "hierarchy": hierarchy.replace("\\", "/"),
+            "task": self._S["AVALON_TASK"],
+            "ext": self.workfile_ext,
+            "version": 1,
+            "username": os.getenv("PYPE_USERNAME", "").strip()
+        }
+
+        avalon_app_name = os.environ.get("AVALON_APP_NAME")
+        if avalon_app_name:
+            application_def = lib.get_application(avalon_app_name)
+            app_dir = application_def.get("application_dir")
+            if app_dir:
+                template_data["app"] = app_dir
+
+        anatomy = Anatomy(project_name)
+        anatomy_filled = anatomy.format_all(template_data).get_solved()
+
+        return anatomy_filled

pype/hosts/celaction/__init__.py

@@ -0,0 +1 @@
+kwargs = None

pype/hosts/celaction/cli.py

@@ -0,0 +1,121 @@
+import os
+import sys
+import copy
+import argparse
+
+from avalon import io
+from avalon.tools import publish
+
+import pyblish.api
+import pyblish.util
+
+from pype.api import Logger
+import pype
+import pype.celaction
+
+log = Logger().get_logger("Celaction_cli_publisher")
+
+publish_host = "celaction"
+
+PUBLISH_PATH = os.path.join(pype.PLUGINS_DIR, publish_host, "publish")
+
+PUBLISH_PATHS = [
+    PUBLISH_PATH,
+    os.path.join(pype.PLUGINS_DIR, "ftrack", "publish")
+]
+
+
+def cli():
+    parser = argparse.ArgumentParser(prog="celaction_publish")
+
+    parser.add_argument("--currentFile",
+                        help="Pass file to Context as `currentFile`")
+
+    parser.add_argument("--chunk",
+                        help=("Render chanks on farm"))
+
+    parser.add_argument("--frameStart",
+                        help=("Start of frame range"))
+
+    parser.add_argument("--frameEnd",
+                        help=("End of frame range"))
+
+    parser.add_argument("--resolutionWidth",
+                        help=("Width of resolution"))
+
+    parser.add_argument("--resolutionHeight",
+                        help=("Height of resolution"))
+
+    # parser.add_argument("--programDir",
+    # help=("Directory with celaction program installation"))
+
+    pype.celaction.kwargs = parser.parse_args(sys.argv[1:]).__dict__
+
+
+def _prepare_publish_environments():
+    """Prepares environments based on request data."""
+    env = copy.deepcopy(os.environ)
+
+    project_name = os.getenv("AVALON_PROJECT")
+    asset_name = os.getenv("AVALON_ASSET")
+
+    io.install()
+    project_doc = io.find_one({
+        "type": "project"
+    })
+    av_asset = io.find_one({
+        "type": "asset",
+        "name": asset_name
+    })
+    parents = av_asset["data"]["parents"]
+    hierarchy = ""
+    if parents:
+        hierarchy = "/".join(parents)
+
+    env["AVALON_PROJECT"] = project_name
+    env["AVALON_ASSET"] = asset_name
+    env["AVALON_TASK"] = os.getenv("AVALON_TASK")
+    env["AVALON_WORKDIR"] = os.getenv("AVALON_WORKDIR")
+    env["AVALON_HIERARCHY"] = hierarchy
+    env["AVALON_PROJECTCODE"] = project_doc["data"].get("code", "")
+    env["AVALON_APP"] = publish_host
+    env["AVALON_APP_NAME"] = "celaction_local"
+
+    env["PYBLISH_HOSTS"] = publish_host
+
+    os.environ.update(env)
+
+
+def main():
+    # prepare all environments
+    _prepare_publish_environments()
+
+    # Registers pype's Global pyblish plugins
+    pype.install()
+
+    for path in PUBLISH_PATHS:
+        path = os.path.normpath(path)
+
+        if not os.path.exists(path):
+            continue
+
+        log.info(f"Registering path: {path}")
+        pyblish.api.register_plugin_path(path)
+
+    pyblish.api.register_host(publish_host)
+
+    # Register project specific plugins
+    project_name = os.environ["AVALON_PROJECT"]
+    project_plugins_paths = os.getenv("PYPE_PROJECT_PLUGINS", "")
+    for path in project_plugins_paths.split(os.pathsep):
+        plugin_path = os.path.join(path, project_name, "plugins")
+        if os.path.exists(plugin_path):
+            pyblish.api.register_plugin_path(plugin_path)
+
+    return publish.show()
+
+
+if __name__ == "__main__":
+    cli()
+    result = main()
+    sys.exit(not bool(result))

pype/lib.py

@@ -469,6 +469,43 @@ def get_version_from_path(file):
         )
 
 
+def get_last_version_from_path(path_dir, filter):
+    """
+    Finds last version of given directory content
+
+    Args:
+        path_dir (string): directory path
+        filter (list): list of strings used as file name filter
+
+    Returns:
+        string: file name with last version
+
+    Example:
+        last_version_file = get_last_version_from_path(
+            "/project/shots/shot01/work", ["shot01", "compositing", "nk"])
+    """
+
+    assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory"
+    assert isinstance(filter, list) and (
+        len(filter) != 0), "`filter` argument needs to be list and not empty"
+
+    filtred_files = list()
+
+    # form regex for filtering
+    patern = r".*".join(filter)
+
+    for f in os.listdir(path_dir):
+        if not re.findall(patern, f):
+            continue
+        filtred_files.append(f)
+
+    if filtred_files:
+        sorted(filtred_files)
+        return filtred_files[-1]
+    else:
+        return None
+
+
 def get_avalon_database():
     if io._database is None:
         set_io_database()
@@ -610,7 +647,7 @@ def get_subsets(asset_name,
 
         if len(repres_out) > 0:
             output_dict[subset["name"]] = {"version": version_sel,
-                                           "representaions": repres_out}
+                                           "representations": repres_out}
 
     return output_dict
 

pype/plugins/celaction/publish/collect_audio.py

@@ -0,0 +1,41 @@
+import pyblish.api
+import os
+
+import pype.api as pype
+from pprint import pformat
+
+
+class AppendCelactionAudio(pyblish.api.ContextPlugin):
+
+    label = "Colect Audio for publishing"
+    order = pyblish.api.CollectorOrder + 0.1
+
+    def process(self, context):
+        self.log.info('Collecting Audio Data')
+        asset_entity = context.data["assetEntity"]
+
+        # get all available representations
+        subsets = pype.get_subsets(asset_entity["name"],
+                                   representations=["audio"]
+                                   )
+        self.log.info(f"subsets is: {pformat(subsets)}")
+
+        if not subsets.get("audioMain"):
+            raise AttributeError("`audioMain` subset does not exist")
+
+        reprs = subsets.get("audioMain", {}).get("representations", [])
+        self.log.info(f"reprs is: {pformat(reprs)}")
+
+        repr = next((r for r in reprs), None)
+        if not repr:
+            raise "Missing `audioMain` representation"
+        self.log.info(f"represetation is: {repr}")
+
+        audio_file = repr.get('data', {}).get('path', "")
+
+        if os.path.exists(audio_file):
+            context.data["audioFile"] = audio_file
+            self.log.info(
+                'audio_file: {}, has been added to context'.format(audio_file))
+        else:
+            self.log.warning("Couldn't find any audio file on Ftrack.")

pype/plugins/celaction/publish/collect_celaction_cli_kwargs.py

@@ -0,0 +1,23 @@
+import pyblish.api
+import pype.celaction
+
+
+class CollectCelactionCliKwargs(pyblish.api.Collector):
+    """ Collects all keyword arguments passed from the terminal """
+
+    label = "Collect Celaction Cli Kwargs"
+    order = pyblish.api.Collector.order - 0.1
+
+    def process(self, context):
+        kwargs = pype.celaction.kwargs.copy()
+
+        self.log.info("Storing kwargs: %s" % kwargs)
+        context.set_data("kwargs", kwargs)
+
+        # get kwargs onto context data as keys with values
+        for k, v in kwargs.items():
+            self.log.info(f"Setting `{k}` to instance.data with value: `{v}`")
+            if k in ["frameStart", "frameEnd"]:
+                context.data[k] = kwargs[k] = int(v)
+            else:
+                context.data[k] = v

pype/plugins/celaction/publish/collect_celaction_instances.py

@@ -0,0 +1,91 @@
+import os
+from avalon import api
+import pyblish.api
+
+
+class CollectCelactionInstances(pyblish.api.ContextPlugin):
+    """ Adds the celaction render instances """
+
+    label = "Collect Celaction Instances"
+    order = pyblish.api.CollectorOrder + 0.1
+
+    def process(self, context):
+        task = api.Session["AVALON_TASK"]
+        current_file = context.data["currentFile"]
+        staging_dir = os.path.dirname(current_file)
+        scene_file = os.path.basename(current_file)
+        version = context.data["version"]
+        asset_entity = context.data["assetEntity"]
+
+        shared_instance_data = {
+            "asset": asset_entity["name"],
+            "frameStart": asset_entity["data"]["frameStart"],
+            "frameEnd": asset_entity["data"]["frameEnd"],
+            "handleStart": asset_entity["data"]["handleStart"],
+            "handleEnd": asset_entity["data"]["handleEnd"],
+            "fps": asset_entity["data"]["fps"],
+            "resolutionWidth": asset_entity["data"]["resolutionWidth"],
+            "resolutionHeight": asset_entity["data"]["resolutionHeight"],
+            "pixelAspect": 1,
+            "step": 1,
+            "version": version
+        }
+
+        celaction_kwargs = context.data.get("kwargs", {})
+
+        if celaction_kwargs:
+            shared_instance_data.update(celaction_kwargs)
+
+        # workfile instance
+        family = "workfile"
+        subset = family + task.capitalize()
+        # Create instance
+        instance = context.create_instance(subset)
+
+        # creating instance data
+        instance.data.update({
+            "subset": subset,
+            "label": scene_file,
+            "family": family,
+            "families": [family],
+            "representations": list()
+        })
+
+        # adding basic script data
+        instance.data.update(shared_instance_data)
+
+        # creating representation
+        representation = {
+            'name': 'scn',
+            'ext': 'scn',
+            'files': scene_file,
+            "stagingDir": staging_dir,
+        }
+
+        instance.data["representations"].append(representation)
+
+        self.log.info('Publishing Celaction workfile')
+
+        # render instance
+        family = "render.farm"
+        subset = f"render{task}Main"
+        instance = context.create_instance(name=subset)
+        # getting instance state
+        instance.data["publish"] = True
+
+        # add assetEntity data into instance
+        instance.data.update({
+            "label": "{} - farm".format(subset),
+            "family": family,
+            "families": [family],
+            "subset": subset
+        })
+
+        # adding basic script data
+        instance.data.update(shared_instance_data)
+
+        self.log.info('Publishing Celaction render instance')
+        self.log.debug(f"Instance data: `{instance.data}`")
+
+        for i in context:
+            self.log.debug(f"{i.data['families']}")

pype/plugins/celaction/publish/collect_render_path.py

@@ -0,0 +1,29 @@
+import os
+import pyblish.api
+
+
+class CollectRenderPath(pyblish.api.InstancePlugin):
+    """Generate file and directory path where rendered images will be"""
+
+    label = "Collect Render Path"
+    order = pyblish.api.CollectorOrder + 0.495
+
+    def process(self, instance):
+        anatomy = instance.context.data["anatomy"]
+        current_file = instance.context.data["currentFile"]
+        work_dir = os.path.dirname(current_file)
+        padding = anatomy.templates.get("frame_padding", 4)
+        render_dir = os.path.join(
+            work_dir, "render", "celaction"
+        )
+        render_path = os.path.join(
+            render_dir,
+            ".".join([instance.data["subset"], f"%0{padding}d", "png"])
+        )
+
+        # create dir if it doesnt exists
+        os.makedirs(render_dir, exist_ok=True)
+
+        instance.data["path"] = render_path
+
+        self.log.info(f"Render output path set to: `{render_path}`")

pype/plugins/celaction/publish/integrate_version_up.py

@@ -0,0 +1,68 @@
+import shutil
+import re
+import pyblish.api
+
+
+class VersionUpScene(pyblish.api.ContextPlugin):
+    order = pyblish.api.IntegratorOrder
+    label = 'Version Up Scene'
+    families = ['scene']
+    optional = True
+    active = True
+
+    def process(self, context):
+        current_file = context.data.get('currentFile')
+        v_up = get_version_up(current_file)
+        self.log.debug('Current file is: {}'.format(current_file))
+        self.log.debug('Version up: {}'.format(v_up))
+
+        shutil.copy2(current_file, v_up)
+        self.log.info('Scene saved into new version: {}'.format(v_up))
+
+
+def version_get(string, prefix, suffix=None):
+    """Extract version information from filenames used by DD (and Weta, apparently)
+    These are _v# or /v# or .v# where v is a prefix string, in our case
+    we use "v" for render version and "c" for camera track version.
+    See the version.py and camera.py plugins for usage."""
+
+    if string is None:
+        raise ValueError("Empty version string - no match")
+
+    regex = r"[/_.]{}\d+".format(prefix)
+    matches = re.findall(regex, string, re.IGNORECASE)
+    if not len(matches):
+        msg = f"No `_{prefix}#` found in `{string}`"
+        raise ValueError(msg)
+    return (matches[-1:][0][1], re.search(r"\d+", matches[-1:][0]).group())
+
+
+def version_set(string, prefix, oldintval, newintval):
+    """Changes version information from filenames used by DD (and Weta, apparently)
+    These are _v# or /v# or .v# where v is a prefix string, in our case
+    we use "v" for render version and "c" for camera track version.
+    See the version.py and camera.py plugins for usage."""
+
+    regex = r"[/_.]{}\d+".format(prefix)
+    matches = re.findall(regex, string, re.IGNORECASE)
+    if not len(matches):
+        return ""
+
+    # Filter to retain only version strings with matching numbers
+    matches = filter(lambda s: int(s[2:]) == oldintval, matches)
+
+    # Replace all version strings with matching numbers
+    for match in matches:
+        # use expression instead of expr so 0 prefix does not make octal
+        fmt = "%%(#)0%dd" % (len(match) - 2)
+        newfullvalue = match[0] + prefix + str(fmt % {"#": newintval})
+        string = re.sub(match, newfullvalue, string)
+    return string
+
+
+def get_version_up(path):
+    """ Returns the next version of the path """
+
+    (prefix, v) = version_get(path, 'v')
+    v = int(v)
+    return version_set(path, prefix, v, v + 1)

pype/plugins/celaction/publish/submit_celaction_deadline.py

@@ -0,0 +1,234 @@
+import os
+import json
+import getpass
+
+from avalon.vendor import requests
+import re
+import pyblish.api
+
+
+class ExtractCelactionDeadline(pyblish.api.InstancePlugin):
+    """Submit CelAction2D scene to Deadline
+
+    Renders are submitted to a Deadline Web Service as
+    supplied via the environment variable DEADLINE_REST_URL
+
+    """
+
+    label = "Submit CelAction to Deadline"
+    order = pyblish.api.IntegratorOrder + 0.1
+    hosts = ["celaction"]
+    families = ["render.farm"]
+
+    deadline_department = ""
+    deadline_priority = 50
+    deadline_pool = ""
+    deadline_pool_secondary = ""
+    deadline_group = ""
+    deadline_chunk_size = 1
+
+    def process(self, instance):
+        context = instance.context
+
+        DEADLINE_REST_URL = os.environ.get("DEADLINE_REST_URL")
+        assert DEADLINE_REST_URL, "Requires DEADLINE_REST_URL"
+
+        self.deadline_url = "{}/api/jobs".format(DEADLINE_REST_URL)
+        self._comment = context.data.get("comment", "")
+        self._deadline_user = context.data.get(
+            "deadlineUser", getpass.getuser())
+        self._frame_start = int(instance.data["frameStart"])
+        self._frame_end = int(instance.data["frameEnd"])
+
+        # get output path
+        render_path = instance.data['path']
+        script_path = context.data["currentFile"]
+
+        response = self.payload_submit(instance,
+                                       script_path,
+                                       render_path
+                                       )
+        # Store output dir for unified publisher (filesequence)
+        instance.data["deadlineSubmissionJob"] = response.json()
+
+        instance.data["outputDir"] = os.path.dirname(
+            render_path).replace("\\", "/")
+
+        instance.data["publishJobState"] = "Suspended"
+        instance.context.data['ftrackStatus'] = "Render"
+
+        # adding 2d render specific family for version identification in Loader
+        instance.data["families"] = ["render2d"]
+
+    def payload_submit(self,
+                       instance,
+                       script_path,
+                       render_path
+                       ):
+        resolution_width = instance.data["resolutionWidth"]
+        resolution_height = instance.data["resolutionHeight"]
+        render_dir = os.path.normpath(os.path.dirname(render_path))
+        script_name = os.path.basename(script_path)
+        jobname = "%s - %s" % (script_name, instance.name)
+
+        output_filename_0 = self.preview_fname(render_path)
+
+        try:
+            # Ensure render folder exists
+            os.makedirs(render_dir)
+        except OSError:
+            pass
+
+        # define chunk and priority
+        chunk_size = instance.context.data.get("chunk")
+        if chunk_size == 0:
+            chunk_size = self.deadline_chunk_size
+
+        # search for %02d pattern in name, and padding number
+        search_results = re.search(r"(.%0)(\d)(d)[._]", render_path).groups()
+        split_patern = "".join(search_results)
+        padding_number = int(search_results[1])
+
+        args = [
+            f"<QUOTE>{script_path}<QUOTE>",
+            "-a",
+            "-s <STARTFRAME>",
+            "-e <ENDFRAME>",
+            f"-d <QUOTE>{render_dir}<QUOTE>",
+            f"-x {resolution_width}",
+            f"-y {resolution_height}",
+            f"-r <QUOTE>{render_path.replace(split_patern, '')}<QUOTE>",
+            f"-= AbsoluteFrameNumber=on -= PadDigits={padding_number}",
+            "-= ClearAttachment=on",
+        ]
+
+        payload = {
+            "JobInfo": {
+                # Job name, as seen in Monitor
+                "Name": jobname,
+
+                # plugin definition
+                "Plugin": "CelAction",
+
+                # Top-level group name
+                "BatchName": script_name,
+
+                # Arbitrary username, for visualisation in Monitor
+                "UserName": self._deadline_user,
+
+                "Department": self.deadline_department,
+                "Priority": self.deadline_priority,
+
+                "Group": self.deadline_group,
+                "Pool": self.deadline_pool,
+                "SecondaryPool": self.deadline_pool_secondary,
+                "ChunkSize": chunk_size,
+
+                "Frames": f"{self._frame_start}-{self._frame_end}",
+                "Comment": self._comment,
+
+                # Optional, enable double-click to preview rendered
+                # frames from Deadline Monitor
+                "OutputFilename0": output_filename_0.replace("\\", "/")
+
+            },
+            "PluginInfo": {
+                # Input
+                "SceneFile": script_path,
+
+                # Output directory
+                "OutputFilePath": render_dir.replace("\\", "/"),
+
+                # Plugin attributes
+                "StartupDirectory": "",
+                "Arguments": " ".join(args),
+
+                # Resolve relative references
+                "ProjectPath": script_path,
+                "AWSAssetFile0": render_path,
+            },
+
+            # Mandatory for Deadline, may be empty
+            "AuxFiles": []
+        }
+
+        plugin = payload["JobInfo"]["Plugin"]
+        self.log.info("using render plugin : {}".format(plugin))
+
+        self.log.info("Submitting..")
+        self.log.info(json.dumps(payload, indent=4, sort_keys=True))
+
+        # adding expectied files to instance.data
+        self.expected_files(instance, render_path)
+        self.log.debug("__ expectedFiles: `{}`".format(
+            instance.data["expectedFiles"]))
+        response = requests.post(self.deadline_url, json=payload)
+
+        if not response.ok:
+            raise Exception(response.text)
+
+        return response
+
+    def preflight_check(self, instance):
+        """Ensure the startFrame, endFrame and byFrameStep are integers"""
+
+        for key in ("frameStart", "frameEnd"):
+            value = instance.data[key]
+
+            if int(value) == value:
+                continue
+
+            self.log.warning(
+                "%f=%d was rounded off to nearest integer"
+                % (value, int(value))
+            )
+
+    def preview_fname(self, path):
+        """Return output file path with #### for padding.
+
+        Deadline requires the path to be formatted with # in place of numbers.
+        For example `/path/to/render.####.png`
+
+        Args:
+            path (str): path to rendered images
+
+        Returns:
+            str
+
+        """
+        self.log.debug("_ path: `{}`".format(path))
+        if "%" in path:
+            search_results = re.search(r"[._](%0)(\d)(d)[._]", path).groups()
+            split_patern = "".join(search_results)
+            split_path = path.split(split_patern)
+            hashes = "#" * int(search_results[1])
+            return "".join([split_path[0], hashes, split_path[-1]])
+        if "#" in path:
+            self.log.debug("_ path: `{}`".format(path))
+            return path
+        else:
+            return path
+
+    def expected_files(self,
+                       instance,
+                       path):
+        """ Create expected files in instance data
+        """
+        if not instance.data.get("expectedFiles"):
+            instance.data["expectedFiles"] = list()
+
+        dir = os.path.dirname(path)
+        file = os.path.basename(path)
+
+        if "#" in file:
+            pparts = file.split("#")
+            padding = "%0{}d".format(len(pparts) - 1)
+            file = pparts[0] + padding + pparts[-1]
+
+        if "%" not in file:
+            instance.data["expectedFiles"].append(path)
+            return
+
+        for i in range(self._frame_start, (self._frame_end + 1)):
+            instance.data["expectedFiles"].append(
+                os.path.join(dir, (file % i)).replace("\\", "/"))

pype/plugins/global/publish/collect_rendered_files.py

@@ -99,6 +99,17 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin):
 
             instance.data["representations"] = representations
 
+            # add audio if in metadata data
+            if data.get("audio"):
+                instance.data.update({
+                    "audio": [{
+                        "filename": data.get("audio"),
+                        "offset": 0
+                    }]
+                })
+                self.log.info(
+                    f"Adding audio to instance: {instance.data['audio']}")
+
     def process(self, context):
         self._context = context
 

pype/plugins/global/publish/extract_review.py

@@ -610,8 +610,8 @@ class ExtractReview(pyblish.api.InstancePlugin):
         # NOTE Skipped using instance's resolution
         full_input_path_single_file = temp_data["full_input_path_single_file"]
         input_data = pype.lib.ffprobe_streams(full_input_path_single_file)[0]
-        input_width = input_data["width"]
-        input_height = input_data["height"]
+        input_width = int(input_data["width"])
+        input_height = int(input_data["height"])
 
         self.log.debug("pixel_aspect: `{}`".format(pixel_aspect))
         self.log.debug("input_width: `{}`".format(input_width))
@@ -631,6 +631,9 @@ class ExtractReview(pyblish.api.InstancePlugin):
             output_width = input_width
             output_height = input_height
 
+        output_width = int(output_width)
+        output_height = int(output_height)
+
         self.log.debug(
             "Output resolution is {}x{}".format(output_width, output_height)
         )

pype/plugins/global/publish/integrate_new.py

@@ -44,7 +44,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         "frameStart"
         "frameEnd"
         'fps'
-        "data": additional metadata for each representation.
     """
 
     label = "Integrate Asset New"
@@ -380,8 +379,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                     dst = "{0}{1}{2}".format(
                         dst_head,
                         dst_padding,
-                        dst_tail
-                    ).replace("..", ".")
+                        dst_tail).replace("..", ".")
 
                     self.log.debug("destination: `{}`".format(dst))
                     src = os.path.join(stagingdir, src_file_name)
@@ -454,15 +452,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             if repre_id is None:
                 repre_id = io.ObjectId()
 
-            data = repre.get("data") or {}
-            data.update({'path': dst, 'template': template})
             representation = {
                 "_id": repre_id,
                 "schema": "pype:representation-2.0",
                 "type": "representation",
                 "parent": version_id,
                 "name": repre['name'],
-                "data": data,
+                "data": {'path': dst, 'template': template},
                 "dependencies": instance.data.get("dependencies", "").split(),
 
                 # Imprint shortcut to context
@@ -562,10 +558,17 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         while True:
             try:
                 copyfile(src, dst)
-            except OSError as e:
-                self.log.critical("Cannot copy {} to {}".format(src, dst))
-                self.log.critical(e)
-                six.reraise(*sys.exc_info())
+            except (OSError, AttributeError) as e:
+                self.log.warning(e)
+                # try it again with shutil
+                import shutil
+                try:
+                    shutil.copyfile(src, dst)
+                    self.log.debug("Copying files with shutil...")
+                except (OSError) as e:
+                    self.log.critical("Cannot copy {} to {}".format(src, dst))
+                    self.log.critical(e)
+                    six.reraise(*sys.exc_info())
             if str(getsize(src)) in str(getsize(dst)):
                 break
 

pype/plugins/global/publish/submit_publish_job.py

@@ -14,6 +14,8 @@ import pyblish.api
 
 def _get_script():
     """Get path to the image sequence script."""
+    from pathlib import Path
+
     try:
         from pype.scripts import publish_filesequence
     except Exception:
@@ -23,7 +25,9 @@ def _get_script():
     if module_path.endswith(".pyc"):
         module_path = module_path[: -len(".pyc")] + ".py"
 
-    return os.path.normpath(module_path)
+    path = Path(os.path.normpath(module_path)).resolve(strict=True)
+
+    return str(path)
 
 
 def get_latest_version(asset_name, subset_name, family):
@@ -145,7 +149,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
     order = pyblish.api.IntegratorOrder + 0.2
     icon = "tractor"
 
-    hosts = ["fusion", "maya", "nuke"]
+    hosts = ["fusion", "maya", "nuke", "celaction"]
 
     families = ["render.farm", "prerener",
                 "renderlayer", "imagesequence", "vrayscene"]
@@ -158,11 +162,16 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
         "FTRACK_SERVER",
         "PYPE_METADATA_FILE",
         "AVALON_PROJECT",
-        "PYPE_LOG_NO_COLORS"
+        "PYPE_LOG_NO_COLORS",
+        "PYPE_PYTHON_EXE"
     ]
 
-    # pool used to do the publishing job
+    # custom deadline atributes
+    deadline_department = ""
     deadline_pool = ""
+    deadline_pool_secondary = ""
+    deadline_group = ""
+    deadline_chunk_size = 1
 
     # regex for finding frame number in string
     R_FRAME_NUMBER = re.compile(r'.+\.(?P<frame>[0-9]+)\..+')
@@ -215,8 +224,15 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
                 "JobDependency0": job["_id"],
                 "UserName": job["Props"]["User"],
                 "Comment": instance.context.data.get("comment", ""),
+
+                "Department": self.deadline_department,
+                "ChunkSize": self.deadline_chunk_size,
                 "Priority": job["Props"]["Pri"],
+
+                "Group": self.deadline_group,
                 "Pool": self.deadline_pool,
+                "SecondaryPool": self.deadline_pool_secondary,
+
                 "OutputDirectory0": output_dir
             },
             "PluginInfo": {
@@ -470,6 +486,9 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
             if bake_render_path:
                 preview = False
 
+            if "celaction" in self.hosts:
+                preview = True
+
             staging = os.path.dirname(list(collection)[0])
             success, rootless_staging_dir = (
                 self.anatomy.find_root_template_from_path(staging)
@@ -819,6 +838,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin):
             "instances": instances
         }
 
+        # add audio to metadata file if available
+        audio_file = context.data.get("audioFile")
+        if os.path.isfile(audio_file):
+            publish_job.update({"audio": audio_file})
+
         # pass Ftrack credentials in case of Muster
         if submission_type == "muster":
             ftrack = {