From 13b4b18d162ed4307408e9d2f64869403c740724 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 May 2022 17:05:55 +0200 Subject: [PATCH 01/25] OP-2787 - WIP implementation --- openpype/api.py | 2 + openpype/hosts/maya/api/pipeline.py | 7 + .../maya/plugins/create/create_animation.py | 4 + .../maya/plugins/create/create_pointcache.py | 4 + .../maya/plugins/publish/collect_animation.py | 3 + .../plugins/publish/collect_pointcache.py | 14 ++ .../maya/plugins/publish/extract_animation.py | 8 + .../plugins/publish/extract_pointcache.py | 8 + openpype/lib/remote_publish.py | 2 +- .../submit_maya_remote_publish_deadline.py | 137 ++++++++++++++++++ openpype/plugin.py | 12 ++ openpype/plugins/publish/integrate_new.py | 3 + openpype/scripts/remote_publish.py | 11 ++ 13 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 openpype/hosts/maya/plugins/publish/collect_pointcache.py create mode 100644 openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py create mode 100644 openpype/scripts/remote_publish.py diff --git a/openpype/api.py b/openpype/api.py index 9ce745b653..e049a683c6 100644 --- a/openpype/api.py +++ b/openpype/api.py @@ -44,6 +44,7 @@ from . import resources from .plugin import ( Extractor, + Integrator, ValidatePipelineOrder, ValidateContentsOrder, @@ -86,6 +87,7 @@ __all__ = [ # plugin classes "Extractor", + "Integrator", # ordering "ValidatePipelineOrder", "ValidateContentsOrder", diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index b0e8fac635..b75af29523 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -71,8 +71,15 @@ def install(): if lib.IS_HEADLESS: log.info(("Running in headless mode, skipping Maya " "save/open/new callback installation..")) + + # Register default "local" target + print("Registering pyblish target: farm") + pyblish.api.register_target("farm") return + print("Registering pyblish target: local") + pyblish.api.register_target("local") + _set_project() _register_callbacks() diff --git a/openpype/hosts/maya/plugins/create/create_animation.py b/openpype/hosts/maya/plugins/create/create_animation.py index 11a668cfc8..5cd1f7090a 100644 --- a/openpype/hosts/maya/plugins/create/create_animation.py +++ b/openpype/hosts/maya/plugins/create/create_animation.py @@ -38,3 +38,7 @@ class CreateAnimation(plugin.Creator): # Default to exporting world-space self.data["worldSpace"] = True + + # Default to not send to farm. + self.data["farm"] = False + self.data["priority"] = 50 diff --git a/openpype/hosts/maya/plugins/create/create_pointcache.py b/openpype/hosts/maya/plugins/create/create_pointcache.py index ede152f1ef..e876015adb 100644 --- a/openpype/hosts/maya/plugins/create/create_pointcache.py +++ b/openpype/hosts/maya/plugins/create/create_pointcache.py @@ -28,3 +28,7 @@ class CreatePointCache(plugin.Creator): # Add options for custom attributes self.data["attr"] = "" self.data["attrPrefix"] = "" + + # Default to not send to farm. + self.data["farm"] = False + self.data["priority"] = 50 diff --git a/openpype/hosts/maya/plugins/publish/collect_animation.py b/openpype/hosts/maya/plugins/publish/collect_animation.py index 9b1e38fd0a..b442113fbc 100644 --- a/openpype/hosts/maya/plugins/publish/collect_animation.py +++ b/openpype/hosts/maya/plugins/publish/collect_animation.py @@ -55,3 +55,6 @@ class CollectAnimationOutputGeometry(pyblish.api.InstancePlugin): # Store data in the instance for the validator instance.data["out_hierarchy"] = hierarchy + + if instance.data.get("farm"): + instance.data["families"].append("deadline") diff --git a/openpype/hosts/maya/plugins/publish/collect_pointcache.py b/openpype/hosts/maya/plugins/publish/collect_pointcache.py new file mode 100644 index 0000000000..b55babe372 --- /dev/null +++ b/openpype/hosts/maya/plugins/publish/collect_pointcache.py @@ -0,0 +1,14 @@ +import pyblish.api + + +class CollectPointcache(pyblish.api.InstancePlugin): + """Collect pointcache data for instance.""" + + order = pyblish.api.CollectorOrder + 0.4 + families = ["pointcache"] + label = "Collect Pointcache" + hosts = ["maya"] + + def process(self, instance): + if instance.data.get("farm"): + instance.data["families"].append("deadline") \ No newline at end of file diff --git a/openpype/hosts/maya/plugins/publish/extract_animation.py b/openpype/hosts/maya/plugins/publish/extract_animation.py index 8a8bd67cd8..87f2d35192 100644 --- a/openpype/hosts/maya/plugins/publish/extract_animation.py +++ b/openpype/hosts/maya/plugins/publish/extract_animation.py @@ -23,6 +23,14 @@ class ExtractAnimation(openpype.api.Extractor): families = ["animation"] def process(self, instance): + if instance.data.get("farm"): + path = os.path.join( + os.path.dirname(instance.context.data["currentFile"]), + "cache", + instance.data["name"] + ".abc" + ) + instance.data["expectedFiles"] = [os.path.normpath(path)] + return # Collect the out set nodes out_sets = [node for node in instance if node.endswith("out_SET")] diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index 60502fdde1..7ad4c6dfa9 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -25,6 +25,14 @@ class ExtractAlembic(openpype.api.Extractor): "vrayproxy"] def process(self, instance): + if instance.data.get("farm"): + path = os.path.join( + os.path.dirname(instance.context.data["currentFile"]), + "cache", + instance.data["name"] + ".abc" + ) + instance.data["expectedFiles"] = [os.path.normpath(path)] + return nodes = instance[:] diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index 8a42daf4e9..da2497e1a5 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -228,7 +228,7 @@ def _get_close_plugin(close_plugin_name, log): if plugin.__name__ == close_plugin_name: return plugin - log.warning("Close plugin not found, app might not close.") + log.debug("Close plugin not found, app might not close.") def get_task_data(batch_dir): diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py new file mode 100644 index 0000000000..761bc8cc95 --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -0,0 +1,137 @@ +import os +import requests + +from maya import cmds + +from openpype.pipeline import legacy_io + +import pyblish.api + + +class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): + """Submit Maya scene to perform a local publish in Deadline. + + Publishing in Deadline can be helpful for scenes that publish very slow. + This way it can process in the background on another machine without the + Artist having to wait for the publish to finish on their local machine. + + Submission is done through the Deadline Web Service. + + Different from `ProcessSubmittedJobOnFarm` which creates publish job + depending on metadata json containing context and instance data of + rendered files. + """ + + label = "Submit Scene to Deadline" + order = pyblish.api.IntegratorOrder + hosts = ["maya"] + families = ["deadline"] + + # custom deadline attributes + deadline_department = "" + deadline_pool = "" + deadline_pool_secondary = "" + deadline_group = "" + deadline_chunk_size = 1 + deadline_priority = 50 + + def process(self, context): + + # Ensure no errors so far + assert all(result["success"] for result in context.data["results"]), ( + "Errors found, aborting integration..") + + # Note that `publish` data member might change in the future. + # See: https://github.com/pyblish/pyblish-base/issues/307 + actives = [i for i in context if i.data["publish"]] + instance_names = sorted(instance.name for instance in actives) + + if not instance_names: + self.log.warning("No active instances found. " + "Skipping submission..") + return + + scene = context.data["currentFile"] + scenename = os.path.basename(scene) + + # Get project code + project_name = legacy_io.Session["AVALON_PROJECT"] + + job_name = "{scene} [PUBLISH]".format(scene=scenename) + batch_name = "{code} - {scene}".format(code=project_name, + scene=scenename) + + # Generate the payload for Deadline submission + payload = { + "JobInfo": { + "Plugin": "MayaBatch", + "BatchName": batch_name, + "Priority": 50, + "Name": job_name, + "UserName": context.data["user"], + # "Comment": instance.context.data.get("comment", ""), + # "InitialStatus": state + "Department": self.deadline_department, + "ChunkSize": self.deadline_chunk_size, + "Priority": self.deadline_priority, + + "Group": self.deadline_group, + + }, + "PluginInfo": { + + "Build": None, # Don't force build + "StrictErrorChecking": True, + "ScriptJob": True, + + # Inputs + "SceneFile": scene, + "ScriptFilename": "{OPENPYPE_ROOT}/scripts/remote_publish.py", + + # Mandatory for Deadline + "Version": cmds.about(version=True), + + # Resolve relative references + "ProjectPath": cmds.workspace(query=True, + rootDirectory=True), + + }, + + # Mandatory for Deadline, may be empty + "AuxFiles": [] + } + + # Include critical environment variables with submission + api.Session + keys = [ + "FTRACK_API_USER", + "FTRACK_API_KEY", + "FTRACK_SERVER" + ] + environment = dict({key: os.environ[key] for key in keys + if key in os.environ}, **legacy_io.Session) + + # TODO replace legacy_io with context.data ? + environment["AVALON_PROJECT"] = legacy_io.Session["AVALON_PROJECT"] + environment["AVALON_ASSET"] = legacy_io.Session["AVALON_ASSET"] + environment["AVALON_TASK"] = legacy_io.Session["AVALON_TASK"] + environment["AVALON_APP_NAME"] = os.environ.get("AVALON_APP_NAME") + environment["OPENPYPE_LOG_NO_COLORS"] = "1" + environment["OPENPYPE_USERNAME"] = context.data["user"] + environment["OPENPYPE_PUBLISH_JOB"] = "1" + environment["OPENPYPE_RENDER_JOB"] = "0" + environment["PYBLISH_ACTIVE_INSTANCES"] = ",".join(instance_names) + + payload["JobInfo"].update({ + "EnvironmentKeyValue%d" % index: "{key}={value}".format( + key=key, + value=environment[key] + ) for index, key in enumerate(environment) + }) + + self.log.info("Submitting Deadline job ...") + deadline_url = context.data["defaultDeadline"] + assert deadline_url, "Requires Deadline Webservice URL" + url = "{}/api/jobs".format(deadline_url) + response = requests.post(url, json=payload, timeout=10) + if not response.ok: + raise Exception(response.text) diff --git a/openpype/plugin.py b/openpype/plugin.py index bb9bc2ff85..f1ee626ffb 100644 --- a/openpype/plugin.py +++ b/openpype/plugin.py @@ -18,6 +18,16 @@ class InstancePlugin(pyblish.api.InstancePlugin): super(InstancePlugin, cls).process(cls, *args, **kwargs) +class Integrator(InstancePlugin): + """Integrator base class. + + Wraps pyblish instance plugin. Targets set to "local" which means all + integrators should run on "local" publishes, by default. + "farm" targets could be used for integrators that should run on a farm. + """ + targets = ["local"] + + class Extractor(InstancePlugin): """Extractor base class. @@ -28,6 +38,8 @@ class Extractor(InstancePlugin): """ + targets = ["local"] + order = 2.0 def staging_dir(self, instance): diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index bf13a4050e..1a4112107a 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -139,6 +139,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): ef, instance.data["family"], instance.data["families"])) return + if "deadline" in instance.data["families"]: + return + self.integrated_file_sizes = {} try: self.register(instance) diff --git a/openpype/scripts/remote_publish.py b/openpype/scripts/remote_publish.py new file mode 100644 index 0000000000..b54c8d931b --- /dev/null +++ b/openpype/scripts/remote_publish.py @@ -0,0 +1,11 @@ +try: + from openpype.api import Logger + import openpype.lib.remote_publish +except ImportError as exc: + # Ensure Deadline fails by output an error that contains "Fatal Error:" + raise ImportError("Fatal Error: %s" % exc) + +if __name__ == "__main__": + # Perform remote publish with thorough error checking + log = Logger.get_logger(__name__) + openpype.lib.remote_publish.publish(log) From 7b65184389640165d7268996dc069600722fe60f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 17:45:15 +0200 Subject: [PATCH 02/25] OP-2787 - replaced target farm with remote Target farm is being used for rendering, this should better differentiate it. --- openpype/hosts/maya/api/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index b75af29523..c2fe8a95a5 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -73,8 +73,8 @@ def install(): "save/open/new callback installation..")) # Register default "local" target - print("Registering pyblish target: farm") - pyblish.api.register_target("farm") + print("Registering pyblish target: remote") + pyblish.api.register_target("remote") return print("Registering pyblish target: local") From c3e13a9e198a7082439588d6b3f6550bbdf98675 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 20 May 2022 18:07:26 +0200 Subject: [PATCH 03/25] modules have ability to modify environments before launch --- openpype/lib/applications.py | 33 ++++++++++++++++++++++++++++----- openpype/modules/base.py | 19 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 6ade33b59c..a81bdeca0f 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1282,7 +1282,13 @@ class EnvironmentPrepData(dict): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name, env_group=None, env=None + project_name, + asset_name, + task_name, + app_name, + env_group=None, + env=None, + modules_manager=None ): """Prepare environment variables by context. Args: @@ -1293,10 +1299,12 @@ def get_app_environments_for_context( by ApplicationManager. env (dict): Initial environment variables. `os.environ` is used when not passed. + modules_manager (ModulesManager): Initialized modules manager. Returns: dict: Environments for passed context and application. """ + from openpype.pipeline import AvalonMongoDB # Avalon database connection @@ -1311,6 +1319,11 @@ def get_app_environments_for_context( "name": asset_name }) + if modules_manager is None: + from openpype.modules import ModulesManager + + modules_manager = ModulesManager() + # Prepare app object which can be obtained only from ApplciationManager app_manager = ApplicationManager() app = app_manager.applications[app_name] @@ -1334,7 +1347,7 @@ def get_app_environments_for_context( "env": env }) - prepare_app_environments(data, env_group) + prepare_app_environments(data, env_group, modules_manager) prepare_context_environments(data, env_group) # Discard avalon connection @@ -1355,9 +1368,12 @@ def _merge_env(env, current_env): return result -def _add_python_version_paths(app, env, logger): +def _add_python_version_paths(app, env, logger, modules_manager): """Add vendor packages specific for a Python version.""" + for module in modules_manager.get_enabled_modules(): + module.modify_application_launch_arguments(app, env) + # Skip adding if host name is not set if not app.host_name: return @@ -1390,7 +1406,9 @@ def _add_python_version_paths(app, env, logger): env["PYTHONPATH"] = os.pathsep.join(python_paths) -def prepare_app_environments(data, env_group=None, implementation_envs=True): +def prepare_app_environments( + data, env_group=None, implementation_envs=True, modules_manager=None +): """Modify launch environments based on launched app and context. Args: @@ -1403,7 +1421,12 @@ def prepare_app_environments(data, env_group=None, implementation_envs=True): log = data["log"] source_env = data["env"].copy() - _add_python_version_paths(app, source_env, log) + if modules_manager is None: + from openpype.modules import ModulesManager + + modules_manager = ModulesManager() + + _add_python_version_paths(app, source_env, log, modules_manager) # Use environments from local settings filtered_local_envs = {} diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 5b49649359..d591df6768 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -370,6 +370,7 @@ def _load_modules(): class _OpenPypeInterfaceMeta(ABCMeta): """OpenPypeInterface meta class to print proper string.""" + def __str__(self): return "<'OpenPypeInterface.{}'>".format(self.__name__) @@ -388,6 +389,7 @@ class OpenPypeInterface: OpenPype modules which means they have to have implemented methods defined in the interface. By default interface does not have any abstract parts. """ + pass @@ -432,10 +434,12 @@ class OpenPypeModule: It is not recommended to override __init__ that's why specific method was implemented. """ + pass def connect_with_modules(self, enabled_modules): """Connect with other enabled modules.""" + pass def get_global_environments(self): @@ -443,8 +447,22 @@ class OpenPypeModule: Environment variables that can be get only from system settings. """ + return {} + def modify_application_launch_arguments(self, app, env): + """Give option to modify launch environments before application launch. + + Implementation is optional. To change environments modify passed + dictionary of environments. + + Args: + app (Application): Application that is launcher. + env (dict): Current environemnt variables. + """ + + pass + def cli(self, module_click_group): """Add commands to click group. @@ -465,6 +483,7 @@ class OpenPypeModule: def mycommand(): print("my_command") """ + pass From 6fc240734c799a37c349f7a6e8945f7feea50ab5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 20 May 2022 18:10:40 +0200 Subject: [PATCH 04/25] ftrack module modify application launch environments in module instead of in prelaunch hook --- openpype/modules/base.py | 4 +- openpype/modules/ftrack/ftrack_module.py | 34 +++++++++++++++ .../ftrack/launch_hooks/pre_python2_vendor.py | 43 ------------------- 3 files changed, 36 insertions(+), 45 deletions(-) delete mode 100644 openpype/modules/ftrack/launch_hooks/pre_python2_vendor.py diff --git a/openpype/modules/base.py b/openpype/modules/base.py index d591df6768..96c1b84a54 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -450,14 +450,14 @@ class OpenPypeModule: return {} - def modify_application_launch_arguments(self, app, env): + def modify_application_launch_arguments(self, application, env): """Give option to modify launch environments before application launch. Implementation is optional. To change environments modify passed dictionary of environments. Args: - app (Application): Application that is launcher. + application (Application): Application that is launched. env (dict): Current environemnt variables. """ diff --git a/openpype/modules/ftrack/ftrack_module.py b/openpype/modules/ftrack/ftrack_module.py index 5c38df2e03..f99e189082 100644 --- a/openpype/modules/ftrack/ftrack_module.py +++ b/openpype/modules/ftrack/ftrack_module.py @@ -88,6 +88,40 @@ class FtrackModule( """Implementation of `ILaunchHookPaths`.""" return os.path.join(FTRACK_MODULE_DIR, "launch_hooks") + def modify_application_launch_arguments(self, application, env): + if not application.use_python_2: + return + + self.log.info("Adding Ftrack Python 2 packages to PYTHONPATH.") + + # Prepare vendor dir path + python_2_vendor = os.path.join(FTRACK_MODULE_DIR, "python2_vendor") + + # Add Python 2 modules + python_paths = [ + # `python-ftrack-api` + os.path.join(python_2_vendor, "ftrack-python-api", "source"), + # `arrow` + os.path.join(python_2_vendor, "arrow"), + # `builtins` from `python-future` + # - `python-future` is strict Python 2 module that cause crashes + # of Python 3 scripts executed through OpenPype + # (burnin script etc.) + os.path.join(python_2_vendor, "builtins"), + # `backports.functools_lru_cache` + os.path.join( + python_2_vendor, "backports.functools_lru_cache" + ) + ] + + # Load PYTHONPATH from current launch context + python_path = env.get("PYTHONPATH") + if python_path: + python_paths.append(python_path) + + # Set new PYTHONPATH to launch context environments + env["PYTHONPATH"] = os.pathsep.join(python_paths) + def connect_with_modules(self, enabled_modules): for module in enabled_modules: if not hasattr(module, "get_ftrack_event_handler_paths"): diff --git a/openpype/modules/ftrack/launch_hooks/pre_python2_vendor.py b/openpype/modules/ftrack/launch_hooks/pre_python2_vendor.py deleted file mode 100644 index 0dd894bebf..0000000000 --- a/openpype/modules/ftrack/launch_hooks/pre_python2_vendor.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -from openpype.lib import PreLaunchHook -from openpype_modules.ftrack import FTRACK_MODULE_DIR - - -class PrePython2Support(PreLaunchHook): - """Add python ftrack api module for Python 2 to PYTHONPATH. - - Path to vendor modules is added to the beggining of PYTHONPATH. - """ - - def execute(self): - if not self.application.use_python_2: - return - - self.log.info("Adding Ftrack Python 2 packages to PYTHONPATH.") - - # Prepare vendor dir path - python_2_vendor = os.path.join(FTRACK_MODULE_DIR, "python2_vendor") - - # Add Python 2 modules - python_paths = [ - # `python-ftrack-api` - os.path.join(python_2_vendor, "ftrack-python-api", "source"), - # `arrow` - os.path.join(python_2_vendor, "arrow"), - # `builtins` from `python-future` - # - `python-future` is strict Python 2 module that cause crashes - # of Python 3 scripts executed through OpenPype (burnin script etc.) - os.path.join(python_2_vendor, "builtins"), - # `backports.functools_lru_cache` - os.path.join( - python_2_vendor, "backports.functools_lru_cache" - ) - ] - - # Load PYTHONPATH from current launch context - python_path = self.launch_context.env.get("PYTHONPATH") - if python_path: - python_paths.append(python_path) - - # Set new PYTHONPATH to launch context environments - self.launch_context.env["PYTHONPATH"] = os.pathsep.join(python_paths) From 38d58182fc9f7d908eca61ab573b7562d1e8af97 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 18:44:11 +0200 Subject: [PATCH 05/25] OP-2787 - added updating of script url Remote publish requires path to script which is known only on DL node. Injection of env var is required for remote publish. --- .../repository/custom/plugins/GlobalJobPreLoad.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index eeb1f7744c..bcd853f374 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -87,6 +87,13 @@ def inject_openpype_environment(deadlinePlugin): for key, value in contents.items(): deadlinePlugin.SetProcessEnvironmentVariable(key, value) + script_url = job.GetJobPluginInfoKeyValue("ScriptFilename") + if script_url: + + script_url = script_url.format(**contents).replace("\\", "/") + print(">>> Setting script path {}".format(script_url)) + job.SetJobPluginInfoKeyValue("ScriptFilename", script_url) + print(">>> Removing temporary file") os.remove(export_url) @@ -196,16 +203,19 @@ def __main__(deadlinePlugin): job.GetJobEnvironmentKeyValue('OPENPYPE_RENDER_JOB') or '0' openpype_publish_job = \ job.GetJobEnvironmentKeyValue('OPENPYPE_PUBLISH_JOB') or '0' + openpype_remote_job = \ + job.GetJobEnvironmentKeyValue('OPENPYPE_REMOTE_JOB') or '0' print("--- Job type - render {}".format(openpype_render_job)) print("--- Job type - publish {}".format(openpype_publish_job)) + print("--- Job type - remote {}".format(openpype_remote_job)) if openpype_publish_job == '1' and openpype_render_job == '1': raise RuntimeError("Misconfiguration. Job couldn't be both " + "render and publish.") if openpype_publish_job == '1': inject_render_job_id(deadlinePlugin) - elif openpype_render_job == '1': + elif openpype_render_job == '1' or openpype_remote_job == '1': inject_openpype_environment(deadlinePlugin) else: pype(deadlinePlugin) # backward compatibility with Pype2 From 529c31c4f912d66bece87a17eb597c1ec5dd86ad Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 19:02:17 +0200 Subject: [PATCH 06/25] OP-2787 - updated validator Checks for exactly 1 out set. --- .../hosts/maya/plugins/publish/validate_animation_content.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/validate_animation_content.py b/openpype/hosts/maya/plugins/publish/validate_animation_content.py index bcea761a01..7638c44b87 100644 --- a/openpype/hosts/maya/plugins/publish/validate_animation_content.py +++ b/openpype/hosts/maya/plugins/publish/validate_animation_content.py @@ -30,6 +30,10 @@ class ValidateAnimationContent(pyblish.api.InstancePlugin): assert 'out_hierarchy' in instance.data, "Missing `out_hierarchy` data" + out_sets = [node for node in instance if node.endswith("out_SET")] + msg = "Couldn't find exactly one out_SET: {0}".format(out_sets) + assert len(out_sets) == 1, msg + # All nodes in the `out_hierarchy` must be among the nodes that are # in the instance. The nodes in the instance are found from the top # group, as such this tests whether all nodes are under that top group. From 62ac633da95772488f94ea4b9d53c2c5a74e9b9c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 19:05:35 +0200 Subject: [PATCH 07/25] OP-2787 - used settings from ProcessSubmittedJobOnFarm --- .../submit_maya_remote_publish_deadline.py | 67 +++++++++---------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index 761bc8cc95..b11698f8e8 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -4,18 +4,22 @@ import requests from maya import cmds from openpype.pipeline import legacy_io +from openpype.settings import get_project_settings import pyblish.api -class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): +class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): """Submit Maya scene to perform a local publish in Deadline. Publishing in Deadline can be helpful for scenes that publish very slow. This way it can process in the background on another machine without the Artist having to wait for the publish to finish on their local machine. - Submission is done through the Deadline Web Service. + Submission is done through the Deadline Web Service. DL then triggers + `openpype/scripts/remote_publish.py`. + + Each publishable instance creates its own full publish job. Different from `ProcessSubmittedJobOnFarm` which creates publish job depending on metadata json containing context and instance data of @@ -27,31 +31,24 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): hosts = ["maya"] families = ["deadline"] - # custom deadline attributes - deadline_department = "" - deadline_pool = "" - deadline_pool_secondary = "" - deadline_group = "" - deadline_chunk_size = 1 - deadline_priority = 50 - - def process(self, context): + def process(self, instance): + settings = get_project_settings(os.getenv("AVALON_PROJECT")) + # use setting for publish job on farm, no reason to have it separately + deadline_publish_job_sett = (settings["deadline"] + ["publish"] + ["ProcessSubmittedJobOnFarm"]) # Ensure no errors so far - assert all(result["success"] for result in context.data["results"]), ( - "Errors found, aborting integration..") + assert (all(result["success"] + for result in instance.context.data["results"]), + ("Errors found, aborting integration..")) - # Note that `publish` data member might change in the future. - # See: https://github.com/pyblish/pyblish-base/issues/307 - actives = [i for i in context if i.data["publish"]] - instance_names = sorted(instance.name for instance in actives) - - if not instance_names: + if not instance.data["publish"]: self.log.warning("No active instances found. " "Skipping submission..") return - scene = context.data["currentFile"] + scene = instance.context.data["currentFile"] scenename = os.path.basename(scene) # Get project code @@ -66,17 +63,15 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): "JobInfo": { "Plugin": "MayaBatch", "BatchName": batch_name, - "Priority": 50, "Name": job_name, - "UserName": context.data["user"], - # "Comment": instance.context.data.get("comment", ""), + "UserName": instance.context.data["user"], + "Comment": instance.context.data.get("comment", ""), # "InitialStatus": state - "Department": self.deadline_department, - "ChunkSize": self.deadline_chunk_size, - "Priority": self.deadline_priority, - - "Group": self.deadline_group, - + "Department": deadline_publish_job_sett["deadline_department"], + "ChunkSize": deadline_publish_job_sett["deadline_chunk_size"], + "Priority": deadline_publish_job_sett["deadline_priority"], + "Group": deadline_publish_job_sett["deadline_group"], + "Pool": deadline_publish_job_sett["deadline_pool"], }, "PluginInfo": { @@ -86,7 +81,7 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): # Inputs "SceneFile": scene, - "ScriptFilename": "{OPENPYPE_ROOT}/scripts/remote_publish.py", + "ScriptFilename": "{OPENPYPE_REPOS_ROOT}/openpype/scripts/remote_publish.py", # noqa # Mandatory for Deadline "Version": cmds.about(version=True), @@ -116,10 +111,9 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): environment["AVALON_TASK"] = legacy_io.Session["AVALON_TASK"] environment["AVALON_APP_NAME"] = os.environ.get("AVALON_APP_NAME") environment["OPENPYPE_LOG_NO_COLORS"] = "1" - environment["OPENPYPE_USERNAME"] = context.data["user"] - environment["OPENPYPE_PUBLISH_JOB"] = "1" - environment["OPENPYPE_RENDER_JOB"] = "0" - environment["PYBLISH_ACTIVE_INSTANCES"] = ",".join(instance_names) + environment["OPENPYPE_REMOTE_JOB"] = "1" + environment["OPENPYPE_USERNAME"] = instance.context.data["user"] + environment["OPENPYPE_PUBLISH_SUBSET"] = instance.data["subset"] payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( @@ -129,7 +123,10 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.ContextPlugin): }) self.log.info("Submitting Deadline job ...") - deadline_url = context.data["defaultDeadline"] + deadline_url = instance.context.data["defaultDeadline"] + # if custom one is set in instance, use that + if instance.data.get("deadlineUrl"): + deadline_url = instance.data.get("deadlineUrl") assert deadline_url, "Requires Deadline Webservice URL" url = "{}/api/jobs".format(deadline_url) response = requests.post(url, json=payload, timeout=10) From bf75d18a7b6852415cbb71b69a8a6a05c0ea3754 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 19:07:21 +0200 Subject: [PATCH 08/25] OP-2787 - added collector for remote publishable instances Filters instances from a workfile and marks only these that should be published on a farm. --- .../publish/collect_publishable_instances.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 openpype/modules/deadline/plugins/publish/collect_publishable_instances.py diff --git a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py new file mode 100644 index 0000000000..9a467428fd --- /dev/null +++ b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +"""Collect instances that should be processed and published on DL. + +""" +import os + +import pyblish.api + + +class CollectDeadlinePublishableInstances(pyblish.api.InstancePlugin): + """Collect instances that should be processed and published on DL. + + Some long running publishes (not just renders) could be offloaded to DL, + this plugin compares theirs name against env variable, marks only + publishable by farm. + + Triggered only when running only in headless mode, eg on a farm. + """ + + order = pyblish.api.CollectorOrder + 0.499 + label = "Collect Deadline Publishable Instance" + targets = ["remote"] + + def process(self, instance): + self.log.debug("CollectDeadlinePublishableInstances") + publish_inst = os.environ.get("OPENPYPE_PUBLISH_SUBSET", '') + assert (publish_inst, + "OPENPYPE_PUBLISH_SUBSET env var required for " + "remote publishing") + + subset_name = instance.data["subset"] + if subset_name == publish_inst: + self.log.debug("Publish {}".format(subset_name)) + instance.data["publish"] = True + instance.data["farm"] = False + instance.data["families"].remove("deadline") + else: + self.log.debug("Skipping {}".format(subset_name)) + instance.data["publish"] = False From 72d8633266048ba19b78d425c879aa0325ba042b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 19:09:35 +0200 Subject: [PATCH 09/25] OP-2787 - changed flag from family to farm It probably makes more sense to check specific flag than a family. --- openpype/plugins/publish/integrate_new.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 1a4112107a..b5a7f11904 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -139,7 +139,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): ef, instance.data["family"], instance.data["families"])) return - if "deadline" in instance.data["families"]: + # instance should be published on a farm + if instance.data["farm"]: return self.integrated_file_sizes = {} From 6b71ff1909c3d32e5bebc0595580f1dcde1c1180 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 May 2022 19:25:55 +0200 Subject: [PATCH 10/25] OP-2787 - removed deadline family deadline family is not used anymore anywhere, filtering on integrate is being done on instance.data["farm"] flag. --- .../maya/plugins/publish/collect_animation.py | 3 --- .../maya/plugins/publish/collect_pointcache.py | 14 -------------- 2 files changed, 17 deletions(-) delete mode 100644 openpype/hosts/maya/plugins/publish/collect_pointcache.py diff --git a/openpype/hosts/maya/plugins/publish/collect_animation.py b/openpype/hosts/maya/plugins/publish/collect_animation.py index b442113fbc..9b1e38fd0a 100644 --- a/openpype/hosts/maya/plugins/publish/collect_animation.py +++ b/openpype/hosts/maya/plugins/publish/collect_animation.py @@ -55,6 +55,3 @@ class CollectAnimationOutputGeometry(pyblish.api.InstancePlugin): # Store data in the instance for the validator instance.data["out_hierarchy"] = hierarchy - - if instance.data.get("farm"): - instance.data["families"].append("deadline") diff --git a/openpype/hosts/maya/plugins/publish/collect_pointcache.py b/openpype/hosts/maya/plugins/publish/collect_pointcache.py deleted file mode 100644 index b55babe372..0000000000 --- a/openpype/hosts/maya/plugins/publish/collect_pointcache.py +++ /dev/null @@ -1,14 +0,0 @@ -import pyblish.api - - -class CollectPointcache(pyblish.api.InstancePlugin): - """Collect pointcache data for instance.""" - - order = pyblish.api.CollectorOrder + 0.4 - families = ["pointcache"] - label = "Collect Pointcache" - hosts = ["maya"] - - def process(self, instance): - if instance.data.get("farm"): - instance.data["families"].append("deadline") \ No newline at end of file From 9244389b585b654f92dbf76a8e5ed2692ac4cb3b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 23 May 2022 17:16:44 +0200 Subject: [PATCH 11/25] OP-2790 - safer querying of farm flag --- openpype/plugins/publish/integrate_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index b5a7f11904..fa0582c65a 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -140,7 +140,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return # instance should be published on a farm - if instance.data["farm"]: + if instance.data.get("farm"): return self.integrated_file_sizes = {} From c9c6fe34fe63fff1e4ba57d73bab6c361e7203ca Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 10:42:37 +0200 Subject: [PATCH 12/25] OP-2787 - changed assert to PublishXmlValidationError --- .../help/submit_maya_remote_publish_deadline.xml | 16 ++++++++++++++++ .../submit_maya_remote_publish_deadline.py | 8 ++++---- 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 openpype/hosts/maya/plugins/publish/help/submit_maya_remote_publish_deadline.xml diff --git a/openpype/hosts/maya/plugins/publish/help/submit_maya_remote_publish_deadline.xml b/openpype/hosts/maya/plugins/publish/help/submit_maya_remote_publish_deadline.xml new file mode 100644 index 0000000000..e92320ccdc --- /dev/null +++ b/openpype/hosts/maya/plugins/publish/help/submit_maya_remote_publish_deadline.xml @@ -0,0 +1,16 @@ + + + +Errors found + +## Publish process has errors + +At least one plugin failed before this plugin, job won't be sent to Deadline for processing before all issues are fixed. + +### How to repair? + +Check all failing plugins (should be highlighted in red) and fix issues if possible. + + + + \ No newline at end of file diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index b11698f8e8..be8c50d7b3 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -3,7 +3,7 @@ import requests from maya import cmds -from openpype.pipeline import legacy_io +from openpype.pipeline import legacy_io, PublishXmlValidationError from openpype.settings import get_project_settings import pyblish.api @@ -39,9 +39,9 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): ["ProcessSubmittedJobOnFarm"]) # Ensure no errors so far - assert (all(result["success"] - for result in instance.context.data["results"]), - ("Errors found, aborting integration..")) + if not (all(result["success"] + for result in instance.context.data["results"])): + raise PublishXmlValidationError("Publish process has errors") if not instance.data["publish"]: self.log.warning("No active instances found. " From 4ca419f0b63c2782d8955b8012674ca131a57ad9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 11:36:49 +0200 Subject: [PATCH 13/25] Revert "OP-2787 - removed deadline family" This reverts commit 6b71ff19 --- .../maya/plugins/publish/collect_animation.py | 3 +++ .../maya/plugins/publish/collect_pointcache.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 openpype/hosts/maya/plugins/publish/collect_pointcache.py diff --git a/openpype/hosts/maya/plugins/publish/collect_animation.py b/openpype/hosts/maya/plugins/publish/collect_animation.py index 9b1e38fd0a..b442113fbc 100644 --- a/openpype/hosts/maya/plugins/publish/collect_animation.py +++ b/openpype/hosts/maya/plugins/publish/collect_animation.py @@ -55,3 +55,6 @@ class CollectAnimationOutputGeometry(pyblish.api.InstancePlugin): # Store data in the instance for the validator instance.data["out_hierarchy"] = hierarchy + + if instance.data.get("farm"): + instance.data["families"].append("deadline") diff --git a/openpype/hosts/maya/plugins/publish/collect_pointcache.py b/openpype/hosts/maya/plugins/publish/collect_pointcache.py new file mode 100644 index 0000000000..b55babe372 --- /dev/null +++ b/openpype/hosts/maya/plugins/publish/collect_pointcache.py @@ -0,0 +1,14 @@ +import pyblish.api + + +class CollectPointcache(pyblish.api.InstancePlugin): + """Collect pointcache data for instance.""" + + order = pyblish.api.CollectorOrder + 0.4 + families = ["pointcache"] + label = "Collect Pointcache" + hosts = ["maya"] + + def process(self, instance): + if instance.data.get("farm"): + instance.data["families"].append("deadline") \ No newline at end of file From c96ea856425550835c7c5dfc42b1965a54ca0902 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 11:40:19 +0200 Subject: [PATCH 14/25] OP-2787 - change families to more generic Not only Deadline could be used for remote publish. DL plugin will be picked only if DL module is enabled. --- openpype/hosts/maya/plugins/publish/collect_animation.py | 2 +- openpype/hosts/maya/plugins/publish/collect_pointcache.py | 2 +- .../plugins/publish/submit_maya_remote_publish_deadline.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/collect_animation.py b/openpype/hosts/maya/plugins/publish/collect_animation.py index b442113fbc..549098863f 100644 --- a/openpype/hosts/maya/plugins/publish/collect_animation.py +++ b/openpype/hosts/maya/plugins/publish/collect_animation.py @@ -57,4 +57,4 @@ class CollectAnimationOutputGeometry(pyblish.api.InstancePlugin): instance.data["out_hierarchy"] = hierarchy if instance.data.get("farm"): - instance.data["families"].append("deadline") + instance.data["families"].append("publish.farm") diff --git a/openpype/hosts/maya/plugins/publish/collect_pointcache.py b/openpype/hosts/maya/plugins/publish/collect_pointcache.py index b55babe372..a841341f72 100644 --- a/openpype/hosts/maya/plugins/publish/collect_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/collect_pointcache.py @@ -11,4 +11,4 @@ class CollectPointcache(pyblish.api.InstancePlugin): def process(self, instance): if instance.data.get("farm"): - instance.data["families"].append("deadline") \ No newline at end of file + instance.data["families"].append("publish.farm") diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index be8c50d7b3..210fefb520 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -29,7 +29,7 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): label = "Submit Scene to Deadline" order = pyblish.api.IntegratorOrder hosts = ["maya"] - families = ["deadline"] + families = ["publish.farm"] def process(self, instance): settings = get_project_settings(os.getenv("AVALON_PROJECT")) From 0d03e3e2f836476310b899a21aec5b49dbd2a7c8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 11:40:50 +0200 Subject: [PATCH 15/25] OP-2787 - removed obsolete part of code Doesn't do anything. --- openpype/hosts/maya/plugins/publish/extract_animation.py | 7 +------ openpype/hosts/maya/plugins/publish/extract_pointcache.py | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_animation.py b/openpype/hosts/maya/plugins/publish/extract_animation.py index 87f2d35192..1ccc8f5cfe 100644 --- a/openpype/hosts/maya/plugins/publish/extract_animation.py +++ b/openpype/hosts/maya/plugins/publish/extract_animation.py @@ -24,12 +24,7 @@ class ExtractAnimation(openpype.api.Extractor): def process(self, instance): if instance.data.get("farm"): - path = os.path.join( - os.path.dirname(instance.context.data["currentFile"]), - "cache", - instance.data["name"] + ".abc" - ) - instance.data["expectedFiles"] = [os.path.normpath(path)] + self.log.debug("Should be processed on farm, skipping.") return # Collect the out set nodes diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index 7ad4c6dfa9..ff3d97ded1 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -26,12 +26,7 @@ class ExtractAlembic(openpype.api.Extractor): def process(self, instance): if instance.data.get("farm"): - path = os.path.join( - os.path.dirname(instance.context.data["currentFile"]), - "cache", - instance.data["name"] + ".abc" - ) - instance.data["expectedFiles"] = [os.path.normpath(path)] + self.log.debug("Should be processed on farm, skipping.") return nodes = instance[:] From de161da68b4035c4fde2e376dc8da1ab2b79d093 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 11:45:29 +0200 Subject: [PATCH 16/25] OP-2787 - created explicit env var HEADLESS_PUBLISH Env var created to differentiate launch of Maya on the farm. lib.IS_HEADLESS might be triggered locally, it is not precise enough. Added same env var to all commands to standardize it a bit. --- openpype/hosts/maya/api/pipeline.py | 5 ++++- .../submit_maya_remote_publish_deadline.py | 2 ++ openpype/pype_commands.py | 18 +++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index c2fe8a95a5..6fc93e864f 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -72,7 +72,10 @@ def install(): log.info(("Running in headless mode, skipping Maya " "save/open/new callback installation..")) - # Register default "local" target + return + + if os.environ.get("HEADLESS_PUBLISH"): + # Maya launched on farm, lib.IS_HEADLESS might be triggered locally too print("Registering pyblish target: remote") pyblish.api.register_target("remote") return diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index 210fefb520..8f50878db4 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -114,6 +114,8 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): environment["OPENPYPE_REMOTE_JOB"] = "1" environment["OPENPYPE_USERNAME"] = instance.context.data["user"] environment["OPENPYPE_PUBLISH_SUBSET"] = instance.data["subset"] + environment["HEADLESS_PUBLISH"] = "1" + payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d945a1f697..90c582a319 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -144,6 +144,7 @@ class PypeCommands: pyblish.api.register_target("farm") os.environ["OPENPYPE_PUBLISH_DATA"] = os.pathsep.join(paths) + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib log.info("Running publish ...") @@ -173,9 +174,11 @@ class PypeCommands: user_email, targets=None): """Opens installed variant of 'host' and run remote publish there. + Eventually should be yanked out to Webpublisher cli. + Currently implemented and tested for Photoshop where customer wants to process uploaded .psd file and publish collected layers - from there. + from there. Triggered by Webpublisher. Checks if no other batches are running (status =='in_progress). If so, it sleeps for SLEEP (this is separate process), @@ -273,7 +276,8 @@ class PypeCommands: def remotepublish(project, batch_path, user_email, targets=None): """Start headless publishing. - Used to publish rendered assets, workfiles etc. + Used to publish rendered assets, workfiles etc via Webpublisher. + Eventually should be yanked out to Webpublisher cli. Publish use json from passed paths argument. @@ -309,6 +313,7 @@ class PypeCommands: os.environ["AVALON_PROJECT"] = project os.environ["AVALON_APP"] = host_name os.environ["USER_EMAIL"] = user_email + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib pyblish.api.register_host(host_name) @@ -331,9 +336,12 @@ class PypeCommands: log.info("Publish finished.") @staticmethod - def extractenvironments( - output_json_path, project, asset, task, app, env_group - ): + def extractenvironments(output_json_path, project, asset, task, app, + env_group): + """Produces json file with environment based on project and app. + + Called by Deadline plugin to propagate environment into render jobs. + """ if all((project, asset, task, app)): from openpype.api import get_app_environments_for_context env = get_app_environments_for_context( From 8cda0ebbeef23ceccc7f1a9d9963eecc93219012 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 26 May 2022 12:09:13 +0200 Subject: [PATCH 17/25] OP-2787 - Hound --- .../plugins/publish/collect_publishable_instances.py | 7 ++++--- .../plugins/publish/submit_maya_remote_publish_deadline.py | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py index 9a467428fd..741a2a5af8 100644 --- a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py +++ b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py @@ -5,6 +5,7 @@ import os import pyblish.api +from openpype.pipeline import PublishValidationError class CollectDeadlinePublishableInstances(pyblish.api.InstancePlugin): @@ -24,9 +25,9 @@ class CollectDeadlinePublishableInstances(pyblish.api.InstancePlugin): def process(self, instance): self.log.debug("CollectDeadlinePublishableInstances") publish_inst = os.environ.get("OPENPYPE_PUBLISH_SUBSET", '') - assert (publish_inst, - "OPENPYPE_PUBLISH_SUBSET env var required for " - "remote publishing") + if not publish_inst: + raise PublishValidationError("OPENPYPE_PUBLISH_SUBSET env var " + "required for remote publishing") subset_name = instance.data["subset"] if subset_name == publish_inst: diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index 8f50878db4..196adc5906 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -116,7 +116,6 @@ class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): environment["OPENPYPE_PUBLISH_SUBSET"] = instance.data["subset"] environment["HEADLESS_PUBLISH"] = "1" - payload["JobInfo"].update({ "EnvironmentKeyValue%d" % index: "{key}={value}".format( key=key, From 67fdfba49f35b585ad5e391271fb6ebe5e43f2d3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 27 May 2022 11:01:40 +0200 Subject: [PATCH 18/25] OP-2790 - added note about remote publish to documentation --- website/docs/artist_hosts_maya.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/docs/artist_hosts_maya.md b/website/docs/artist_hosts_maya.md index 73e89384e8..48e1093753 100644 --- a/website/docs/artist_hosts_maya.md +++ b/website/docs/artist_hosts_maya.md @@ -312,6 +312,10 @@ Example setup: ![Maya - Point Cache Example](assets/maya-pointcache_setup.png) +:::note Publish on farm +If your studio has Deadline configured, artists could choose to offload potentially long running export of pointache and publish it to the farm. +Only thing that is necessary is to toggle `Farm` property in created pointcache instance to True. + ### Loading Point Caches Loading point cache means creating reference to **abc** file with Go **OpenPype → Load...**. From 5464fd40850ae1d25c1b467149249bd5edaaae9e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 13:58:11 +0200 Subject: [PATCH 19/25] OP-2787 - fix extractors could be run on a farm --- openpype/hosts/maya/plugins/publish/extract_animation.py | 3 +++ openpype/hosts/maya/plugins/publish/extract_pointcache.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_animation.py b/openpype/hosts/maya/plugins/publish/extract_animation.py index 1ccc8f5cfe..8f2bc26d08 100644 --- a/openpype/hosts/maya/plugins/publish/extract_animation.py +++ b/openpype/hosts/maya/plugins/publish/extract_animation.py @@ -16,11 +16,14 @@ class ExtractAnimation(openpype.api.Extractor): Positions and normals, uvs, creases are preserved, but nothing more, for plain and predictable point caches. + Plugin can run locally or remotely (on a farm - if instance is marked with + "farm" it will be skipped in local processing, but processed on farm) """ label = "Extract Animation" hosts = ["maya"] families = ["animation"] + targets = ["local", "remote"] def process(self, instance): if instance.data.get("farm"): diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index ff3d97ded1..5606ea9459 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -16,6 +16,8 @@ class ExtractAlembic(openpype.api.Extractor): Positions and normals, uvs, creases are preserved, but nothing more, for plain and predictable point caches. + Plugin can run locally or remotely (on a farm - if instance is marked with + "farm" it will be skipped in local processing, but processed on farm) """ label = "Extract Pointcache (Alembic)" @@ -23,6 +25,7 @@ class ExtractAlembic(openpype.api.Extractor): families = ["pointcache", "model", "vrayproxy"] + targets = ["local", "remote"] def process(self, instance): if instance.data.get("farm"): From 376ddc41329206173f2d8f4e9d568e0aef4cebc3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:01:23 +0200 Subject: [PATCH 20/25] OP-2787 - added raising error for Deadline --- openpype/lib/remote_publish.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index da2497e1a5..d7884d0200 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -60,7 +60,7 @@ def start_webpublish_log(dbcon, batch_id, user): }).inserted_id -def publish(log, close_plugin_name=None): +def publish(log, close_plugin_name=None, raise_error=False): """Loops through all plugins, logs to console. Used for tests. Args: @@ -79,10 +79,15 @@ def publish(log, close_plugin_name=None): result["plugin"].label, record.msg)) if result["error"]: - log.error(error_format.format(**result)) + error_message = error_format.format(**result) + log.error(error_message) if close_plugin: # close host app explicitly after error context = pyblish.api.Context() close_plugin().process(context) + if raise_error: + # Fatal Error is because of Deadline + error_message = "Fatal Error: " + error_format.format(**result) + raise RuntimeError(error_message) def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): From 8de1cbf7320f792e0d6cf8e2709f918a9d8c4ddd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:15:05 +0200 Subject: [PATCH 21/25] OP-2787 - fixed resolution order --- openpype/hosts/maya/api/pipeline.py | 16 ++++++++-------- openpype/scripts/remote_publish.py | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/maya/api/pipeline.py b/openpype/hosts/maya/api/pipeline.py index 6fc93e864f..0261694be2 100644 --- a/openpype/hosts/maya/api/pipeline.py +++ b/openpype/hosts/maya/api/pipeline.py @@ -66,20 +66,20 @@ def install(): log.info("Installing callbacks ... ") register_event_callback("init", on_init) - # Callbacks below are not required for headless mode, the `init` however - # is important to load referenced Alembics correctly at rendertime. + if os.environ.get("HEADLESS_PUBLISH"): + # Maya launched on farm, lib.IS_HEADLESS might be triggered locally too + # target "farm" == rendering on farm, expects OPENPYPE_PUBLISH_DATA + # target "remote" == remote execution + print("Registering pyblish target: remote") + pyblish.api.register_target("remote") + return + if lib.IS_HEADLESS: log.info(("Running in headless mode, skipping Maya " "save/open/new callback installation..")) return - if os.environ.get("HEADLESS_PUBLISH"): - # Maya launched on farm, lib.IS_HEADLESS might be triggered locally too - print("Registering pyblish target: remote") - pyblish.api.register_target("remote") - return - print("Registering pyblish target: local") pyblish.api.register_target("local") diff --git a/openpype/scripts/remote_publish.py b/openpype/scripts/remote_publish.py index b54c8d931b..8e5c91d663 100644 --- a/openpype/scripts/remote_publish.py +++ b/openpype/scripts/remote_publish.py @@ -1,6 +1,7 @@ try: from openpype.api import Logger import openpype.lib.remote_publish + import pyblish.api except ImportError as exc: # Ensure Deadline fails by output an error that contains "Fatal Error:" raise ImportError("Fatal Error: %s" % exc) @@ -8,4 +9,4 @@ except ImportError as exc: if __name__ == "__main__": # Perform remote publish with thorough error checking log = Logger.get_logger(__name__) - openpype.lib.remote_publish.publish(log) + openpype.lib.remote_publish.publish(log, raise_error=True) From 2dd79b32e7ebbc3caaf863484806569bf62ab1f3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:15:25 +0200 Subject: [PATCH 22/25] OP-2787 - removed unnecessary family --- .../deadline/plugins/publish/collect_publishable_instances.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py index 741a2a5af8..b00381b6cf 100644 --- a/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py +++ b/openpype/modules/deadline/plugins/publish/collect_publishable_instances.py @@ -34,7 +34,6 @@ class CollectDeadlinePublishableInstances(pyblish.api.InstancePlugin): self.log.debug("Publish {}".format(subset_name)) instance.data["publish"] = True instance.data["farm"] = False - instance.data["families"].remove("deadline") else: self.log.debug("Skipping {}".format(subset_name)) instance.data["publish"] = False From d37815467a059cfa2ad2dbde6ce4025ec00def2d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:20:50 +0200 Subject: [PATCH 23/25] OP-2787 - added extracted path to explicit cleanup --- openpype/hosts/maya/plugins/publish/extract_animation.py | 2 ++ openpype/hosts/maya/plugins/publish/extract_pointcache.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_animation.py b/openpype/hosts/maya/plugins/publish/extract_animation.py index 8f2bc26d08..abe5ed3bf5 100644 --- a/openpype/hosts/maya/plugins/publish/extract_animation.py +++ b/openpype/hosts/maya/plugins/publish/extract_animation.py @@ -95,4 +95,6 @@ class ExtractAnimation(openpype.api.Extractor): } instance.data["representations"].append(representation) + instance.context.data["cleanupFullPaths"].append(path) + self.log.info("Extracted {} to {}".format(instance, dirname)) diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index 5606ea9459..c4c8610ebb 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -98,4 +98,6 @@ class ExtractAlembic(openpype.api.Extractor): } instance.data["representations"].append(representation) + instance.context.data["cleanupFullPaths"].append(path) + self.log.info("Extracted {} to {}".format(instance, dirname)) From 0d7d43316b94685f78fb0a7e2e39153a98996b36 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:44:37 +0200 Subject: [PATCH 24/25] OP-2787 - changed class to api.Integrator This plugin should run only locally, not no a farm. --- .../plugins/publish/submit_maya_remote_publish_deadline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py index 196adc5906..c31052be07 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_remote_publish_deadline.py @@ -5,11 +5,12 @@ from maya import cmds from openpype.pipeline import legacy_io, PublishXmlValidationError from openpype.settings import get_project_settings +import openpype.api import pyblish.api -class MayaSubmitRemotePublishDeadline(pyblish.api.InstancePlugin): +class MayaSubmitRemotePublishDeadline(openpype.api.Integrator): """Submit Maya scene to perform a local publish in Deadline. Publishing in Deadline can be helpful for scenes that publish very slow. From 55a69074c6b550b4e30d99b658b9ec664d30214b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 May 2022 14:46:05 +0200 Subject: [PATCH 25/25] OP-2787 - Hound --- openpype/scripts/remote_publish.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/scripts/remote_publish.py b/openpype/scripts/remote_publish.py index 8e5c91d663..d322f369d1 100644 --- a/openpype/scripts/remote_publish.py +++ b/openpype/scripts/remote_publish.py @@ -1,7 +1,6 @@ try: from openpype.api import Logger import openpype.lib.remote_publish - import pyblish.api except ImportError as exc: # Ensure Deadline fails by output an error that contains "Fatal Error:" raise ImportError("Fatal Error: %s" % exc)