diff --git a/openpype/cli.py b/openpype/cli.py index 512bd4663b..e582d8e3af 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -166,7 +166,7 @@ def publish(debug, paths, targets): @click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets", default=None, multiple=True) -def remotepublishfromapp(debug, project, path, host, targets=None, user=None): +def remotepublishfromapp(debug, project, path, host, user=None, targets=None): """Start CLI publishing. Publish collects json from paths provided as an argument. @@ -174,18 +174,19 @@ def remotepublishfromapp(debug, project, path, host, targets=None, user=None): """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands.remotepublishfromapp(project, path, host, user, - targets=targets) + PypeCommands.remotepublishfromapp( + project, path, host, user, targets=targets + ) + @main.command() @click.argument("path") @click.option("-d", "--debug", is_flag=True, help="Print debug messages") -@click.option("-h", "--host", help="Host") @click.option("-u", "--user", help="User email address") @click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets", default=None, multiple=True) -def remotepublish(debug, project, path, host, targets=None, user=None): +def remotepublish(debug, project, path, user=None, targets=None): """Start CLI publishing. Publish collects json from paths provided as an argument. @@ -193,7 +194,7 @@ def remotepublish(debug, project, path, host, targets=None, user=None): """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands.remotepublish(project, path, host, user, targets=targets) + PypeCommands.remotepublish(project, path, user, targets=targets) @main.command() diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py new file mode 100644 index 0000000000..a710fcb3e8 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py @@ -0,0 +1,84 @@ +"""Loads batch context from json and continues in publish process. + +Provides: + context -> Loaded batch file. +""" + +import os + +import pyblish.api +from avalon import io +from openpype.lib.plugin_tools import ( + parse_json, + get_batch_asset_task_info +) +from openpype.lib.remote_publish import get_webpublish_conn + + +class CollectBatchData(pyblish.api.ContextPlugin): + """Collect batch data from json stored in 'OPENPYPE_PUBLISH_DATA' env dir. + + The directory must contain 'manifest.json' file where batch data should be + stored. + """ + # must be really early, context values are only in json file + order = pyblish.api.CollectorOrder - 0.495 + label = "Collect batch data" + host = ["webpublisher"] + + def process(self, context): + batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") + + assert batch_dir, ( + "Missing `OPENPYPE_PUBLISH_DATA`") + + assert os.path.exists(batch_dir), \ + "Folder {} doesn't exist".format(batch_dir) + + project_name = os.environ.get("AVALON_PROJECT") + if project_name is None: + raise AssertionError( + "Environment `AVALON_PROJECT` was not found." + "Could not set project `root` which may cause issues." + ) + + batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) + + context.data["batchDir"] = batch_dir + context.data["batchData"] = batch_data + + asset_name, task_name, task_type = get_batch_asset_task_info( + batch_data["context"] + ) + + os.environ["AVALON_ASSET"] = asset_name + io.Session["AVALON_ASSET"] = asset_name + os.environ["AVALON_TASK"] = task_name + io.Session["AVALON_TASK"] = task_name + + context.data["asset"] = asset_name + context.data["task"] = task_name + context.data["taskType"] = task_type + + self._set_ctx_path(batch_data) + + def _set_ctx_path(self, batch_data): + dbcon = get_webpublish_conn() + + batch_id = batch_data["batch"] + ctx_path = batch_data["context"]["path"] + self.log.info("ctx_path: {}".format(ctx_path)) + self.log.info("batch_id: {}".format(batch_id)) + if ctx_path and batch_id: + self.log.info("Updating log record") + dbcon.update_one( + { + "batch_id": batch_id, + "status": "in_progress" + }, + { + "$set": { + "path": ctx_path + } + } + ) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_fps.py b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py index 79fe53176a..b5e665c761 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_fps.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py @@ -20,9 +20,8 @@ class CollectFPS(pyblish.api.InstancePlugin): hosts = ["webpublisher"] def process(self, instance): - fps = instance.context.data["fps"] + instance_fps = instance.data.get("fps") + if instance_fps is None: + instance.data["fps"] = instance.context.data["fps"] - instance.data.update({ - "fps": fps - }) self.log.debug(f"instance.data: {pformat(instance.data)}") diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index ecd65ebae4..d2754b3df3 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -1,21 +1,19 @@ -"""Loads publishing context from json and continues in publish process. +"""Create instances from batch data and continues in publish process. Requires: - anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + CollectBatchData Provides: context, instances -> All data from previous publishing process. """ import os -import json import clique import tempfile - -import pyblish.api from avalon import io +import pyblish.api from openpype.lib import prepare_template_data -from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info +from openpype.lib.plugin_tools import parse_json class CollectPublishedFiles(pyblish.api.ContextPlugin): @@ -28,28 +26,28 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder - 0.490 label = "Collect rendered frames" host = ["webpublisher"] - - _context = None + targets = ["filespublish"] # from Settings task_type_to_family = {} - def _process_batch(self, dir_url): - task_subfolders = [ - os.path.join(dir_url, o) - for o in os.listdir(dir_url) - if os.path.isdir(os.path.join(dir_url, o))] + def process(self, context): + batch_dir = context.data["batchDir"] + task_subfolders = [] + for folder_name in os.listdir(batch_dir): + full_path = os.path.join(batch_dir, folder_name) + if os.path.isdir(full_path): + task_subfolders.append(full_path) + self.log.info("task_sub:: {}".format(task_subfolders)) + + asset_name = context.data["asset"] + task_name = context.data["task"] + task_type = context.data["taskType"] for task_dir in task_subfolders: task_data = parse_json(os.path.join(task_dir, "manifest.json")) self.log.info("task_data:: {}".format(task_data)) - ctx = task_data["context"] - - asset, task_name, task_type = get_batch_asset_task_info(ctx) - - if task_name: - os.environ["AVALON_TASK"] = task_name is_sequence = len(task_data["files"]) > 1 @@ -60,26 +58,20 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): is_sequence, extension.replace(".", '')) - subset = self._get_subset_name(family, subset_template, task_name, - task_data["variant"]) + subset = self._get_subset_name( + family, subset_template, task_name, task_data["variant"] + ) + version = self._get_last_version(asset_name, subset) + 1 - os.environ["AVALON_ASSET"] = asset - io.Session["AVALON_ASSET"] = asset - - instance = self._context.create_instance(subset) - instance.data["asset"] = asset + instance = context.create_instance(subset) + instance.data["asset"] = asset_name instance.data["subset"] = subset instance.data["family"] = family instance.data["families"] = families - instance.data["version"] = \ - self._get_last_version(asset, subset) + 1 + instance.data["version"] = version instance.data["stagingDir"] = tempfile.mkdtemp() instance.data["source"] = "webpublisher" - # to store logging info into DB openpype.webpublishes - instance.data["ctx_path"] = ctx["path"] - instance.data["batch_id"] = task_data["batch"] - # to convert from email provided into Ftrack username instance.data["user_email"] = task_data["user"] @@ -230,23 +222,3 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): return version[0].get("version") or 0 else: return 0 - - def process(self, context): - self._context = context - - batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") - - assert batch_dir, ( - "Missing `OPENPYPE_PUBLISH_DATA`") - - assert os.path.exists(batch_dir), \ - "Folder {} doesn't exist".format(batch_dir) - - project_name = os.environ.get("AVALON_PROJECT") - if project_name is None: - raise AssertionError( - "Environment `AVALON_PROJECT` was not found." - "Could not set project `root` which may cause issues." - ) - - self._process_batch(batch_dir) diff --git a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py deleted file mode 100644 index 419c065e16..0000000000 --- a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py +++ /dev/null @@ -1,38 +0,0 @@ -import os - -import pyblish.api -from openpype.lib import OpenPypeMongoConnection - - -class IntegrateContextToLog(pyblish.api.ContextPlugin): - """ Adds context information to log document for displaying in front end""" - - label = "Integrate Context to Log" - order = pyblish.api.IntegratorOrder - 0.1 - hosts = ["webpublisher"] - - def process(self, context): - self.log.info("Integrate Context to Log") - - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - dbcon = mongo_client[database_name]["webpublishes"] - - for instance in context: - self.log.info("ctx_path: {}".format(instance.data.get("ctx_path"))) - self.log.info("batch_id: {}".format(instance.data.get("batch_id"))) - if instance.data.get("ctx_path") and instance.data.get("batch_id"): - self.log.info("Updating log record") - dbcon.update_one( - { - "batch_id": instance.data.get("batch_id"), - "status": "in_progress" - }, - {"$set": - { - "path": instance.data.get("ctx_path") - - }} - ) - - return diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 920ed042dc..73e5113f38 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -176,23 +176,48 @@ class TaskNode(Node): class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): """Triggers headless publishing of batch.""" async def post(self, request) -> Response: - # for postprocessing in host, currently only PS - host_map = {"photoshop": [".psd", ".psb"]} + # Validate existence of openpype executable + openpype_app = self.resource.executable + if not openpype_app or not os.path.exists(openpype_app): + msg = "Non existent OpenPype executable {}".format(openpype_app) + raise RuntimeError(msg) + # for postprocessing in host, currently only PS output = {} log.info("WebpublisherBatchPublishEndpoint called") content = await request.json() - batch_path = os.path.join(self.resource.upload_dir, - content["batch"]) + # Each filter have extensions which are checked on first task item + # - first filter with extensions that are on first task is used + # - filter defines command and can extend arguments dictionary + # This is used only if 'studio_processing' is enabled on batch + studio_processing_filters = [ + # Photoshop filter + { + "extensions": [".psd", ".psb"], + "command": "remotepublishfromapp", + "arguments": { + # Command 'remotepublishfromapp' requires --host argument + "host": "photoshop", + # Make sure targets are set to None for cases that default + # would change + # - targets argument is not used in 'remotepublishfromapp' + "targets": None + } + } + ] - add_args = { - "host": "webpublisher", - "project": content["project_name"], - "user": content["user"] - } + batch_path = os.path.join(self.resource.upload_dir, content["batch"]) + # Default command and arguments command = "remotepublish" + add_args = { + # All commands need 'project' and 'user' + "project": content["project_name"], + "user": content["user"], + + "targets": ["filespublish"] + } if content.get("studio_processing"): log.info("Post processing called") @@ -208,32 +233,34 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): raise ValueError( "Cannot parse batch meta in {} folder".format(task_data)) - command = "remotepublishfromapp" - for host, extensions in host_map.items(): - for ext in extensions: - for file_name in task_data["files"]: - if ext in file_name: - add_args["host"] = host - break + for process_filter in studio_processing_filters: + filter_extensions = process_filter.get("extensions") or [] + for file_name in task_data["files"]: + file_ext = os.path.splitext(file_name)[-1].lower() + if file_ext in filter_extensions: + # Change command + command = process_filter["command"] + # Update arguments + add_args.update( + process_filter.get("arguments") or {} + ) + break - if not add_args.get("host"): - raise ValueError( - "Couldn't discern host from {}".format(task_data["files"])) - - openpype_app = self.resource.executable args = [ openpype_app, command, batch_path ] - if not openpype_app or not os.path.exists(openpype_app): - msg = "Non existent OpenPype executable {}".format(openpype_app) - raise RuntimeError(msg) - for key, value in add_args.items(): - args.append("--{}".format(key)) - args.append(value) + # Skip key values where value is None + if value is not None: + args.append("--{}".format(key)) + # Extend list into arguments (targets can be a list) + if isinstance(value, (tuple, list)): + args.extend(value) + else: + args.append(value) log.info("args:: {}".format(args)) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5fac5cacc7..e355bb74a3 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -137,16 +137,13 @@ class PypeCommands: SLEEP = 5 # seconds for another loop check for concurrently runs WAIT_FOR = 300 # seconds to wait for conc. runs - from openpype import install, uninstall from openpype.api import Logger + from openpype.lib import ApplicationManager log = Logger.get_logger() log.info("remotepublishphotoshop command") - install() - - from openpype.lib import ApplicationManager application_manager = ApplicationManager() found_variant_key = find_variant_key(application_manager, host) @@ -220,10 +217,8 @@ class PypeCommands: while launched_app.poll() is None: time.sleep(0.5) - uninstall() - @staticmethod - def remotepublish(project, batch_path, host, user, targets=None): + def remotepublish(project, batch_path, user, targets=None): """Start headless publishing. Used to publish rendered assets, workfiles etc. @@ -235,10 +230,9 @@ class PypeCommands: per call of remotepublish batch_path (str): Path batch folder. Contains subfolders with resources (workfile, another subfolder 'renders' etc.) - targets (string): What module should be targeted - (to choose validator for example) - host (string) user (string): email address for webpublisher + targets (list): Pyblish targets + (to choose validator for example) Raises: RuntimeError: When there is no path to process. @@ -246,21 +240,22 @@ class PypeCommands: if not batch_path: raise RuntimeError("No publish paths specified") - from openpype import install, uninstall - from openpype.api import Logger - # Register target and host import pyblish.api import pyblish.util + import avalon.api + from openpype.hosts.webpublisher import api as webpublisher - log = Logger.get_logger() + log = PypeLogger.get_logger() log.info("remotepublish command") - install() + host_name = "webpublisher" + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + os.environ["AVALON_PROJECT"] = project + os.environ["AVALON_APP"] = host_name - if host: - pyblish.api.register_host(host) + pyblish.api.register_host(host_name) if targets: if isinstance(targets, str): @@ -268,13 +263,6 @@ class PypeCommands: for target in targets: pyblish.api.register_target(target) - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host - - import avalon.api - from openpype.hosts.webpublisher import api as webpublisher - avalon.api.install(webpublisher) log.info("Running publish ...") @@ -286,7 +274,6 @@ class PypeCommands: publish_and_log(dbcon, _id, log) log.info("Publish finished.") - uninstall() @staticmethod def extractenvironments(output_json_path, project, asset, task, app):