From caf9e014bdb8150ac2abcc8dd23cbe5cb88ab09d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:32:46 +0200 Subject: [PATCH 01/17] implemented webpublish addon --- openpype/hosts/webpublisher/__init__.py | 10 ++++++++++ openpype/hosts/webpublisher/addon.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 openpype/hosts/webpublisher/addon.py diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index e69de29bb2..4e918c5d7d 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -0,0 +1,10 @@ +from .addon import ( + WebpublisherAddon, + WEBPUBLISHER_ROOT_DIR, +) + + +__all__ = ( + "WebpublisherAddon", + "WEBPUBLISHER_ROOT_DIR", +) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py new file mode 100644 index 0000000000..3d76115df1 --- /dev/null +++ b/openpype/hosts/webpublisher/addon.py @@ -0,0 +1,13 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +WEBPUBLISHER_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class WebpublisherAddon(OpenPypeModule, IHostModule): + name = "webpublisher" + host_name = "webpublisher" + + def initialize(self, module_settings): + self.enabled = True From b188afe97569a7141f6a3c3e14dc7966b1e3b853 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:39:19 +0200 Subject: [PATCH 02/17] reorganized imports in pype commands --- openpype/pype_commands.py | 43 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 67b0b8ad76..cb84fac3c7 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -5,19 +5,6 @@ import sys import json import time -from openpype.api import get_app_environments_for_context -from openpype.lib.plugin_tools import get_batch_asset_task_info -from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - publish_and_log, - fail_batch, - find_variant_key, - get_task_data, - get_timeout, - IN_PROGRESS_STATUS -) - class PypeCommands: """Class implementing commands used by Pype. @@ -100,6 +87,7 @@ class PypeCommands: """ from openpype.lib import Logger + from openpype.lib.applications import get_app_environments_for_context from openpype.modules import ModulesManager from openpype.pipeline import install_openpype_plugins from openpype.tools.utils.host_tools import show_publish @@ -199,9 +187,23 @@ class PypeCommands: """ import pyblish.api - from openpype.lib import ApplicationManager from openpype.lib import Logger + from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, + ) + from openpype.lib.plugin_tools import get_batch_asset_task_info + from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + fail_batch, + find_variant_key, + get_task_data, + get_timeout, + IN_PROGRESS_STATUS + ) + log = Logger.get_logger("CLI-remotepublishfromapp") log.info("remotepublishphotoshop command") @@ -318,9 +320,16 @@ class PypeCommands: import pyblish.api import pyblish.util - from openpype.lib import Logger from openpype.pipeline import install_host from openpype.hosts.webpublisher import api as webpublisher + from openpype.lib import Logger + from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + publish_and_log, + fail_batch, + get_task_data, + ) log = Logger.get_logger("remotepublish") @@ -366,8 +375,10 @@ class PypeCommands: Called by Deadline plugin to propagate environment into render jobs. """ + + from openpype.lib.applications import get_app_environments_for_context + if all((project, asset, task, app)): - from openpype.api import get_app_environments_for_context env = get_app_environments_for_context( project, asset, task, app, env_group ) From c2332507f49eb863aecd98540b9b71b421c2f1ec Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:41:56 +0200 Subject: [PATCH 03/17] implement webpublisher host with HostBase --- openpype/hosts/webpublisher/api/__init__.py | 36 ++++++++------------- openpype/pype_commands.py | 11 +++---- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 18e3a16cf5..afea838e2c 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -1,31 +1,23 @@ import os import logging -from pyblish import api as pyblish -import openpype.hosts.webpublisher -from openpype.pipeline import legacy_io +import pyblish.api + +from openpype.host import HostBase +from openpype.hosts.webpublisher import WEBPUBLISHER_ROOT_DIR log = logging.getLogger("openpype.hosts.webpublisher") -HOST_DIR = os.path.dirname(os.path.abspath( - openpype.hosts.webpublisher.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") -PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") +class WebpublisherHost(HostBase): + name = "webpublisher" -def install(): - print("Installing Pype config...") + def install(self): + print("Installing Pype config...") + pyblish.api.register_host(self.name) - pyblish.register_plugin_path(PUBLISH_PATH) - log.info(PUBLISH_PATH) - - legacy_io.install() - - -def uninstall(): - pyblish.deregister_plugin_path(PUBLISH_PATH) - - -# to have required methods for interface -def ls(): - pass + publish_plugin_dir = os.path.join( + WEBPUBLISHER_ROOT_DIR, "plugins", "publish" + ) + pyblish.api.register_plugin_path(publish_plugin_dir) + self.log.info(publish_plugin_dir) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index cb84fac3c7..6a65b78dfc 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -321,7 +321,7 @@ class PypeCommands: import pyblish.util from openpype.pipeline import install_host - from openpype.hosts.webpublisher import api as webpublisher + from openpype.hosts.webpublisher.api import WebpublisherHost from openpype.lib import Logger from openpype.lib.remote_publish import ( get_webpublish_conn, @@ -335,22 +335,21 @@ class PypeCommands: log.info("remotepublish command") - host_name = "webpublisher" + webpublisher_host = WebpublisherHost() + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host_name + os.environ["AVALON_APP"] = webpublisher_host.name os.environ["USER_EMAIL"] = user_email os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - pyblish.api.register_host(host_name) - if targets: if isinstance(targets, str): targets = [targets] for target in targets: pyblish.api.register_target(target) - install_host(webpublisher) + install_host(webpublisher_host) log.info("Running publish ...") From 61690d84774268b49904789f0ea5cd8e5171caf7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:54:42 +0200 Subject: [PATCH 04/17] omved cli functions into webpublisher --- openpype/hosts/webpublisher/addon.py | 50 +++++ openpype/hosts/webpublisher/cli_functions.py | 204 +++++++++++++++++++ openpype/pype_commands.py | 159 +-------------- 3 files changed, 259 insertions(+), 154 deletions(-) create mode 100644 openpype/hosts/webpublisher/cli_functions.py diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 3d76115df1..1a4370c9a5 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -1,4 +1,7 @@ import os + +import click + from openpype.modules import OpenPypeModule from openpype.modules.interfaces import IHostModule @@ -11,3 +14,50 @@ class WebpublisherAddon(OpenPypeModule, IHostModule): def initialize(self, module_settings): self.enabled = True + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group( + WebpublisherAddon.name, + help="Webpublisher related commands.") +def cli_main(): + pass + + +@cli_main.command() +@click.argument("path") +@click.option("-u", "--user", help="User email address") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets", default=None, + multiple=True) +def publish(project, path, user=None, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + + from .cli_functions import publish + + publish(project, path, user, targets) + + +@cli_main.command() +@click.argument("path") +@click.option("-h", "--host", help="Host") +@click.option("-u", "--user", help="User email address") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets", default=None, + multiple=True) +def publishfromapp(project, path, user=None, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + + from .cli_functions import publish_from_app + + publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py new file mode 100644 index 0000000000..cb2e59fac2 --- /dev/null +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -0,0 +1,204 @@ +import os +import time +import pyblish.api +import pyblish.util + +from openpype.lib import Logger +from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + publish_and_log, + fail_batch, + find_variant_key, + get_task_data, + get_timeout, + IN_PROGRESS_STATUS +) +from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, +) +from openpype.lib.plugin_tools import get_batch_asset_task_info +from openpype.pipeline import install_host +from openpype.hosts.webpublisher.api import WebpublisherHost + + +def publish(project_name, batch_path, user_email, targets): + """Start headless publishing. + + Used to publish rendered assets, workfiles etc via Webpublisher. + Eventually should be yanked out to Webpublisher cli. + + Publish use json from passed paths argument. + + Args: + project_name (str): project to publish (only single context is + expected per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) + user_email (string): email address for webpublisher - used to + find Ftrack user with same email + targets (list): Pyblish targets + (to choose validator for example) + + Raises: + RuntimeError: When there is no path to process. + """ + + if not batch_path: + raise RuntimeError("No publish paths specified") + + log = Logger.get_logger("remotepublish") + log.info("remotepublish command") + + # Register target and host + webpublisher_host = WebpublisherHost() + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + os.environ["AVALON_PROJECT"] = project_name + os.environ["AVALON_APP"] = webpublisher_host.name + os.environ["USER_EMAIL"] = user_email + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib + + if targets: + if isinstance(targets, str): + targets = [targets] + for target in targets: + pyblish.api.register_target(target) + + install_host(webpublisher_host) + + log.info("Running publish ...") + + _, batch_id = os.path.split(batch_path) + dbcon = get_webpublish_conn() + _id = start_webpublish_log(dbcon, batch_id, user_email) + + task_data = get_task_data(batch_path) + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + + publish_and_log(dbcon, _id, log, batch_id=batch_id) + + log.info("Publish finished.") + + +def publish_from_app( + project_name, batch_path, host_name, user_email, targets +): + """Opens installed variant of 'host' and run remote publish there. + + Eventually should be yanked out to Webpublisher cli. + + Currently implemented and tested for Photoshop where customer + wants to process uploaded .psd file and publish collected layers + from there. Triggered by Webpublisher. + + Checks if no other batches are running (status =='in_progress). If + so, it sleeps for SLEEP (this is separate process), + waits for WAIT_FOR seconds altogether. + + Requires installed host application on the machine. + + Runs publish process as user would, in automatic fashion. + + Args: + project_name (str): project to publish (only single context is + expected per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) + host_name (str): 'photoshop' + user_email (string): email address for webpublisher - used to + find Ftrack user with same email + targets (list): Pyblish targets + (to choose validator for example) + """ + + log = Logger.get_logger("RemotePublishFromApp") + + log.info("remotepublishphotoshop command") + + task_data = get_task_data(batch_path) + + workfile_path = os.path.join(batch_path, + task_data["task"], + task_data["files"][0]) + + print("workfile_path {}".format(workfile_path)) + + batch_id = task_data["batch"] + dbcon = get_webpublish_conn() + # safer to start logging here, launch might be broken altogether + _id = start_webpublish_log(dbcon, batch_id, user_email) + + batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS})) + if len(batches_in_progress) > 1: + running_batches = [str(batch["_id"]) + for batch in batches_in_progress + if batch["_id"] != _id] + msg = "There are still running batches {}\n". \ + format("\n".join(running_batches)) + msg += "Ask admin to check them and reprocess current batch" + fail_batch(_id, dbcon, msg) + + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + + asset_name, task_name, task_type = get_batch_asset_task_info( + task_data["context"]) + + application_manager = ApplicationManager() + found_variant_key = find_variant_key(application_manager, host_name) + app_name = "{}/{}".format(host_name, found_variant_key) + + # must have for proper launch of app + env = get_app_environments_for_context( + project_name, + asset_name, + task_name, + app_name + ) + print("env:: {}".format(env)) + os.environ.update(env) + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + # must pass identifier to update log lines for a batch + os.environ["BATCH_LOG_ID"] = str(_id) + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib + os.environ["USER_EMAIL"] = user_email + + pyblish.api.register_host(host_name) + if targets: + if isinstance(targets, str): + targets = [targets] + current_targets = os.environ.get("PYBLISH_TARGETS", "").split( + os.pathsep) + for target in targets: + current_targets.append(target) + + os.environ["PYBLISH_TARGETS"] = os.pathsep.join( + set(current_targets)) + + data = { + "last_workfile_path": workfile_path, + "start_last_workfile": True, + "project_name": project_name, + "asset_name": asset_name, + "task_name": task_name + } + + launched_app = application_manager.launch(app_name, **data) + + timeout = get_timeout(project_name, host_name, task_type) + + time_start = time.time() + while launched_app.poll() is None: + time.sleep(0.5) + if time.time() - time_start > timeout: + launched_app.terminate() + msg = "Timeout reached" + fail_batch(_id, dbcon, msg) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 6a65b78dfc..1817724df1 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -186,110 +186,11 @@ class PypeCommands: (to choose validator for example) """ - import pyblish.api + from openpype.hosts.webpublisher.cli_functions import publish_from_app - from openpype.lib import Logger - from openpype.lib.applications import ( - ApplicationManager, - get_app_environments_for_context, + publish_from_app( + project_name, batch_path, host_name, user_email, targets ) - from openpype.lib.plugin_tools import get_batch_asset_task_info - from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - fail_batch, - find_variant_key, - get_task_data, - get_timeout, - IN_PROGRESS_STATUS - ) - - log = Logger.get_logger("CLI-remotepublishfromapp") - - log.info("remotepublishphotoshop command") - - task_data = get_task_data(batch_path) - - workfile_path = os.path.join(batch_path, - task_data["task"], - task_data["files"][0]) - - print("workfile_path {}".format(workfile_path)) - - batch_id = task_data["batch"] - dbcon = get_webpublish_conn() - # safer to start logging here, launch might be broken altogether - _id = start_webpublish_log(dbcon, batch_id, user_email) - - batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS})) - if len(batches_in_progress) > 1: - running_batches = [str(batch["_id"]) - for batch in batches_in_progress - if batch["_id"] != _id] - msg = "There are still running batches {}\n". \ - format("\n".join(running_batches)) - msg += "Ask admin to check them and reprocess current batch" - fail_batch(_id, dbcon, msg) - - if not task_data["context"]: - msg = "Batch manifest must contain context data" - msg += "Create new batch and set context properly." - fail_batch(_id, dbcon, msg) - - asset_name, task_name, task_type = get_batch_asset_task_info( - task_data["context"]) - - application_manager = ApplicationManager() - found_variant_key = find_variant_key(application_manager, host_name) - app_name = "{}/{}".format(host_name, found_variant_key) - - # must have for proper launch of app - env = get_app_environments_for_context( - project_name, - asset_name, - task_name, - app_name - ) - print("env:: {}".format(env)) - os.environ.update(env) - - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - # must pass identifier to update log lines for a batch - os.environ["BATCH_LOG_ID"] = str(_id) - os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - os.environ["USER_EMAIL"] = user_email - - pyblish.api.register_host(host_name) - if targets: - if isinstance(targets, str): - targets = [targets] - current_targets = os.environ.get("PYBLISH_TARGETS", "").split( - os.pathsep) - for target in targets: - current_targets.append(target) - - os.environ["PYBLISH_TARGETS"] = os.pathsep.join( - set(current_targets)) - - data = { - "last_workfile_path": workfile_path, - "start_last_workfile": True, - "project_name": project_name, - "asset_name": asset_name, - "task_name": task_name - } - - launched_app = application_manager.launch(app_name, **data) - - timeout = get_timeout(project_name, host_name, task_type) - - time_start = time.time() - while launched_app.poll() is None: - time.sleep(0.5) - if time.time() - time_start > timeout: - launched_app.terminate() - msg = "Timeout reached" - fail_batch(_id, dbcon, msg) @staticmethod def remotepublish(project, batch_path, user_email, targets=None): @@ -313,59 +214,10 @@ class PypeCommands: Raises: RuntimeError: When there is no path to process. """ - if not batch_path: - raise RuntimeError("No publish paths specified") - # Register target and host - import pyblish.api - import pyblish.util + from openpype.hosts.webpublisher.cli_functions import publish - from openpype.pipeline import install_host - from openpype.hosts.webpublisher.api import WebpublisherHost - from openpype.lib import Logger - from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - publish_and_log, - fail_batch, - get_task_data, - ) - - log = Logger.get_logger("remotepublish") - - log.info("remotepublish command") - - webpublisher_host = WebpublisherHost() - - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = webpublisher_host.name - os.environ["USER_EMAIL"] = user_email - os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - - if targets: - if isinstance(targets, str): - targets = [targets] - for target in targets: - pyblish.api.register_target(target) - - install_host(webpublisher_host) - - log.info("Running publish ...") - - _, batch_id = os.path.split(batch_path) - dbcon = get_webpublish_conn() - _id = start_webpublish_log(dbcon, batch_id, user_email) - - task_data = get_task_data(batch_path) - if not task_data["context"]: - msg = "Batch manifest must contain context data" - msg += "Create new batch and set context properly." - fail_batch(_id, dbcon, msg) - - publish_and_log(dbcon, _id, log, batch_id=batch_id) - - log.info("Publish finished.") + publish(project, batch_path, user_email, targets) @staticmethod def extractenvironments(output_json_path, project, asset, task, app, @@ -479,7 +331,6 @@ class PypeCommands: sync_server_module.server_init() sync_server_module.server_start() - import time while True: time.sleep(1.0) From eed9789287adeb1ba000262544368344be353ff9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:59:20 +0200 Subject: [PATCH 05/17] changed function names --- openpype/hosts/webpublisher/addon.py | 8 ++++---- openpype/hosts/webpublisher/cli_functions.py | 4 ++-- openpype/pype_commands.py | 12 ++++++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 1a4370c9a5..9e63030fe2 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -39,9 +39,9 @@ def publish(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import publish + from .cli_functions import cli_publish - publish(project, path, user, targets) + cli_publish(project, path, user, targets) @cli_main.command() @@ -58,6 +58,6 @@ def publishfromapp(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import publish_from_app + from .cli_functions import cli_publish_from_app - publish_from_app(project, path, user, targets) + cli_publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py index cb2e59fac2..ad3bb596fb 100644 --- a/openpype/hosts/webpublisher/cli_functions.py +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -23,7 +23,7 @@ from openpype.pipeline import install_host from openpype.hosts.webpublisher.api import WebpublisherHost -def publish(project_name, batch_path, user_email, targets): +def cli_publish(project_name, batch_path, user_email, targets): """Start headless publishing. Used to publish rendered assets, workfiles etc via Webpublisher. @@ -85,7 +85,7 @@ def publish(project_name, batch_path, user_email, targets): log.info("Publish finished.") -def publish_from_app( +def cli_publish_from_app( project_name, batch_path, host_name, user_email, targets ): """Opens installed variant of 'host' and run remote publish there. diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 1817724df1..b6c1228ade 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -186,9 +186,11 @@ class PypeCommands: (to choose validator for example) """ - from openpype.hosts.webpublisher.cli_functions import publish_from_app + from openpype.hosts.webpublisher.cli_functions import ( + cli_publish_from_app + ) - publish_from_app( + cli_publish_from_app( project_name, batch_path, host_name, user_email, targets ) @@ -215,9 +217,11 @@ class PypeCommands: RuntimeError: When there is no path to process. """ - from openpype.hosts.webpublisher.cli_functions import publish + from openpype.hosts.webpublisher.cli_functions import ( + cli_publish + ) - publish(project, batch_path, user_email, targets) + cli_publish(project, batch_path, user_email, targets) @staticmethod def extractenvironments(output_json_path, project, asset, task, app, From dec6335ece01b37e6f0396807526ab16b5db1a6e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:41:18 +0200 Subject: [PATCH 06/17] move remote_publish funtion into pipeline publish --- openpype/pipeline/publish/lib.py | 40 ++++++++++++++++++++++++++++++ openpype/scripts/remote_publish.py | 7 +++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index d5494cd8a4..9060a0bf4b 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -273,3 +273,43 @@ def filter_pyblish_plugins(plugins): option, value, plugin.__name__)) setattr(plugin, option, value) + + +def find_close_plugin(close_plugin_name, log): + if close_plugin_name: + plugins = pyblish.api.discover() + for plugin in plugins: + if plugin.__name__ == close_plugin_name: + return plugin + + log.debug("Close plugin not found, app might not close.") + + +def remote_publish(log, close_plugin_name=None, raise_error=False): + """Loops through all plugins, logs to console. Used for tests. + + Args: + log (openpype.lib.Logger) + close_plugin_name (str): name of plugin with responsibility to + close host app + """ + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + close_plugin = find_close_plugin(close_plugin_name, log) + + for result in pyblish.util.publish_iter(): + for record in result["records"]: + log.info("{}: {}".format( + result["plugin"].label, record.msg)) + + if result["error"]: + error_message = error_format.format(**result) + log.error(error_message) + if close_plugin: # close host app explicitly after error + context = pyblish.api.Context() + close_plugin().process(context) + if raise_error: + # Fatal Error is because of Deadline + error_message = "Fatal Error: " + error_format.format(**result) + raise RuntimeError(error_message) diff --git a/openpype/scripts/remote_publish.py b/openpype/scripts/remote_publish.py index d322f369d1..37df35e36c 100644 --- a/openpype/scripts/remote_publish.py +++ b/openpype/scripts/remote_publish.py @@ -1,11 +1,12 @@ try: - from openpype.api import Logger - import openpype.lib.remote_publish + from openpype.lib import Logger + from openpype.pipeline.publish.lib import remote_publish except ImportError as exc: # Ensure Deadline fails by output an error that contains "Fatal Error:" raise ImportError("Fatal Error: %s" % exc) + if __name__ == "__main__": # Perform remote publish with thorough error checking log = Logger.get_logger(__name__) - openpype.lib.remote_publish.publish(log, raise_error=True) + remote_publish(log, raise_error=True) From c1a7b9aff5024c4df92493169e2741f044558b2d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:49:24 +0200 Subject: [PATCH 07/17] moved webpublisher specific functions into webpublisher --- openpype/hosts/webpublisher/lib.py | 278 +++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 openpype/hosts/webpublisher/lib.py diff --git a/openpype/hosts/webpublisher/lib.py b/openpype/hosts/webpublisher/lib.py new file mode 100644 index 0000000000..dde875c934 --- /dev/null +++ b/openpype/hosts/webpublisher/lib.py @@ -0,0 +1,278 @@ +import os +from datetime import datetime +import collections +import json + +from bson.objectid import ObjectId + +import pyblish.util +import pyblish.api + +from openpype.client.mongo import OpenPypeMongoConnection +from openpype.settings import get_project_settings +from openpype.lib import Logger +from openpype.lib.profiles_filtering import filter_profiles + +ERROR_STATUS = "error" +IN_PROGRESS_STATUS = "in_progress" +REPROCESS_STATUS = "reprocess" +SENT_REPROCESSING_STATUS = "sent_for_reprocessing" +FINISHED_REPROCESS_STATUS = "republishing_finished" +FINISHED_OK_STATUS = "finished_ok" + +log = Logger.get_logger(__name__) + + +def parse_json(path): + """Parses json file at 'path' location + + Returns: + (dict) or None if unparsable + Raises: + AsssertionError if 'path' doesn't exist + """ + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + log.error( + "Error loading json: {} - Exception: {}".format(path, exc) + ) + return data + + +def get_batch_asset_task_info(ctx): + """Parses context data from webpublisher's batch metadata + + Returns: + (tuple): asset, task_name (Optional), task_type + """ + task_type = "default_task_type" + task_name = None + asset = None + + if ctx["type"] == "task": + items = ctx["path"].split('/') + asset = items[-2] + task_name = ctx["name"] + task_type = ctx["attributes"]["type"] + else: + asset = ctx["name"] + + return asset, task_name, task_type + + +def get_webpublish_conn(): + """Get connection to OP 'webpublishes' collection.""" + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + return mongo_client[database_name]["webpublishes"] + + +def start_webpublish_log(dbcon, batch_id, user): + """Start new log record for 'batch_id' + + Args: + dbcon (OpenPypeMongoConnection) + batch_id (str) + user (str) + Returns + (ObjectId) from DB + """ + return dbcon.insert_one({ + "batch_id": batch_id, + "start_date": datetime.now(), + "user": user, + "status": IN_PROGRESS_STATUS, + "progress": 0 # integer 0-100, percentage + }).inserted_id + + +def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): + """Loops through all plugins, logs ok and fails into OP DB. + + Args: + dbcon (OpenPypeMongoConnection) + _id (str) - id of current job in DB + log (openpype.lib.Logger) + batch_id (str) - id sent from frontend + close_plugin_name (str): name of plugin with responsibility to + close host app + """ + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" + error_format += "-" * 80 + "\n" + + close_plugin = _get_close_plugin(close_plugin_name, log) + + if isinstance(_id, str): + _id = ObjectId(_id) + + log_lines = [] + processed = 0 + log_every = 5 + for result in pyblish.util.publish_iter(): + for record in result["records"]: + log_lines.append("{}: {}".format( + result["plugin"].label, record.msg)) + processed += 1 + + if result["error"]: + log.error(error_format.format(**result)) + log_lines = [error_format.format(**result)] + log_lines + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": ERROR_STATUS, + "log": os.linesep.join(log_lines) + + }} + ) + if close_plugin: # close host app explicitly after error + context = pyblish.api.Context() + close_plugin().process(context) + return + elif processed % log_every == 0: + # pyblish returns progress in 0.0 - 2.0 + progress = min(round(result["progress"] / 2 * 100), 99) + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "progress": progress, + "log": os.linesep.join(log_lines) + }} + ) + + # final update + if batch_id: + dbcon.update_many( + {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS}, + { + "$set": + { + "finish_date": datetime.now(), + "status": FINISHED_REPROCESS_STATUS, + } + } + ) + + dbcon.update_one( + {"_id": _id}, + { + "$set": + { + "finish_date": datetime.now(), + "status": FINISHED_OK_STATUS, + "progress": 100, + "log": os.linesep.join(log_lines) + } + } + ) + + +def fail_batch(_id, dbcon, msg): + """Set current batch as failed as there is some problem. + + Raises: + ValueError + """ + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": ERROR_STATUS, + "log": msg + + }} + ) + raise ValueError(msg) + + +def find_variant_key(application_manager, host): + """Searches for latest installed variant for 'host' + + Args: + application_manager (ApplicationManager) + host (str) + Returns + (string) (optional) + Raises: + (ValueError) if no variant found + """ + app_group = application_manager.app_groups.get(host) + if not app_group or not app_group.enabled: + raise ValueError("No application {} configured".format(host)) + + found_variant_key = None + # finds most up-to-date variant if any installed + sorted_variants = collections.OrderedDict( + sorted(app_group.variants.items())) + for variant_key, variant in sorted_variants.items(): + for executable in variant.executables: + if executable.exists(): + found_variant_key = variant_key + + if not found_variant_key: + raise ValueError("No executable for {} found".format(host)) + + return found_variant_key + + +def _get_close_plugin(close_plugin_name, log): + if close_plugin_name: + plugins = pyblish.api.discover() + for plugin in plugins: + if plugin.__name__ == close_plugin_name: + return plugin + + log.debug("Close plugin not found, app might not close.") + + +def get_task_data(batch_dir): + """Return parsed data from first task manifest.json + + Used for `remotepublishfromapp` command where batch contains only + single task with publishable workfile. + + Returns: + (dict) + Throws: + (ValueError) if batch or task manifest not found or broken + """ + batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) + if not batch_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(batch_dir)) + task_dir_name = batch_data["tasks"][0] + task_data = parse_json(os.path.join(batch_dir, task_dir_name, + "manifest.json")) + if not task_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(task_data)) + + return task_data + + +def get_timeout(project_name, host_name, task_type): + """Returns timeout(seconds) from Setting profile.""" + filter_data = { + "task_types": task_type, + "hosts": host_name + } + timeout_profiles = (get_project_settings(project_name)["webpublisher"] + ["timeout_profiles"]) + matching_item = filter_profiles(timeout_profiles, filter_data) + timeout = 3600 + if matching_item: + timeout = matching_item["timeout"] + + return timeout From 6c330c48969bffba508a83aa0d98cf93d804142f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:50:54 +0200 Subject: [PATCH 08/17] use lib functions from webpublisher --- .../plugins/publish/collect_batch_data.py | 8 ++++---- openpype/hosts/webpublisher/cli_functions.py | 17 +++++++++-------- .../plugins/publish/collect_batch_data.py | 11 ++++++----- .../plugins/publish/collect_published_files.py | 6 ++---- .../publish/collect_tvpaint_workfile_data.py | 2 +- .../webserver_service/webpublish_routes.py | 12 +++++------- .../webserver_service/webserver_cli.py | 14 +++++++------- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py index 2881ef0ea6..5d50a78914 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py @@ -17,11 +17,11 @@ import os import pyblish.api -from openpype.lib.plugin_tools import ( - parse_json, - get_batch_asset_task_info -) from openpype.pipeline import legacy_io +from openpype_modules.webpublisher.lib import ( + get_batch_asset_task_info, + parse_json +) class CollectBatchData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py index ad3bb596fb..83f53ced68 100644 --- a/openpype/hosts/webpublisher/cli_functions.py +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -4,7 +4,15 @@ import pyblish.api import pyblish.util from openpype.lib import Logger -from openpype.lib.remote_publish import ( +from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, +) +from openpype.pipeline import install_host +from openpype.hosts.webpublisher.api import WebpublisherHost + +from .lib import ( + get_batch_asset_task_info, get_webpublish_conn, start_webpublish_log, publish_and_log, @@ -14,13 +22,6 @@ from openpype.lib.remote_publish import ( get_timeout, IN_PROGRESS_STATUS ) -from openpype.lib.applications import ( - ApplicationManager, - get_app_environments_for_context, -) -from openpype.lib.plugin_tools import get_batch_asset_task_info -from openpype.pipeline import install_host -from openpype.hosts.webpublisher.api import WebpublisherHost def cli_publish(project_name, batch_path, user_email, targets): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py index 9ff779636a..eb2737b276 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py @@ -13,12 +13,13 @@ import os import pyblish.api -from openpype.lib.plugin_tools import ( - parse_json, - get_batch_asset_task_info -) -from openpype.lib.remote_publish import get_webpublish_conn, IN_PROGRESS_STATUS from openpype.pipeline import legacy_io +from openpype_modules.webpublisher.lib import ( + parse_json, + get_batch_asset_task_info, + get_webpublish_conn, + IN_PROGRESS_STATUS +) class CollectBatchData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 20e277d794..454f78ce9d 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -23,10 +23,8 @@ from openpype.lib import ( get_ffprobe_streams, convert_ffprobe_fps_value, ) -from openpype.lib.plugin_tools import ( - parse_json, - get_subset_name_with_asset_doc -) +from openpype.lib.plugin_tools import get_subset_name_with_asset_doc +from openpype_modules.webpublisher.lib import parse_json class CollectPublishedFiles(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py index f0f29260a2..b5f8ed9c8f 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py @@ -16,11 +16,11 @@ import uuid import json import shutil import pyblish.api -from openpype.lib.plugin_tools import parse_json from openpype.hosts.tvpaint.worker import ( SenderTVPaintCommands, CollectSceneData ) +from openpype_modules.webpublisher.lib import parse_json class CollectTVPaintWorkfileData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 2e9d460a98..e3de555ace 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -10,16 +10,16 @@ from aiohttp.web_response import Response from openpype.client import ( get_projects, get_assets, - OpenPypeMongoConnection, ) from openpype.lib import Logger -from openpype.lib.remote_publish import ( +from openpype.settings import get_project_settings +from openpype_modules.webserver.base_routes import RestApiEndpoint +from openpype_modules.webpublisher.lib import ( + get_webpublish_conn, get_task_data, ERROR_STATUS, REPROCESS_STATUS ) -from openpype.settings import get_project_settings -from openpype_modules.webserver.base_routes import RestApiEndpoint log = Logger.get_logger("WebpublishRoutes") @@ -77,9 +77,7 @@ class WebpublishRestApiResource(JsonApiResource): """Resource carrying OP DB connection for storing batch info into DB.""" def __init__(self): - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - self.dbcon = mongo_client[database_name]["webpublishes"] + self.dbcon = get_webpublish_conn() class ProjectsEndpoint(ResourceRestApiEndpoint): diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 936bd9735f..47c792a575 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -7,8 +7,15 @@ import json import subprocess from openpype.client import OpenPypeMongoConnection +from openpype.modules import ModulesManager from openpype.lib import Logger +from openpype_modules.webpublisher.lib import ( + ERROR_STATUS, + REPROCESS_STATUS, + SENT_REPROCESSING_STATUS +) + from .webpublish_routes import ( RestApiResource, WebpublishRestApiResource, @@ -21,19 +28,12 @@ from .webpublish_routes import ( TaskPublishEndpoint, UserReportEndpoint ) -from openpype.lib.remote_publish import ( - ERROR_STATUS, - REPROCESS_STATUS, - SENT_REPROCESSING_STATUS -) - log = Logger.get_logger("webserver_gui") def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" - from openpype.modules import ModulesManager manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] From 233d70bdd8d21062842aa88b15841ac1fb61f0a4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:54:05 +0200 Subject: [PATCH 09/17] headless_publish is a method on webpublisher addon --- openpype/hosts/aftereffects/api/lib.py | 19 ++++++++++++++----- openpype/hosts/photoshop/api/lib.py | 9 +++++---- openpype/hosts/webpublisher/addon.py | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/aftereffects/api/lib.py b/openpype/hosts/aftereffects/api/lib.py index ce4cbf09af..d5583ee862 100644 --- a/openpype/hosts/aftereffects/api/lib.py +++ b/openpype/hosts/aftereffects/api/lib.py @@ -3,11 +3,12 @@ import sys import contextlib import traceback import logging +from functools import partial from Qt import QtWidgets from openpype.pipeline import install_host -from openpype.lib.remote_publish import headless_publish +from openpype.modules import ModulesManager from openpype.tools.utils import host_tools from .launch_logic import ProcessLauncher, get_stub @@ -35,10 +36,18 @@ def main(*subprocess_args): launcher.start() if os.environ.get("HEADLESS_PUBLISH"): - launcher.execute_in_main_thread(lambda: headless_publish( - log, - "CloseAE", - os.environ.get("IS_TEST"))) + manager = ModulesManager() + webpublisher_addon = manager["webpublisher"] + + launcher.execute_in_main_thread( + partial( + webpublisher_addon.headless_publish, + log, + "CloseAE", + os.environ.get("IS_TEST") + ) + ) + elif os.environ.get("AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH", True): save = False if os.getenv("WORKFILES_SAVE_AS"): diff --git a/openpype/hosts/photoshop/api/lib.py b/openpype/hosts/photoshop/api/lib.py index 2f57d64464..73a546604f 100644 --- a/openpype/hosts/photoshop/api/lib.py +++ b/openpype/hosts/photoshop/api/lib.py @@ -5,11 +5,10 @@ import traceback from Qt import QtWidgets -from openpype.api import Logger +from openpype.lib import env_value_to_bool, Logger +from openpype.modules import ModulesManager from openpype.pipeline import install_host from openpype.tools.utils import host_tools -from openpype.lib.remote_publish import headless_publish -from openpype.lib import env_value_to_bool from .launch_logic import ProcessLauncher, stub @@ -35,8 +34,10 @@ def main(*subprocess_args): launcher.start() if env_value_to_bool("HEADLESS_PUBLISH"): + manager = ModulesManager() + webpublisher_addon = manager["webpublisher"] launcher.execute_in_main_thread( - headless_publish, + webpublisher_addon.headless_publish, log, "ClosePS", os.environ.get("IS_TEST") diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 9e63030fe2..0bba8adc4b 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -15,6 +15,30 @@ class WebpublisherAddon(OpenPypeModule, IHostModule): def initialize(self, module_settings): self.enabled = True + def headless_publish(self, log, close_plugin_name=None, is_test=False): + """Runs publish in a opened host with a context. + + Close Python process at the end. + """ + + from openpype.pipeline.publish.lib import remote_publish + from .lib import get_webpublish_conn, publish_and_log + + if is_test: + remote_publish(log, close_plugin_name) + return + + dbcon = get_webpublish_conn() + _id = os.environ.get("BATCH_LOG_ID") + if not _id: + log.warning("Unable to store log records, " + "batch will be unfinished!") + return + + publish_and_log( + dbcon, _id, log, close_plugin_name=close_plugin_name + ) + def cli(self, click_group): click_group.add_command(cli_main) From a98c7953aaf4fecf465b7e9b95357de2056e3018 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:54:30 +0200 Subject: [PATCH 10/17] use 'find_close_plugin' --- openpype/hosts/webpublisher/lib.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/webpublisher/lib.py b/openpype/hosts/webpublisher/lib.py index dde875c934..4bc3f1db80 100644 --- a/openpype/hosts/webpublisher/lib.py +++ b/openpype/hosts/webpublisher/lib.py @@ -12,6 +12,7 @@ from openpype.client.mongo import OpenPypeMongoConnection from openpype.settings import get_project_settings from openpype.lib import Logger from openpype.lib.profiles_filtering import filter_profiles +from openpype.pipeline.publish.lib import find_close_plugin ERROR_STATUS = "error" IN_PROGRESS_STATUS = "in_progress" @@ -108,7 +109,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" error_format += "-" * 80 + "\n" - close_plugin = _get_close_plugin(close_plugin_name, log) + close_plugin = find_close_plugin(close_plugin_name, log) if isinstance(_id, str): _id = ObjectId(_id) @@ -227,16 +228,6 @@ def find_variant_key(application_manager, host): return found_variant_key -def _get_close_plugin(close_plugin_name, log): - if close_plugin_name: - plugins = pyblish.api.discover() - for plugin in plugins: - if plugin.__name__ == close_plugin_name: - return plugin - - log.debug("Close plugin not found, app might not close.") - - def get_task_data(batch_dir): """Return parsed data from first task manifest.json From 9c3e37e3f4ab83e465b71d908bcec439df011385 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:55:04 +0200 Subject: [PATCH 11/17] removed unused functions from openpype lib --- openpype/lib/plugin_tools.py | 45 ------ openpype/lib/remote_publish.py | 277 --------------------------------- 2 files changed, 322 deletions(-) delete mode 100644 openpype/lib/remote_publish.py diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 060db94ae0..659210e6e3 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -427,48 +427,3 @@ def get_background_layers(file_url): layer.get("filename")). replace("\\", "/")) return layers - - -def parse_json(path): - """Parses json file at 'path' location - - Returns: - (dict) or None if unparsable - Raises: - AsssertionError if 'path' doesn't exist - """ - path = path.strip('\"') - assert os.path.isfile(path), ( - "Path to json file doesn't exist. \"{}\"".format(path) - ) - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data - - -def get_batch_asset_task_info(ctx): - """Parses context data from webpublisher's batch metadata - - Returns: - (tuple): asset, task_name (Optional), task_type - """ - task_type = "default_task_type" - task_name = None - asset = None - - if ctx["type"] == "task": - items = ctx["path"].split('/') - asset = items[-2] - task_name = ctx["name"] - task_type = ctx["attributes"]["type"] - else: - asset = ctx["name"] - - return asset, task_name, task_type diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py deleted file mode 100644 index 2a901544cc..0000000000 --- a/openpype/lib/remote_publish.py +++ /dev/null @@ -1,277 +0,0 @@ -import os -from datetime import datetime -import collections - -from bson.objectid import ObjectId - -import pyblish.util -import pyblish.api - -from openpype.client.mongo import OpenPypeMongoConnection -from openpype.lib.plugin_tools import parse_json -from openpype.lib.profiles_filtering import filter_profiles -from openpype.api import get_project_settings - -ERROR_STATUS = "error" -IN_PROGRESS_STATUS = "in_progress" -REPROCESS_STATUS = "reprocess" -SENT_REPROCESSING_STATUS = "sent_for_reprocessing" -FINISHED_REPROCESS_STATUS = "republishing_finished" -FINISHED_OK_STATUS = "finished_ok" - - -def headless_publish(log, close_plugin_name=None, is_test=False): - """Runs publish in a opened host with a context and closes Python process. - """ - if not is_test: - dbcon = get_webpublish_conn() - _id = os.environ.get("BATCH_LOG_ID") - if not _id: - log.warning("Unable to store log records, " - "batch will be unfinished!") - return - - publish_and_log(dbcon, _id, log, close_plugin_name=close_plugin_name) - else: - publish(log, close_plugin_name) - - -def get_webpublish_conn(): - """Get connection to OP 'webpublishes' collection.""" - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - return mongo_client[database_name]["webpublishes"] - - -def start_webpublish_log(dbcon, batch_id, user): - """Start new log record for 'batch_id' - - Args: - dbcon (OpenPypeMongoConnection) - batch_id (str) - user (str) - Returns - (ObjectId) from DB - """ - return dbcon.insert_one({ - "batch_id": batch_id, - "start_date": datetime.now(), - "user": user, - "status": IN_PROGRESS_STATUS, - "progress": 0 # integer 0-100, percentage - }).inserted_id - - -def publish(log, close_plugin_name=None, raise_error=False): - """Loops through all plugins, logs to console. Used for tests. - - Args: - log (openpype.lib.Logger) - close_plugin_name (str): name of plugin with responsibility to - close host app - """ - # Error exit as soon as any error occurs. - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" - - close_plugin = _get_close_plugin(close_plugin_name, log) - - for result in pyblish.util.publish_iter(): - for record in result["records"]: - log.info("{}: {}".format( - result["plugin"].label, record.msg)) - - if result["error"]: - error_message = error_format.format(**result) - log.error(error_message) - if close_plugin: # close host app explicitly after error - context = pyblish.api.Context() - close_plugin().process(context) - if raise_error: - # Fatal Error is because of Deadline - error_message = "Fatal Error: " + error_format.format(**result) - raise RuntimeError(error_message) - - -def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): - """Loops through all plugins, logs ok and fails into OP DB. - - Args: - dbcon (OpenPypeMongoConnection) - _id (str) - id of current job in DB - log (openpype.lib.Logger) - batch_id (str) - id sent from frontend - close_plugin_name (str): name of plugin with responsibility to - close host app - """ - # Error exit as soon as any error occurs. - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" - error_format += "-" * 80 + "\n" - - close_plugin = _get_close_plugin(close_plugin_name, log) - - if isinstance(_id, str): - _id = ObjectId(_id) - - log_lines = [] - processed = 0 - log_every = 5 - for result in pyblish.util.publish_iter(): - for record in result["records"]: - log_lines.append("{}: {}".format( - result["plugin"].label, record.msg)) - processed += 1 - - if result["error"]: - log.error(error_format.format(**result)) - log_lines = [error_format.format(**result)] + log_lines - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": ERROR_STATUS, - "log": os.linesep.join(log_lines) - - }} - ) - if close_plugin: # close host app explicitly after error - context = pyblish.api.Context() - close_plugin().process(context) - return - elif processed % log_every == 0: - # pyblish returns progress in 0.0 - 2.0 - progress = min(round(result["progress"] / 2 * 100), 99) - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "progress": progress, - "log": os.linesep.join(log_lines) - }} - ) - - # final update - if batch_id: - dbcon.update_many( - {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS}, - { - "$set": - { - "finish_date": datetime.now(), - "status": FINISHED_REPROCESS_STATUS, - } - } - ) - - dbcon.update_one( - {"_id": _id}, - { - "$set": - { - "finish_date": datetime.now(), - "status": FINISHED_OK_STATUS, - "progress": 100, - "log": os.linesep.join(log_lines) - } - } - ) - - -def fail_batch(_id, dbcon, msg): - """Set current batch as failed as there is some problem. - - Raises: - ValueError - """ - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": ERROR_STATUS, - "log": msg - - }} - ) - raise ValueError(msg) - - -def find_variant_key(application_manager, host): - """Searches for latest installed variant for 'host' - - Args: - application_manager (ApplicationManager) - host (str) - Returns - (string) (optional) - Raises: - (ValueError) if no variant found - """ - app_group = application_manager.app_groups.get(host) - if not app_group or not app_group.enabled: - raise ValueError("No application {} configured".format(host)) - - found_variant_key = None - # finds most up-to-date variant if any installed - sorted_variants = collections.OrderedDict( - sorted(app_group.variants.items())) - for variant_key, variant in sorted_variants.items(): - for executable in variant.executables: - if executable.exists(): - found_variant_key = variant_key - - if not found_variant_key: - raise ValueError("No executable for {} found".format(host)) - - return found_variant_key - - -def _get_close_plugin(close_plugin_name, log): - if close_plugin_name: - plugins = pyblish.api.discover() - for plugin in plugins: - if plugin.__name__ == close_plugin_name: - return plugin - - log.debug("Close plugin not found, app might not close.") - - -def get_task_data(batch_dir): - """Return parsed data from first task manifest.json - - Used for `remotepublishfromapp` command where batch contains only - single task with publishable workfile. - - Returns: - (dict) - Throws: - (ValueError) if batch or task manifest not found or broken - """ - batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) - if not batch_data: - raise ValueError( - "Cannot parse batch meta in {} folder".format(batch_dir)) - task_dir_name = batch_data["tasks"][0] - task_data = parse_json(os.path.join(batch_dir, task_dir_name, - "manifest.json")) - if not task_data: - raise ValueError( - "Cannot parse batch meta in {} folder".format(task_data)) - - return task_data - - -def get_timeout(project_name, host_name, task_type): - """Returns timeout(seconds) from Setting profile.""" - filter_data = { - "task_types": task_type, - "hosts": host_name - } - timeout_profiles = (get_project_settings(project_name)["webpublisher"] - ["timeout_profiles"]) - matching_item = filter_profiles(timeout_profiles, filter_data) - timeout = 3600 - if matching_item: - timeout = matching_item["timeout"] - - return timeout From d5f6ad9fdc1727cd3c631698ae258ed8485ca479 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:58:09 +0200 Subject: [PATCH 12/17] renamed 'cli_functions' to 'publish_functions' --- openpype/hosts/webpublisher/addon.py | 4 ++-- .../webpublisher/{cli_functions.py => publish_functions.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename openpype/hosts/webpublisher/{cli_functions.py => publish_functions.py} (100%) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 0bba8adc4b..cb639db3fa 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -63,7 +63,7 @@ def publish(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import cli_publish + from .publish_functions import cli_publish cli_publish(project, path, user, targets) @@ -82,6 +82,6 @@ def publishfromapp(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import cli_publish_from_app + from .publish_functions import cli_publish_from_app cli_publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/publish_functions.py similarity index 100% rename from openpype/hosts/webpublisher/cli_functions.py rename to openpype/hosts/webpublisher/publish_functions.py From 338d12e60cd7f2fe9a15efecac07ce7ae8449d57 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:03:09 +0200 Subject: [PATCH 13/17] added cli command for webserver --- openpype/hosts/webpublisher/addon.py | 19 +++++++++++++++++++ .../webserver_service/__init__.py | 6 ++++++ .../webserver_service/webserver_cli.py | 16 ++++++++++------ 3 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 openpype/hosts/webpublisher/webserver_service/__init__.py diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index cb639db3fa..85e16de4a6 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -85,3 +85,22 @@ def publishfromapp(project, path, user=None, targets=None): from .publish_functions import cli_publish_from_app cli_publish_from_app(project, path, user, targets) + + +@cli_main.command() +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) +@click.option("-e", "--executable", help="Executable") +@click.option("-u", "--upload_dir", help="Upload dir") +def webserver(executable, upload_dir, host=None, port=None): + """Starts webserver for communication with Webpublish FR via command line + + OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND + FTRACK_BOT_API_KEY provided with api key from Ftrack. + + Expect "pype.club" user created on Ftrack. + """ + + from .webserver_service import run_webserver + + run_webserver(executable, upload_dir, host, port) diff --git a/openpype/hosts/webpublisher/webserver_service/__init__.py b/openpype/hosts/webpublisher/webserver_service/__init__.py new file mode 100644 index 0000000000..e43f3f063a --- /dev/null +++ b/openpype/hosts/webpublisher/webserver_service/__init__.py @@ -0,0 +1,6 @@ +from .webserver_cli import run_webserver + + +__all__ = ( + "run_webserver", +) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 47c792a575..093b53d9d3 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -32,21 +32,25 @@ from .webpublish_routes import ( log = Logger.get_logger("webserver_gui") -def run_webserver(*args, **kwargs): +def run_webserver(executable, upload_dir, host=None, port=None): """Runs webserver in command line, adds routes.""" + if not host: + host = "localhost" + if not port: + port = 8079 + manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = kwargs.get("host") or "localhost" - port = kwargs.get("port") or 8079 + server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url # queue for remotepublishfromapp tasks studio_task_queue = collections.deque() resource = RestApiResource(server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"], + upload_dir=upload_dir, + executable=executable, studio_task_queue=studio_task_queue) projects_endpoint = ProjectsEndpoint(resource) server_manager.add_route( @@ -111,7 +115,7 @@ def run_webserver(*args, **kwargs): last_reprocessed = time.time() while True: if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"], webserver_url) + reprocess_failed(upload_dir, webserver_url) last_reprocessed = time.time() if studio_task_queue: args = studio_task_queue.popleft() From e2c83c142684ccf0f400ffd345ddcf530bf49ed7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:03:40 +0200 Subject: [PATCH 14/17] renamed webserver_cli.py into webserver.py --- openpype/hosts/webpublisher/webserver_service/__init__.py | 2 +- .../webserver_service/{webserver_cli.py => webserver.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename openpype/hosts/webpublisher/webserver_service/{webserver_cli.py => webserver.py} (100%) diff --git a/openpype/hosts/webpublisher/webserver_service/__init__.py b/openpype/hosts/webpublisher/webserver_service/__init__.py index e43f3f063a..73111d286e 100644 --- a/openpype/hosts/webpublisher/webserver_service/__init__.py +++ b/openpype/hosts/webpublisher/webserver_service/__init__.py @@ -1,4 +1,4 @@ -from .webserver_cli import run_webserver +from .webserver import run_webserver __all__ = ( diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver.py similarity index 100% rename from openpype/hosts/webpublisher/webserver_service/webserver_cli.py rename to openpype/hosts/webpublisher/webserver_service/webserver.py From 971ae6d1ed0ad07713524dd4e7066e100fd22b34 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:04:37 +0200 Subject: [PATCH 15/17] fix import in global commands --- openpype/pype_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index b6c1228ade..fe46a4bc54 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -61,8 +61,8 @@ class PypeCommands: @staticmethod def launch_webpublisher_webservercli(*args, **kwargs): - from openpype.hosts.webpublisher.webserver_service.webserver_cli \ - import (run_webserver) + from openpype.hosts.webpublisher.webserver_service import run_webserver + return run_webserver(*args, **kwargs) @staticmethod From ae11ae16d5fd0b9f1cdb86263a25ace48bbf9b04 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:12:20 +0200 Subject: [PATCH 16/17] modify launch arguments --- .../webserver_service/webpublish_routes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index e3de555ace..4039d2c8ec 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -14,6 +14,7 @@ from openpype.client import ( from openpype.lib import Logger from openpype.settings import get_project_settings from openpype_modules.webserver.base_routes import RestApiEndpoint +from openpype_modules.webpublisher import WebpublisherAddon from openpype_modules.webpublisher.lib import ( get_webpublish_conn, get_task_data, @@ -213,7 +214,7 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): # TVPaint filter { "extensions": [".tvpp"], - "command": "remotepublish", + "command": "publish", "arguments": { "targets": ["tvpaint_worker"] }, @@ -222,13 +223,13 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): # Photoshop filter { "extensions": [".psd", ".psb"], - "command": "remotepublishfromapp", + "command": "publishfromapp", "arguments": { - # Command 'remotepublishfromapp' requires --host argument + # Command 'publishfromapp' requires --host argument "host": "photoshop", # Make sure targets are set to None for cases that default # would change - # - targets argument is not used in 'remotepublishfromapp' + # - targets argument is not used in 'publishfromapp' "targets": ["remotepublish"] }, # does publish need to be handled by a queue, eg. only @@ -240,7 +241,7 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): batch_dir = os.path.join(self.resource.upload_dir, content["batch"]) # Default command and arguments - command = "remotepublish" + command = "publish" add_args = { # All commands need 'project' and 'user' "project": content["project_name"], @@ -271,6 +272,8 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): args = [ openpype_app, + "module", + WebpublisherAddon.name, command, batch_dir ] From 45c112eb84ae741d4b102ea89ac5c64c01f591f5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 26 Aug 2022 10:12:35 +0200 Subject: [PATCH 17/17] fixed arguments --- openpype/hosts/webpublisher/addon.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 85e16de4a6..7d26d5a7ff 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -57,7 +57,7 @@ def cli_main(): @click.option("-t", "--targets", help="Targets", default=None, multiple=True) def publish(project, path, user=None, targets=None): - """Start CLI publishing. + """Start publishing (Inner command). Publish collects json from paths provided as an argument. More than one path is allowed. @@ -70,13 +70,13 @@ def publish(project, path, user=None, targets=None): @cli_main.command() @click.argument("path") +@click.option("-p", "--project", help="Project") @click.option("-h", "--host", help="Host") @click.option("-u", "--user", help="User email address") -@click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets", default=None, multiple=True) -def publishfromapp(project, path, user=None, targets=None): - """Start CLI publishing. +def publishfromapp(project, path, host, user=None, targets=None): + """Start publishing through application (Inner command). Publish collects json from paths provided as an argument. More than one path is allowed. @@ -84,16 +84,16 @@ def publishfromapp(project, path, user=None, targets=None): from .publish_functions import cli_publish_from_app - cli_publish_from_app(project, path, user, targets) + cli_publish_from_app(project, path, host, user, targets) @cli_main.command() -@click.option("-h", "--host", help="Host", default=None) -@click.option("-p", "--port", help="Port", default=None) @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) def webserver(executable, upload_dir, host=None, port=None): - """Starts webserver for communication with Webpublish FR via command line + """Start service for communication with Webpublish Front end. OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND FTRACK_BOT_API_KEY provided with api key from Ftrack.