From cc59f9bd3efc20c0cc946972164f4b88d2c1a5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LORRAIN?= Date: Fri, 5 Nov 2021 18:00:26 +0100 Subject: [PATCH 01/25] Add following workfile versioning option on publish --- .../publish/collect_anatomy_instance_data.py | 7 ++++++- openpype/plugins/publish/validate_version.py | 8 ++++---- .../settings/defaults/project_settings/global.json | 3 +++ .../schemas/schema_global_publish.json | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/openpype/plugins/publish/collect_anatomy_instance_data.py b/openpype/plugins/publish/collect_anatomy_instance_data.py index 4fd657167c..e0eb1618b5 100644 --- a/openpype/plugins/publish/collect_anatomy_instance_data.py +++ b/openpype/plugins/publish/collect_anatomy_instance_data.py @@ -38,6 +38,8 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + 0.49 label = "Collect Anatomy Instance data" + follow_workfile_version = False + def process(self, context): self.log.info("Collecting anatomy data for all instances.") @@ -213,7 +215,10 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): context_asset_doc = context.data["assetEntity"] for instance in context: - version_number = instance.data.get("version") + if self.follow_workfile_version: + version_number = context.data('version') + else: + version_number = instance.data.get("version") # If version is not specified for instance or context if version_number is None: # TODO we should be able to change default version by studio diff --git a/openpype/plugins/publish/validate_version.py b/openpype/plugins/publish/validate_version.py index 927e024476..b86d72a658 100644 --- a/openpype/plugins/publish/validate_version.py +++ b/openpype/plugins/publish/validate_version.py @@ -21,8 +21,8 @@ class ValidateVersion(pyblish.api.InstancePlugin): if latest_version is not None: msg = ( - "Version `{0}` that you are trying to publish, already exists" - " in the database. Version in database: `{1}`. Please version " - "up your workfile to a higher version number than: `{1}`." - ).format(version, latest_version) + "Version `{0}` from instance `{1}` that you are trying to publish, already exists" + " in the database. Version in database: `{2}`. Please version " + "up your workfile to a higher version number than: `{2}`." + ).format(version, instance.data["name"], latest_version) assert (int(version) > int(latest_version)), msg diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 45c1a59d17..134435d909 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -1,5 +1,8 @@ { "publish": { + "CollectAnatomyInstanceData": { + "follow_workfile_version": false + }, "ValidateEditorialAssetName": { "enabled": true, "optional": false diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index c50f383f02..375f0c26da 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -4,6 +4,20 @@ "key": "publish", "label": "Publish plugins", "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectAnatomyInstanceData", + "label": "Collect Anatomy Instance Data", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "follow_workfile_version", + "label": "Follow workfile version" + } + ] + }, { "type": "dict", "collapsible": true, From f8e594be800010b4e535de9e051a93982365f369 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:55:11 +0100 Subject: [PATCH 02/25] added module command --- openpype/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index 512bd4663b..e1f9265c1f 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -57,6 +57,12 @@ def tray(debug=False): PypeCommands().launch_tray(debug) +@main.group(help="Run command line arguments of OpenPype modules") +@click.pass_context +def module(ctx): + pass + + @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("--ftrack-url", envvar="FTRACK_SERVER", From dee8156a0c2263ac4d75fba9a957ebfd582dff14 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:56:17 +0100 Subject: [PATCH 03/25] implemented 'add_modules' which will call cli method on module to give ability to register custom cli commands --- openpype/cli.py | 1 + openpype/pype_commands.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index e1f9265c1f..d68cba45c6 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -57,6 +57,7 @@ def tray(debug=False): PypeCommands().launch_tray(debug) +@PypeCommands.add_modules @main.group(help="Run command line arguments of OpenPype modules") @click.pass_context def module(ctx): diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5fac5cacc7..f4a29091d0 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -41,6 +41,17 @@ class PypeCommands: user_role = "manager" settings.main(user_role) + @staticmethod + def add_modules(click_func): + """Modules/Addons can add their cli commands dynamically.""" + from openpype.modules import ModulesManager + + manager = ModulesManager() + for module in manager.modules: + if hasattr(module, "cli"): + module.cli(click_func) + return click_func + @staticmethod def launch_eventservercli(*args): from openpype_modules.ftrack.ftrack_server.event_server_cli import ( From afd93a403567f9d79bf4569b8b98419993186c6d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:57:02 +0100 Subject: [PATCH 04/25] ftrack uses new cli option to register it's cli commands --- .../default_modules/ftrack/ftrack_module.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index 73a4dfee82..cc9210c216 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -1,8 +1,10 @@ import os import json import collections -from openpype.modules import OpenPypeModule +import click + +from openpype.modules import OpenPypeModule from openpype_interfaces import ( ITrayModule, IPluginPaths, @@ -409,3 +411,57 @@ class FtrackModule( return 0 hours_logged = (task_entity["time_logged"] / 60) / 60 return hours_logged + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group( + FtrackModule.name, + help="Application job server. Can be used as render farm." +) +def cli_main(): + pass + + +@cli_main.command() +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("--ftrack-url", envvar="FTRACK_SERVER", + help="Ftrack server url") +@click.option("--ftrack-user", envvar="FTRACK_API_USER", + help="Ftrack api user") +@click.option("--ftrack-api-key", envvar="FTRACK_API_KEY", + help="Ftrack api key") +@click.option("--legacy", is_flag=True, + help="run event server without mongo storing") +@click.option("--clockify-api-key", envvar="CLOCKIFY_API_KEY", + help="Clockify API key.") +@click.option("--clockify-workspace", envvar="CLOCKIFY_WORKSPACE", + help="Clockify workspace") +def eventserver( + debug, + ftrack_url, + ftrack_user, + ftrack_api_key, + legacy, + clockify_api_key, + clockify_workspace +): + """Launch ftrack event server. + + This should be ideally used by system service (such us systemd or upstart + on linux and window service). + """ + if debug: + os.environ["OPENPYPE_DEBUG"] = "3" + + from .ftrack_server.event_server_cli import run_event_server + + return run_event_server( + ftrack_url, + ftrack_user, + ftrack_api_key, + legacy, + clockify_api_key, + clockify_workspace + ) From 88ac91c82c79f036108ccd37cb95103243588ce6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:50:44 +0100 Subject: [PATCH 05/25] fix help --- openpype/modules/default_modules/ftrack/ftrack_module.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index cc9210c216..6db80e6c4a 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -416,10 +416,7 @@ class FtrackModule( click_group.add_command(cli_main) -@click.group( - FtrackModule.name, - help="Application job server. Can be used as render farm." -) +@click.group(FtrackModule.name, help="Ftrack module related commands.") def cli_main(): pass From 2f4ee4447e0d24b1213dc6290ea16af3e27e9531 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:51:45 +0100 Subject: [PATCH 06/25] example addon has cli commands --- .../example_addons/example_addon/addon.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/openpype/modules/example_addons/example_addon/addon.py b/openpype/modules/example_addons/example_addon/addon.py index 5573e33cc1..f4a1aa4f4e 100644 --- a/openpype/modules/example_addons/example_addon/addon.py +++ b/openpype/modules/example_addons/example_addon/addon.py @@ -8,10 +8,12 @@ in global space here until are required or used. """ import os +import click from openpype.modules import ( JsonFilesSettingsDef, - OpenPypeAddOn + OpenPypeAddOn, + ModulesManager ) # Import interface defined by this addon to be able find other addons using it from openpype_interfaces import ( @@ -130,3 +132,32 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): return { "publish": [os.path.join(current_dir, "plugins", "publish")] } + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group(ExampleAddon.name, help="Example addon dynamic cli commands.") +def cli_main(): + pass + + +@cli_main.command() +def nothing(): + """Does nothing but print a message.""" + print("You've triggered \"nothing\" command.") + + +@cli_main.command() +def show_dialog(): + """Show ExampleAddon dialog. + + We don't have access to addon directly through cli so we have to create + it again. + """ + from openpype.tools.utils.lib import qt_app_context + + manager = ModulesManager() + example_addon = manager.modules_by_name[ExampleAddon.name] + with qt_app_context(): + example_addon.show_dialog() From bf9d4442ea904004ce080a550a5ea1b5aa476d93 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:55:31 +0100 Subject: [PATCH 07/25] added cli info to modules readme --- openpype/modules/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/modules/README.md b/openpype/modules/README.md index 5716324365..7c17025487 100644 --- a/openpype/modules/README.md +++ b/openpype/modules/README.md @@ -22,6 +22,10 @@ OpenPype modules should contain separated logic of specific kind of implementati - `__init__` should not be overridden and `initialize` should not do time consuming part but only prepare base data about module - also keep in mind that they may be initialized in headless mode - connection with other modules is made with help of interfaces +- `cli` method - can add cli commands specific for the module + - command line arguments are handled using `click` python module + - `cli` method should expect single argument which is click group on which can be called any group specific methods (e.g. `add_command` to add another click group as children see `ExampleAddon`) + - it is possible to add trigger cli commands using `./openpype_console module *args` ## Addon class `OpenPypeAddOn` - inherits from `OpenPypeModule` but is enabled by default and doesn't have to implement `initialize` and `connect_with_modules` methods @@ -140,4 +144,4 @@ class ClockifyModule( ### TrayModulesManager - inherits from `ModulesManager` -- has specific implementation for Pype Tray tool and handle `ITrayModule` methods \ No newline at end of file +- has specific implementation for Pype Tray tool and handle `ITrayModule` methods From 887e4ba004446392f89b605727d6f7447b1b5452 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:55:51 +0100 Subject: [PATCH 08/25] do not crash of failed calling of 'cli' method --- openpype/pype_commands.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index f4a29091d0..d4cba6f93e 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -47,9 +47,18 @@ class PypeCommands: from openpype.modules import ModulesManager manager = ModulesManager() + log = PypeLogger.get_logger("AddModulesCLI") for module in manager.modules: if hasattr(module, "cli"): - module.cli(click_func) + try: + module.cli(click_func) + + except Exception: + log.warning( + "Failed to add cli command for module \"{}\"".format( + module.name + ) + ) return click_func @staticmethod From 901eb5bc9dbd850a32c43d3e4f4ea0acfabb5aa3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 18:02:19 +0100 Subject: [PATCH 09/25] added cli method to default methods of openpypemodule --- openpype/modules/README.md | 2 +- openpype/modules/base.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/openpype/modules/README.md b/openpype/modules/README.md index 7c17025487..86afdb9d91 100644 --- a/openpype/modules/README.md +++ b/openpype/modules/README.md @@ -22,7 +22,7 @@ OpenPype modules should contain separated logic of specific kind of implementati - `__init__` should not be overridden and `initialize` should not do time consuming part but only prepare base data about module - also keep in mind that they may be initialized in headless mode - connection with other modules is made with help of interfaces -- `cli` method - can add cli commands specific for the module +- `cli` method - add cli commands specific for the module - command line arguments are handled using `click` python module - `cli` method should expect single argument which is click group on which can be called any group specific methods (e.g. `add_command` to add another click group as children see `ExampleAddon`) - it is possible to add trigger cli commands using `./openpype_console module *args` diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..5773c684b6 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -431,6 +431,28 @@ class OpenPypeModule: """ return {} + def cli(self, module_click_group): + """Add commands to click group. + + The best practise is to create click group for whole module which is + used to separate commands. + + class MyPlugin(OpenPypeModule): + ... + def cli(self, module_click_group): + module_click_group.add_command(cli_main) + + + @click.group(, help="") + def cli_main(): + pass + + @cli_main.command() + def mycommand(): + print("my_command") + """ + pass + class OpenPypeAddOn(OpenPypeModule): # Enable Addon by default From 2709ece76ad9823921b668427b647958131a2110 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 18:08:30 +0100 Subject: [PATCH 10/25] check of cli existence is not needed anymore --- openpype/pype_commands.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d4cba6f93e..5ff57ab6ad 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -49,16 +49,15 @@ class PypeCommands: manager = ModulesManager() log = PypeLogger.get_logger("AddModulesCLI") for module in manager.modules: - if hasattr(module, "cli"): - try: - module.cli(click_func) + try: + module.cli(click_func) - except Exception: - log.warning( - "Failed to add cli command for module \"{}\"".format( - module.name - ) + except Exception: + log.warning( + "Failed to add cli command for module \"{}\"".format( + module.name ) + ) return click_func @staticmethod From d4f04f72da7a952cca2709df0fd6e42a6b374637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LORRAIN?= Date: Wed, 10 Nov 2021 12:07:07 +0100 Subject: [PATCH 11/25] fix line sizes --- openpype/plugins/publish/validate_version.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/plugins/publish/validate_version.py b/openpype/plugins/publish/validate_version.py index b86d72a658..e48ce6e3c3 100644 --- a/openpype/plugins/publish/validate_version.py +++ b/openpype/plugins/publish/validate_version.py @@ -21,8 +21,9 @@ class ValidateVersion(pyblish.api.InstancePlugin): if latest_version is not None: msg = ( - "Version `{0}` from instance `{1}` that you are trying to publish, already exists" - " in the database. Version in database: `{2}`. Please version " - "up your workfile to a higher version number than: `{2}`." + "Version `{0}` from instance `{1}` that you are trying to" + " publish, already exists in the database. Version in" + " database: `{2}`. Please version up your workfile to a higher" + " version number than: `{2}`." ).format(version, instance.data["name"], latest_version) assert (int(version) > int(latest_version)), msg From cc0672d42974a3381108d62fb175e7447dbca987 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 17:38:17 +0100 Subject: [PATCH 12/25] OP-2017 - fix - enum for color coding in PS Some keys are really weird in PS --- .../projects_schema/schema_project_photoshop.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index f00bf78fe4..ca388de60c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -67,7 +67,17 @@ "type": "list", "key": "color_code", "label": "Color codes for layers", - "object_type": "text" + "type": "enum", + "multiselection": true, + "enum_items": [ + { "red": "red" }, + { "orange": "orange" }, + { "yellowColor": "yellow" }, + { "grain": "green" }, + { "blue": "blue" }, + { "violet": "violet" }, + { "gray": "gray" } + ] }, { "type": "list", From e6d76c5b580252a4eea6be22cf02de0380e2786d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 14:17:09 +0100 Subject: [PATCH 13/25] fix python 2 unicode conversion --- openpype/lib/python_module_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/lib/python_module_tools.py b/openpype/lib/python_module_tools.py index cb5f285ddd..69da4cc661 100644 --- a/openpype/lib/python_module_tools.py +++ b/openpype/lib/python_module_tools.py @@ -22,6 +22,9 @@ def import_filepath(filepath, module_name=None): if module_name is None: module_name = os.path.splitext(os.path.basename(filepath))[0] + # Make sure it is not 'unicode' in Python 2 + module_name = str(module_name) + # Prepare module object where content of file will be parsed module = types.ModuleType(module_name) From 5acdb6e182efede7d01d136c431c41329e77e203 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 14:53:57 +0100 Subject: [PATCH 14/25] added docstring --- openpype/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index d68cba45c6..6a4d8f1120 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -61,6 +61,10 @@ def tray(debug=False): @main.group(help="Run command line arguments of OpenPype modules") @click.pass_context def module(ctx): + """Module specific commands created dynamically. + + These commands are generated dynamically by currently loaded addon/modules. + """ pass From adbd0b1c5e8b5d2d2a7a9a862798a373b982b431 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Nov 2021 18:01:14 +0100 Subject: [PATCH 15/25] OP-2015 - added queue for studio processing in PS --- .../webserver_service/webpublish_routes.py | 20 ++++++++++++------- .../webserver_service/webserver_cli.py | 11 +++++++++- .../defaults/project_settings/photoshop.json | 3 +-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 73e5113f38..a56b7c48c3 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -20,11 +20,16 @@ log = PypeLogger.get_logger("WebServer") class RestApiResource: """Resource carrying needed info and Avalon DB connection for publish.""" - def __init__(self, server_manager, executable, upload_dir): + def __init__(self, server_manager, executable, upload_dir, + studio_task_queue=None): self.server_manager = server_manager self.upload_dir = upload_dir self.executable = executable + if studio_task_queue is None: + studio_task_queue = collections.deque().dequeu + self.studio_task_queue = studio_task_queue + self.dbcon = AvalonMongoDB() self.dbcon.install() @@ -182,8 +187,6 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): msg = "Non existent OpenPype executable {}".format(openpype_app) raise RuntimeError(msg) - # for postprocessing in host, currently only PS - output = {} log.info("WebpublisherBatchPublishEndpoint called") content = await request.json() @@ -225,13 +228,13 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): batch_data = parse_json(os.path.join(batch_path, "manifest.json")) if not batch_data: raise ValueError( - "Cannot parse batch meta in {} folder".format(batch_path)) + "Cannot parse batch manifest in {}".format(batch_path)) task_dir_name = batch_data["tasks"][0] task_data = parse_json(os.path.join(batch_path, task_dir_name, "manifest.json")) if not task_data: raise ValueError( - "Cannot parse batch meta in {} folder".format(task_data)) + "Cannot parse task manifest in {}".format(task_data)) for process_filter in studio_processing_filters: filter_extensions = process_filter.get("extensions") or [] @@ -263,11 +266,14 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): args.append(value) log.info("args:: {}".format(args)) + if content.get("studio_processing"): + log.debug("Adding to queue") + self.resource.studio_task_queue.append(args) + else: + subprocess.call(args) - subprocess.call(args) return Response( status=200, - body=self.resource.encode(output), content_type="application/json" ) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index d00d269059..b784105461 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -1,8 +1,10 @@ +import collections import time import os from datetime import datetime import requests import json +import subprocess from openpype.lib import PypeLogger @@ -31,10 +33,13 @@ def run_webserver(*args, **kwargs): port = kwargs.get("port") or 8079 server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url + # queue for remotepublishfromapp tasks + studio_task_queue = collections.deque() resource = RestApiResource(server_manager, upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) + executable=kwargs["executable"], + studio_task_queue=studio_task_queue) projects_endpoint = WebpublisherProjectsEndpoint(resource) server_manager.add_route( "GET", @@ -88,6 +93,10 @@ def run_webserver(*args, **kwargs): if time.time() - last_reprocessed > 20: reprocess_failed(kwargs["upload_dir"], webserver_url) last_reprocessed = time.time() + if studio_task_queue: + args = studio_task_queue.popleft() + subprocess.call(args) # blocking call + time.sleep(1.0) diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index eb9f96e348..0c24c943ec 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -30,8 +30,7 @@ }, "ExtractReview": { "jpg_options": { - "tags": [ - ] + "tags": [] }, "mov_options": { "tags": [ From 79fce6a5b453ed88fe53374578fec797dded539a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:05:20 +0100 Subject: [PATCH 16/25] example addon is not using inteface --- .../example_addons/example_addon/addon.py | 16 ----------- .../example_addon/interfaces.py | 28 ------------------- .../example_addons/example_addon/widgets.py | 12 ++------ 3 files changed, 2 insertions(+), 54 deletions(-) delete mode 100644 openpype/modules/example_addons/example_addon/interfaces.py diff --git a/openpype/modules/example_addons/example_addon/addon.py b/openpype/modules/example_addons/example_addon/addon.py index 5573e33cc1..162868e4d4 100644 --- a/openpype/modules/example_addons/example_addon/addon.py +++ b/openpype/modules/example_addons/example_addon/addon.py @@ -15,7 +15,6 @@ from openpype.modules import ( ) # Import interface defined by this addon to be able find other addons using it from openpype_interfaces import ( - IExampleInterface, IPluginPaths, ITrayAction ) @@ -75,19 +74,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): self._create_dialog() - def connect_with_modules(self, enabled_modules): - """Method where you should find connected modules. - - It is triggered by OpenPype modules manager at the best possible time. - Some addons and modules may required to connect with other modules - before their main logic is executed so changes would require to restart - whole process. - """ - self._connected_modules = [] - for module in enabled_modules: - if isinstance(module, IExampleInterface): - self._connected_modules.append(module) - def _create_dialog(self): # Don't recreate dialog if already exists if self._dialog is not None: @@ -106,8 +92,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): """ # Make sure dialog is created self._create_dialog() - # Change value of dialog by current state - self._dialog.set_connected_modules(self.get_connected_modules()) # Show dialog self._dialog.open() diff --git a/openpype/modules/example_addons/example_addon/interfaces.py b/openpype/modules/example_addons/example_addon/interfaces.py deleted file mode 100644 index 371536efc7..0000000000 --- a/openpype/modules/example_addons/example_addon/interfaces.py +++ /dev/null @@ -1,28 +0,0 @@ -""" Using interfaces is one way of connecting multiple OpenPype Addons/Modules. - -Interfaces must be in `interfaces.py` file (or folder). Interfaces should not -import module logic or other module in global namespace. That is because -all of them must be imported before all OpenPype AddOns and Modules. - -Ideally they should just define abstract and helper methods. If interface -require any logic or connection it should be defined in module. - -Keep in mind that attributes and methods will be added to other addon -attributes and methods so they should be unique and ideally contain -addon name in it's name. -""" - -from abc import abstractmethod -from openpype.modules import OpenPypeInterface - - -class IExampleInterface(OpenPypeInterface): - """Example interface of addon.""" - _example_module = None - - def get_example_module(self): - return self._example_module - - @abstractmethod - def example_method_of_example_interface(self): - pass diff --git a/openpype/modules/example_addons/example_addon/widgets.py b/openpype/modules/example_addons/example_addon/widgets.py index 0acf238409..c0a0a7e510 100644 --- a/openpype/modules/example_addons/example_addon/widgets.py +++ b/openpype/modules/example_addons/example_addon/widgets.py @@ -9,7 +9,8 @@ class MyExampleDialog(QtWidgets.QDialog): self.setWindowTitle("Connected modules") - label_widget = QtWidgets.QLabel(self) + msg = "This is example dialog of example addon." + label_widget = QtWidgets.QLabel(msg, self) ok_btn = QtWidgets.QPushButton("OK", self) btns_layout = QtWidgets.QHBoxLayout() @@ -28,12 +29,3 @@ class MyExampleDialog(QtWidgets.QDialog): def _on_ok_clicked(self): self.done(1) - - def set_connected_modules(self, connected_modules): - if connected_modules: - message = "\n".join(connected_modules) - else: - message = ( - "Other enabled modules/addons are not using my interface." - ) - self._label_widget.setText(message) From f0a913e02493ba0c2a8885c69d77ec740d5d936d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:05:33 +0100 Subject: [PATCH 17/25] moved interfaces to modules directory --- openpype/modules/{default_modules => }/interfaces.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openpype/modules/{default_modules => }/interfaces.py (100%) diff --git a/openpype/modules/default_modules/interfaces.py b/openpype/modules/interfaces.py similarity index 100% rename from openpype/modules/default_modules/interfaces.py rename to openpype/modules/interfaces.py From b2b532de36327c845da37cb4c8c30c88debbfdc1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:06:09 +0100 Subject: [PATCH 18/25] modifid current interfaces import to keep backwards compatibility --- openpype/modules/base.py | 55 +++++++--------------------------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..01d54d9051 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -212,54 +212,17 @@ def _load_interfaces(): _InterfacesClass(modules_key) ) - log = PypeLogger.get_logger("InterfacesLoader") + from . import interfaces - dirpaths = get_module_dirs() - - interface_paths = [] - interface_paths.append( - os.path.join(get_default_modules_dir(), "interfaces.py") - ) - for dirpath in dirpaths: - if not os.path.exists(dirpath): + for attr_name in dir(interfaces): + attr = getattr(interfaces, attr_name) + if ( + not inspect.isclass(attr) + or attr is OpenPypeInterface + or not issubclass(attr, OpenPypeInterface) + ): continue - - for filename in os.listdir(dirpath): - if filename in ("__pycache__", ): - continue - - full_path = os.path.join(dirpath, filename) - if not os.path.isdir(full_path): - continue - - interfaces_path = os.path.join(full_path, "interfaces.py") - if os.path.exists(interfaces_path): - interface_paths.append(interfaces_path) - - for full_path in interface_paths: - if not os.path.exists(full_path): - continue - - try: - # Prepare module object where content of file will be parsed - module = import_filepath(full_path) - - except Exception: - log.warning( - "Failed to load path: \"{0}\"".format(full_path), - exc_info=True - ) - continue - - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - not inspect.isclass(attr) - or attr is OpenPypeInterface - or not issubclass(attr, OpenPypeInterface) - ): - continue - setattr(openpype_interfaces, attr_name, attr) + setattr(openpype_interfaces, attr_name, attr) def load_modules(force=False): From a051f4e8b6cdc5f069bde46187d3583f13d8dc35 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:18:04 +0100 Subject: [PATCH 19/25] moved 'ISettingsChangeListener' to global interfaces --- .../settings_module/interfaces.py | 30 ------------------- openpype/modules/interfaces.py | 28 +++++++++++++++++ 2 files changed, 28 insertions(+), 30 deletions(-) delete mode 100644 openpype/modules/default_modules/settings_module/interfaces.py diff --git a/openpype/modules/default_modules/settings_module/interfaces.py b/openpype/modules/default_modules/settings_module/interfaces.py deleted file mode 100644 index 42db395649..0000000000 --- a/openpype/modules/default_modules/settings_module/interfaces.py +++ /dev/null @@ -1,30 +0,0 @@ -from abc import abstractmethod -from openpype.modules import OpenPypeInterface - - -class ISettingsChangeListener(OpenPypeInterface): - """Module has plugin paths to return. - - Expected result is dictionary with keys "publish", "create", "load" or - "actions" and values as list or string. - { - "publish": ["path/to/publish_plugins"] - } - """ - @abstractmethod - def on_system_settings_save( - self, old_value, new_value, changes, new_value_metadata - ): - pass - - @abstractmethod - def on_project_settings_save( - self, old_value, new_value, changes, project_name, new_value_metadata - ): - pass - - @abstractmethod - def on_project_anatomy_save( - self, old_value, new_value, changes, project_name, new_value_metadata - ): - pass diff --git a/openpype/modules/interfaces.py b/openpype/modules/interfaces.py index a60c5fa606..e6e84a0d42 100644 --- a/openpype/modules/interfaces.py +++ b/openpype/modules/interfaces.py @@ -263,3 +263,31 @@ class ITrayService(ITrayModule): """Change icon of an QAction to orange circle.""" if self.menu_action: self.menu_action.setIcon(self.get_icon_idle()) + + +class ISettingsChangeListener(OpenPypeInterface): + """Module has plugin paths to return. + + Expected result is dictionary with keys "publish", "create", "load" or + "actions" and values as list or string. + { + "publish": ["path/to/publish_plugins"] + } + """ + @abstractmethod + def on_system_settings_save( + self, old_value, new_value, changes, new_value_metadata + ): + pass + + @abstractmethod + def on_project_settings_save( + self, old_value, new_value, changes, project_name, new_value_metadata + ): + pass + + @abstractmethod + def on_project_anatomy_save( + self, old_value, new_value, changes, project_name, new_value_metadata + ): + pass From 9a74a6a1e8602cb8000b088630c585927141851c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:18:19 +0100 Subject: [PATCH 20/25] removed unused import of ITimersManager --- .../modules/default_modules/timers_manager/timers_manager.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/timers_manager/timers_manager.py b/openpype/modules/default_modules/timers_manager/timers_manager.py index 7687d056f8..1aeccbb958 100644 --- a/openpype/modules/default_modules/timers_manager/timers_manager.py +++ b/openpype/modules/default_modules/timers_manager/timers_manager.py @@ -1,10 +1,7 @@ import os import platform from openpype.modules import OpenPypeModule -from openpype_interfaces import ( - ITimersManager, - ITrayService -) +from openpype_interfaces import ITrayService from avalon.api import AvalonMongoDB From b09382bf8dbd3c34666306bc5db8e2c5cf046d6d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:20:09 +0100 Subject: [PATCH 21/25] raise ImportError if interface does not exists --- openpype/modules/base.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 01d54d9051..cb39666626 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -107,12 +107,9 @@ class _InterfacesClass(_ModuleClass): if attr_name in ("__path__", "__file__"): return None - # Fake Interface if is not missing - self.__attributes__[attr_name] = type( - attr_name, - (MissingInteface, ), - {} - ) + raise ImportError(( + "cannot import name '{}' from 'openpype_interfaces'" + ).format(attr_name)) return self.__attributes__[attr_name] From d3fd330f944d212e90b3a5548bad38b78d6955e6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:21:38 +0100 Subject: [PATCH 22/25] removed unused 'MissingInterface' --- openpype/modules/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index cb39666626..3f59d5900b 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -329,14 +329,6 @@ class OpenPypeInterface: pass -class MissingInteface(OpenPypeInterface): - """Class representing missing interface class. - - Used when interface is not available from currently registered paths. - """ - pass - - @six.add_metaclass(ABCMeta) class OpenPypeModule: """Base class of pype module. From 992986fd1be27612b95e4a3007aac926c53dbd77 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:31:42 +0100 Subject: [PATCH 23/25] skip module directories without init file --- openpype/modules/base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..692495600c 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -333,6 +333,15 @@ def _load_modules(): # - check manifest and content of manifest try: if os.path.isdir(fullpath): + # Module without init file can't be used as OpenPype module + # because the module class could not be imported + init_file = os.path.join(fullpath, "__init__.py") + if not os.path.exists(init_file): + log.info(( + "Skipping module directory because of" + " missing \"__init__.py\" file. \"{}\"" + ).format(fullpath)) + continue import_module_from_dirpath(dirpath, filename, modules_key) elif ext in (".py", ): From 1431bc5ac9887c6097cdc91887e4e1954cefecc3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 13:30:37 +0100 Subject: [PATCH 24/25] OP-2015 - fix - adding to queue decided by configuration --- .../webpublisher/webserver_service/webpublish_routes.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index a56b7c48c3..445fa071c5 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -205,7 +205,10 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): # Make sure targets are set to None for cases that default # would change # - targets argument is not used in 'remotepublishfromapp' - "targets": None + "targets": None, + # does publish need to be handled by a queue, eg. only + # single process running concurrently? + "add_to_queue": True } } ] @@ -222,6 +225,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): "targets": ["filespublish"] } + add_to_queue = False if content.get("studio_processing"): log.info("Post processing called") @@ -247,6 +251,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): add_args.update( process_filter.get("arguments") or {} ) + add_to_queue = process_filter["add_to_queue"] break args = [ @@ -266,7 +271,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): args.append(value) log.info("args:: {}".format(args)) - if content.get("studio_processing"): + if add_to_queue: log.debug("Adding to queue") self.resource.studio_task_queue.append(args) else: From 27b6197ee8def8259080d0a90f0c12e3771c8324 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 13:52:28 +0100 Subject: [PATCH 25/25] OP-2015 - fix - adding to queue decided by configuration --- .../webserver_service/webpublish_routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 445fa071c5..e34a899c4b 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -205,11 +205,11 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): # Make sure targets are set to None for cases that default # would change # - targets argument is not used in 'remotepublishfromapp' - "targets": None, - # does publish need to be handled by a queue, eg. only - # single process running concurrently? - "add_to_queue": True - } + "targets": None + }, + # does publish need to be handled by a queue, eg. only + # single process running concurrently? + "add_to_queue": True } ]