Merge branch 'develop' into feature/OP-1978_Use-new-site-sync-entity-in-settings-and-modify-their-loading

openpype/cli.py

@@ -57,6 +57,17 @@ def tray(debug=False):
     PypeCommands().launch_tray(debug)
 
 
+@PypeCommands.add_modules
+@main.group(help="Run command line arguments of OpenPype modules")
+@click.pass_context
+def module(ctx):
+    """Module specific commands created dynamically.
+
+    These commands are generated dynamically by currently loaded addon/modules.
+    """
+    pass
+
+
 @main.command()
 @click.option("-d", "--debug", is_flag=True, help="Print debug messages")
 @click.option("--ftrack-url", envvar="FTRACK_SERVER",

openpype/hosts/webpublisher/webserver_service/webpublish_routes.py

@@ -20,11 +20,16 @@ log = PypeLogger.get_logger("WebServer")
 
 class RestApiResource:
     """Resource carrying needed info and Avalon DB connection for publish."""
-    def __init__(self, server_manager, executable, upload_dir):
+    def __init__(self, server_manager, executable, upload_dir,
+                 studio_task_queue=None):
         self.server_manager = server_manager
         self.upload_dir = upload_dir
         self.executable = executable
 
+        if studio_task_queue is None:
+            studio_task_queue = collections.deque().dequeu
+        self.studio_task_queue = studio_task_queue
+
         self.dbcon = AvalonMongoDB()
         self.dbcon.install()
 
@@ -182,8 +187,6 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
             msg = "Non existent OpenPype executable {}".format(openpype_app)
             raise RuntimeError(msg)
 
-        # for postprocessing in host, currently only PS
-        output = {}
         log.info("WebpublisherBatchPublishEndpoint called")
         content = await request.json()
 
@@ -203,7 +206,10 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
                     #   would change
                     # - targets argument is not used in 'remotepublishfromapp'
                     "targets": None
-                }
+                },
+                # does publish need to be handled by a queue, eg. only
+                # single process running concurrently?
+                "add_to_queue": True
             }
         ]
 
@@ -219,19 +225,20 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
             "targets": ["filespublish"]
         }
 
+        add_to_queue = False
         if content.get("studio_processing"):
             log.info("Post processing called")
 
             batch_data = parse_json(os.path.join(batch_path, "manifest.json"))
             if not batch_data:
                 raise ValueError(
-                    "Cannot parse batch meta in {} folder".format(batch_path))
+                    "Cannot parse batch manifest in {}".format(batch_path))
             task_dir_name = batch_data["tasks"][0]
             task_data = parse_json(os.path.join(batch_path, task_dir_name,
                                                 "manifest.json"))
             if not task_data:
                 raise ValueError(
-                    "Cannot parse batch meta in {} folder".format(task_data))
+                    "Cannot parse task manifest in {}".format(task_data))
 
             for process_filter in studio_processing_filters:
                 filter_extensions = process_filter.get("extensions") or []
@@ -244,6 +251,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
                         add_args.update(
                             process_filter.get("arguments") or {}
                         )
+                        add_to_queue = process_filter["add_to_queue"]
                         break
 
         args = [
@@ -263,11 +271,14 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint):
                     args.append(value)
 
         log.info("args:: {}".format(args))
+        if add_to_queue:
+            log.debug("Adding to queue")
+            self.resource.studio_task_queue.append(args)
+        else:
+            subprocess.call(args)
 
-        subprocess.call(args)
         return Response(
             status=200,
-            body=self.resource.encode(output),
             content_type="application/json"
         )
 

openpype/hosts/webpublisher/webserver_service/webserver_cli.py

@@ -1,8 +1,10 @@
+import collections
 import time
 import os
 from datetime import datetime
 import requests
 import json
+import subprocess
 
 from openpype.lib import PypeLogger
 
@@ -31,10 +33,13 @@ def run_webserver(*args, **kwargs):
     port = kwargs.get("port") or 8079
     server_manager = webserver_module.create_new_server_manager(port, host)
     webserver_url = server_manager.url
+    # queue for remotepublishfromapp tasks
+    studio_task_queue = collections.deque()
 
     resource = RestApiResource(server_manager,
                                upload_dir=kwargs["upload_dir"],
-                               executable=kwargs["executable"])
+                               executable=kwargs["executable"],
+                               studio_task_queue=studio_task_queue)
     projects_endpoint = WebpublisherProjectsEndpoint(resource)
     server_manager.add_route(
         "GET",
@@ -88,6 +93,10 @@ def run_webserver(*args, **kwargs):
         if time.time() - last_reprocessed > 20:
             reprocess_failed(kwargs["upload_dir"], webserver_url)
             last_reprocessed = time.time()
+        if studio_task_queue:
+            args = studio_task_queue.popleft()
+            subprocess.call(args)  # blocking call
+
         time.sleep(1.0)
 
 

openpype/lib/python_module_tools.py

@@ -22,6 +22,9 @@ def import_filepath(filepath, module_name=None):
     if module_name is None:
         module_name = os.path.splitext(os.path.basename(filepath))[0]
 
+    # Make sure it is not 'unicode' in Python 2
+    module_name = str(module_name)
+
     # Prepare module object where content of file will be parsed
     module = types.ModuleType(module_name)
 

openpype/modules/README.md

@@ -22,6 +22,10 @@ OpenPype modules should contain separated logic of specific kind of implementati
 - `__init__` should not be overridden and `initialize` should not do time consuming part but only prepare base data about module
  - also keep in mind that they may be initialized in headless mode
 - connection with other modules is made with help of interfaces
+- `cli` method - add cli commands specific for the module
+    - command line arguments are handled using `click` python module
+    - `cli` method should expect single argument which is click group on which can be called any group specific methods (e.g. `add_command` to add another click group as children see `ExampleAddon`)
+    - it is possible to add trigger cli commands using `./openpype_console module <module_name> <command> *args`
 
 ## Addon class `OpenPypeAddOn`
 - inherits from `OpenPypeModule` but is enabled by default and doesn't have to implement `initialize` and `connect_with_modules` methods
@@ -140,4 +144,4 @@ class ClockifyModule(
 
 ### TrayModulesManager
 - inherits from `ModulesManager`
-- has specific implementation for Pype Tray tool and handle `ITrayModule` methods
+- has specific implementation for Pype Tray tool and handle `ITrayModule` methods

openpype/modules/base.py

@@ -107,12 +107,9 @@ class _InterfacesClass(_ModuleClass):
             if attr_name in ("__path__", "__file__"):
                 return None
 
-            # Fake Interface if is not missing
-            self.__attributes__[attr_name] = type(
-                attr_name,
-                (MissingInteface, ),
-                {}
-            )
+            raise ImportError((
+                "cannot import name '{}' from 'openpype_interfaces'"
+            ).format(attr_name))
 
         return self.__attributes__[attr_name]
 
@@ -212,54 +209,17 @@ def _load_interfaces():
         _InterfacesClass(modules_key)
     )
 
-    log = PypeLogger.get_logger("InterfacesLoader")
+    from . import interfaces
 
-    dirpaths = get_module_dirs()
-
-    interface_paths = []
-    interface_paths.append(
-        os.path.join(get_default_modules_dir(), "interfaces.py")
-    )
-    for dirpath in dirpaths:
-        if not os.path.exists(dirpath):
+    for attr_name in dir(interfaces):
+        attr = getattr(interfaces, attr_name)
+        if (
+            not inspect.isclass(attr)
+            or attr is OpenPypeInterface
+            or not issubclass(attr, OpenPypeInterface)
+        ):
             continue
-
-        for filename in os.listdir(dirpath):
-            if filename in ("__pycache__", ):
-                continue
-
-            full_path = os.path.join(dirpath, filename)
-            if not os.path.isdir(full_path):
-                continue
-
-            interfaces_path = os.path.join(full_path, "interfaces.py")
-            if os.path.exists(interfaces_path):
-                interface_paths.append(interfaces_path)
-
-    for full_path in interface_paths:
-        if not os.path.exists(full_path):
-            continue
-
-        try:
-            # Prepare module object where content of file will be parsed
-            module = import_filepath(full_path)
-
-        except Exception:
-            log.warning(
-                "Failed to load path: \"{0}\"".format(full_path),
-                exc_info=True
-            )
-            continue
-
-        for attr_name in dir(module):
-            attr = getattr(module, attr_name)
-            if (
-                not inspect.isclass(attr)
-                or attr is OpenPypeInterface
-                or not issubclass(attr, OpenPypeInterface)
-            ):
-                continue
-            setattr(openpype_interfaces, attr_name, attr)
+        setattr(openpype_interfaces, attr_name, attr)
 
 
 def load_modules(force=False):
@@ -333,6 +293,15 @@ def _load_modules():
             # - check manifest and content of manifest
             try:
                 if os.path.isdir(fullpath):
+                    # Module without init file can't be used as OpenPype module
+                    #   because the module class could not be imported
+                    init_file = os.path.join(fullpath, "__init__.py")
+                    if not os.path.exists(init_file):
+                        log.info((
+                            "Skipping module directory because of"
+                            " missing \"__init__.py\" file. \"{}\""
+                        ).format(fullpath))
+                        continue
                     import_module_from_dirpath(dirpath, filename, modules_key)
 
                 elif ext in (".py", ):
@@ -369,14 +338,6 @@ class OpenPypeInterface:
     pass
 
 
-class MissingInteface(OpenPypeInterface):
-    """Class representing missing interface class.
-
-    Used when interface is not available from currently registered paths.
-    """
-    pass
-
-
 @six.add_metaclass(ABCMeta)
 class OpenPypeModule:
     """Base class of pype module.
@@ -431,6 +392,28 @@ class OpenPypeModule:
         """
         return {}
 
+    def cli(self, module_click_group):
+        """Add commands to click group.
+
+        The best practise is to create click group for whole module which is
+        used to separate commands.
+
+        class MyPlugin(OpenPypeModule):
+            ...
+            def cli(self, module_click_group):
+                module_click_group.add_command(cli_main)
+
+
+        @click.group(<module name>, help="<Any help shown in cmd>")
+        def cli_main():
+            pass
+
+        @cli_main.command()
+        def mycommand():
+            print("my_command")
+        """
+        pass
+
 
 class OpenPypeAddOn(OpenPypeModule):
     # Enable Addon by default

openpype/modules/default_modules/ftrack/ftrack_module.py

@@ -1,8 +1,10 @@
 import os
 import json
 import collections
-from openpype.modules import OpenPypeModule
 
+import click
+
+from openpype.modules import OpenPypeModule
 from openpype_interfaces import (
     ITrayModule,
     IPluginPaths,
@@ -409,3 +411,54 @@ class FtrackModule(
             return 0
         hours_logged = (task_entity["time_logged"] / 60) / 60
         return hours_logged
+
+    def cli(self, click_group):
+        click_group.add_command(cli_main)
+
+
+@click.group(FtrackModule.name, help="Ftrack module related commands.")
+def cli_main():
+    pass
+
+
+@cli_main.command()
+@click.option("-d", "--debug", is_flag=True, help="Print debug messages")
+@click.option("--ftrack-url", envvar="FTRACK_SERVER",
+              help="Ftrack server url")
+@click.option("--ftrack-user", envvar="FTRACK_API_USER",
+              help="Ftrack api user")
+@click.option("--ftrack-api-key", envvar="FTRACK_API_KEY",
+              help="Ftrack api key")
+@click.option("--legacy", is_flag=True,
+              help="run event server without mongo storing")
+@click.option("--clockify-api-key", envvar="CLOCKIFY_API_KEY",
+              help="Clockify API key.")
+@click.option("--clockify-workspace", envvar="CLOCKIFY_WORKSPACE",
+              help="Clockify workspace")
+def eventserver(
+    debug,
+    ftrack_url,
+    ftrack_user,
+    ftrack_api_key,
+    legacy,
+    clockify_api_key,
+    clockify_workspace
+):
+    """Launch ftrack event server.
+
+    This should be ideally used by system service (such us systemd or upstart
+    on linux and window service).
+    """
+    if debug:
+        os.environ["OPENPYPE_DEBUG"] = "3"
+
+    from .ftrack_server.event_server_cli import run_event_server
+
+    return run_event_server(
+        ftrack_url,
+        ftrack_user,
+        ftrack_api_key,
+        legacy,
+        clockify_api_key,
+        clockify_workspace
+    )

openpype/modules/default_modules/settings_module/interfaces.py

@@ -1,30 +0,0 @@
-from abc import abstractmethod
-from openpype.modules import OpenPypeInterface
-
-
-class ISettingsChangeListener(OpenPypeInterface):
-    """Module has plugin paths to return.
-
-    Expected result is dictionary with keys "publish", "create", "load" or
-    "actions" and values as list or string.
-    {
-        "publish": ["path/to/publish_plugins"]
-    }
-    """
-    @abstractmethod
-    def on_system_settings_save(
-        self, old_value, new_value, changes, new_value_metadata
-    ):
-        pass
-
-    @abstractmethod
-    def on_project_settings_save(
-        self, old_value, new_value, changes, project_name, new_value_metadata
-    ):
-        pass
-
-    @abstractmethod
-    def on_project_anatomy_save(
-        self, old_value, new_value, changes, project_name, new_value_metadata
-    ):
-        pass

openpype/modules/default_modules/sync_server/sync_server_module.py

@@ -821,6 +821,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             self._add_site(collection, query,
                            [representation], elem,
                            alt_site, file_id=file_id, force=True)
+
     """ End of Public API """
 
     def get_local_file_path(self, collection, site_name, file_path):

openpype/modules/default_modules/timers_manager/timers_manager.py

@@ -1,10 +1,7 @@
 import os
 import platform
 from openpype.modules import OpenPypeModule
-from openpype_interfaces import (
-    ITimersManager,
-    ITrayService
-)
+from openpype_interfaces import ITrayService
 from avalon.api import AvalonMongoDB
 
 

openpype/modules/example_addons/example_addon/addon.py

@@ -8,14 +8,15 @@ in global space here until are required or used.
 """
 
 import os
+import click
 
 from openpype.modules import (
     JsonFilesSettingsDef,
-    OpenPypeAddOn
+    OpenPypeAddOn,
+    ModulesManager
 )
 # Import interface defined by this addon to be able find other addons using it
 from openpype_interfaces import (
-    IExampleInterface,
     IPluginPaths,
     ITrayAction
 )
@@ -75,19 +76,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction):
 
         self._create_dialog()
 
-    def connect_with_modules(self, enabled_modules):
-        """Method where you should find connected modules.
-
-        It is triggered by OpenPype modules manager at the best possible time.
-        Some addons and modules may required to connect with other modules
-        before their main logic is executed so changes would require to restart
-        whole process.
-        """
-        self._connected_modules = []
-        for module in enabled_modules:
-            if isinstance(module, IExampleInterface):
-                self._connected_modules.append(module)
-
     def _create_dialog(self):
         # Don't recreate dialog if already exists
         if self._dialog is not None:
@@ -106,8 +94,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction):
         """
         # Make sure dialog is created
         self._create_dialog()
-        # Change value of dialog by current state
-        self._dialog.set_connected_modules(self.get_connected_modules())
         # Show dialog
         self._dialog.open()
 
@@ -130,3 +116,32 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction):
         return {
             "publish": [os.path.join(current_dir, "plugins", "publish")]
         }
+
+    def cli(self, click_group):
+        click_group.add_command(cli_main)
+
+
+@click.group(ExampleAddon.name, help="Example addon dynamic cli commands.")
+def cli_main():
+    pass
+
+
+@cli_main.command()
+def nothing():
+    """Does nothing but print a message."""
+    print("You've triggered \"nothing\" command.")
+
+
+@cli_main.command()
+def show_dialog():
+    """Show ExampleAddon dialog.
+
+    We don't have access to addon directly through cli so we have to create
+    it again.
+    """
+    from openpype.tools.utils.lib import qt_app_context
+
+    manager = ModulesManager()
+    example_addon = manager.modules_by_name[ExampleAddon.name]
+    with qt_app_context():
+        example_addon.show_dialog()

openpype/modules/example_addons/example_addon/interfaces.py

@@ -1,28 +0,0 @@
-""" Using interfaces is one way of connecting multiple OpenPype Addons/Modules.
-
-Interfaces must be in `interfaces.py` file (or folder). Interfaces should not
-import module logic or other module in global namespace. That is because
-all of them must be imported before all OpenPype AddOns and Modules.
-
-Ideally they should just define abstract and helper methods. If interface
-require any logic or connection it should be defined in module.
-
-Keep in mind that attributes and methods will be added to other addon
-attributes and methods so they should be unique and ideally contain
-addon name in it's name.
-"""
-
-from abc import abstractmethod
-from openpype.modules import OpenPypeInterface
-
-
-class IExampleInterface(OpenPypeInterface):
-    """Example interface of addon."""
-    _example_module = None
-
-    def get_example_module(self):
-        return self._example_module
-
-    @abstractmethod
-    def example_method_of_example_interface(self):
-        pass

openpype/modules/example_addons/example_addon/widgets.py

@@ -9,7 +9,8 @@ class MyExampleDialog(QtWidgets.QDialog):
 
         self.setWindowTitle("Connected modules")
 
-        label_widget = QtWidgets.QLabel(self)
+        msg = "This is example dialog of example addon."
+        label_widget = QtWidgets.QLabel(msg, self)
 
         ok_btn = QtWidgets.QPushButton("OK", self)
         btns_layout = QtWidgets.QHBoxLayout()
@@ -28,12 +29,3 @@ class MyExampleDialog(QtWidgets.QDialog):
 
     def _on_ok_clicked(self):
         self.done(1)
-
-    def set_connected_modules(self, connected_modules):
-        if connected_modules:
-            message = "\n".join(connected_modules)
-        else:
-            message = (
-                "Other enabled modules/addons are not using my interface."
-            )
-        self._label_widget.setText(message)

openpype/modules/interfaces.py

@@ -263,3 +263,31 @@ class ITrayService(ITrayModule):
         """Change icon of an QAction to orange circle."""
         if self.menu_action:
             self.menu_action.setIcon(self.get_icon_idle())
+
+
+class ISettingsChangeListener(OpenPypeInterface):
+    """Module has plugin paths to return.
+
+    Expected result is dictionary with keys "publish", "create", "load" or
+    "actions" and values as list or string.
+    {
+        "publish": ["path/to/publish_plugins"]
+    }
+    """
+    @abstractmethod
+    def on_system_settings_save(
+        self, old_value, new_value, changes, new_value_metadata
+    ):
+        pass
+
+    @abstractmethod
+    def on_project_settings_save(
+        self, old_value, new_value, changes, project_name, new_value_metadata
+    ):
+        pass
+
+    @abstractmethod
+    def on_project_anatomy_save(
+        self, old_value, new_value, changes, project_name, new_value_metadata
+    ):
+        pass

openpype/plugins/publish/collect_anatomy_instance_data.py

@@ -38,6 +38,8 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
     order = pyblish.api.CollectorOrder + 0.49
     label = "Collect Anatomy Instance data"
 
+    follow_workfile_version = False
+
     def process(self, context):
         self.log.info("Collecting anatomy data for all instances.")
 
@@ -213,7 +215,10 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
         context_asset_doc = context.data["assetEntity"]
 
         for instance in context:
-            version_number = instance.data.get("version")
+            if self.follow_workfile_version:
+                version_number = context.data('version')
+            else:
+                version_number = instance.data.get("version")
             # If version is not specified for instance or context
             if version_number is None:
                 # TODO we should be able to change default version by studio

openpype/plugins/publish/validate_version.py

@@ -21,8 +21,9 @@ class ValidateVersion(pyblish.api.InstancePlugin):
 
         if latest_version is not None:
             msg = (
-                "Version `{0}` that you are trying to publish, already exists"
-                " in the database. Version in database: `{1}`. Please version "
-                "up your workfile to a higher version number than: `{1}`."
-            ).format(version, latest_version)
+                "Version `{0}` from instance `{1}` that you are trying to"
+                " publish, already exists in the database. Version in"
+                " database: `{2}`. Please version up your workfile to a higher"
+                " version number than: `{2}`."
+            ).format(version, instance.data["name"], latest_version)
             assert (int(version) > int(latest_version)), msg

openpype/pype_commands.py

@@ -41,6 +41,25 @@ class PypeCommands:
             user_role = "manager"
         settings.main(user_role)
 
+    @staticmethod
+    def add_modules(click_func):
+        """Modules/Addons can add their cli commands dynamically."""
+        from openpype.modules import ModulesManager
+
+        manager = ModulesManager()
+        log = PypeLogger.get_logger("AddModulesCLI")
+        for module in manager.modules:
+            try:
+                module.cli(click_func)
+
+            except Exception:
+                log.warning(
+                    "Failed to add cli command for module \"{}\"".format(
+                        module.name
+                    )
+                )
+        return click_func
+
     @staticmethod
     def launch_eventservercli(*args):
         from openpype_modules.ftrack.ftrack_server.event_server_cli import (

openpype/settings/defaults/project_settings/global.json

@@ -1,5 +1,8 @@
 {
     "publish": {
+        "CollectAnatomyInstanceData": {
+            "follow_workfile_version": false
+        },
         "ValidateEditorialAssetName": {
             "enabled": true,
             "optional": false

openpype/settings/defaults/project_settings/photoshop.json

@@ -30,8 +30,7 @@
         },
         "ExtractReview": {
             "jpg_options": {
-                "tags": [
-                ]
+                "tags": []
             },
             "mov_options": {
                 "tags": [

openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json

@@ -67,7 +67,17 @@
                                         "type": "list",
                                         "key": "color_code",
                                         "label": "Color codes for layers",
-                                        "object_type": "text"
+                                        "type": "enum",
+                                        "multiselection": true,
+                                        "enum_items": [
+                                            { "red": "red" },
+                                            { "orange": "orange" },
+                                            { "yellowColor": "yellow" },
+                                            { "grain": "green" },
+                                            { "blue": "blue" },
+                                            { "violet": "violet" },
+                                            { "gray": "gray" }
+                                        ]
                                     },
                                     {
                                         "type": "list",

openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json

@@ -4,6 +4,20 @@
     "key": "publish",
     "label": "Publish plugins",
     "children": [
+        {
+            "type": "dict",
+            "collapsible": true,
+            "key": "CollectAnatomyInstanceData",
+            "label": "Collect Anatomy Instance Data",
+            "is_group": true,
+            "children": [
+                {
+                    "type": "boolean",
+                    "key": "follow_workfile_version",
+                    "label": "Follow workfile version"
+                }
+            ]
+        },
         {
             "type": "dict",
             "collapsible": true,