From e4c4e1072c28b50d55185df9f85b67cba9937d76 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 22 Jul 2021 21:29:40 +0200 Subject: [PATCH 01/88] Webpublisher backend - added new command for webserver WIP webserver_cli --- openpype/cli.py | 9 + openpype/modules/webserver/webserver_cli.py | 217 ++++++++++++++++++ .../modules/webserver/webserver_module.py | 6 +- openpype/pype_commands.py | 7 + 4 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 openpype/modules/webserver/webserver_cli.py diff --git a/openpype/cli.py b/openpype/cli.py index ec5b04c468..1065152adb 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -94,6 +94,15 @@ def eventserver(debug, ) +@main.command() +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +def webpublisherwebserver(debug): + if debug: + os.environ['OPENPYPE_DEBUG'] = "3" + + PypeCommands().launch_webpublisher_webservercli() + + @main.command() @click.argument("output_json_path") @click.option("--project", help="Project name", default=None) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py new file mode 100644 index 0000000000..f3f2fc73d1 --- /dev/null +++ b/openpype/modules/webserver/webserver_cli.py @@ -0,0 +1,217 @@ +import attr +import time +import json +import datetime +from bson.objectid import ObjectId +import collections +from aiohttp.web_response import Response + +from avalon.api import AvalonMongoDB +from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint + +from openpype.api import get_hierarchy + + +class WebpublisherProjectsEndpoint(_RestApiEndpoint): + async def get(self) -> Response: + output = [] + for project_name in self.dbcon.database.collection_names(): + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + if project_doc: + ret_val = { + "id": project_doc["_id"], + "name": project_doc["name"] + } + output.append(ret_val) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +@attr.s +class AssetItem(object): + """Data class for Render Layer metadata.""" + id = attr.ib() + name = attr.ib() + + # Render Products + children = attr.ib(init=False, default=attr.Factory(list)) + + +class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + async def get(self, project_name) -> Response: + output = [] + query_projection = { + "_id": 1, + "data.tasks": 1, + "data.visualParent": 1, + "name": 1, + "type": 1, + } + + asset_docs = self.dbcon.database[project_name].find( + {"type": "asset"}, + query_projection + ) + asset_docs_by_id = { + asset_doc["_id"]: asset_doc + for asset_doc in asset_docs + } + + asset_ids = list(asset_docs_by_id.keys()) + result = [] + if asset_ids: + result = self.dbcon.database[project_name].aggregate([ + { + "$match": { + "type": "subset", + "parent": {"$in": asset_ids} + } + }, + { + "$group": { + "_id": "$parent", + "count": {"$sum": 1} + } + } + ]) + + asset_docs_by_parent_id = collections.defaultdict(list) + for asset_doc in asset_docs_by_id.values(): + parent_id = asset_doc["data"].get("visualParent") + asset_docs_by_parent_id[parent_id].append(asset_doc) + + appending_queue = collections.deque() + appending_queue.append((None, "root")) + + asset_items_by_id = {} + non_modifiable_items = set() + assets = {} + + # # # while appending_queue: + # # assets = self._recur_hiearchy(asset_docs_by_parent_id, + # # appending_queue, + # # assets, None) + # while asset_docs_by_parent_id: + # for parent_id, asset_docs in asset_items_by_id.items(): + # asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + while appending_queue: + parent_id, parent_item_name = appending_queue.popleft() + + asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + asset_item = assets.get(parent_id) + if not asset_item: + asset_item = AssetItem(str(parent_id), parent_item_name) + + for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): + child_item = AssetItem(str(asset_doc["_id"]), + asset_doc["name"]) + asset_item.children.append(child_item) + if not asset_doc["data"]["tasks"]: + appending_queue.append((asset_doc["_id"], + child_item.name)) + + else: + asset_item = child_item + for task_name, _ in asset_doc["data"]["tasks"].items(): + child_item = AssetItem(str(asset_doc["_id"]), + task_name) + asset_item.children.append(child_item) + assets[parent_id] = attr.asdict(asset_item) + + + return Response( + status=200, + body=self.resource.encode(assets), + content_type="application/json" + ) + + def _recur_hiearchy(self, asset_docs_by_parent_id, + appending_queue, assets, asset_item): + parent_id, parent_item_name = appending_queue.popleft() + + asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + + if not asset_item: + asset_item = assets.get(parent_id) + if not asset_item: + asset_item = AssetItem(str(parent_id), parent_item_name) + + for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): + child_item = AssetItem(str(asset_doc["_id"]), + asset_doc["name"]) + asset_item.children.append(child_item) + if not asset_doc["data"]["tasks"]: + appending_queue.append((asset_doc["_id"], + child_item.name)) + asset_item = child_item + assets = self._recur_hiearchy(asset_docs_by_parent_id, appending_queue, + assets, asset_item) + else: + asset_item = child_item + for task_name, _ in asset_doc["data"]["tasks"].items(): + child_item = AssetItem(str(asset_doc["_id"]), + task_name) + asset_item.children.append(child_item) + assets[asset_item.id] = attr.asdict(asset_item) + + return assets + +class RestApiResource: + def __init__(self, server_manager): + self.server_manager = server_manager + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + @staticmethod + def json_dump_handler(value): + print("valuetype:: {}".format(type(value))) + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, ObjectId): + return str(value) + raise TypeError(value) + + @classmethod + def encode(cls, data): + return json.dumps( + data, + indent=4, + default=cls.json_dump_handler + ).encode("utf-8") + + +def run_webserver(): + print("webserver") + from openpype.modules import ModulesManager + + manager = ModulesManager() + webserver_module = manager.modules_by_name["webserver"] + webserver_module.create_server_manager() + + resource = RestApiResource(webserver_module.server_manager) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/webpublisher/projects", + projects_endpoint.dispatch + ) + + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/webpublisher/hiearchy/{project_name}", + hiearchy_endpoint.dispatch + ) + + webserver_module.start_server() + while True: + time.sleep(0.5) + diff --git a/openpype/modules/webserver/webserver_module.py b/openpype/modules/webserver/webserver_module.py index b61619acde..4832038575 100644 --- a/openpype/modules/webserver/webserver_module.py +++ b/openpype/modules/webserver/webserver_module.py @@ -50,10 +50,8 @@ class WebServerModule(PypeModule, ITrayService): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) - webserver_url = "http://localhost:{}".format(self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( - webserver_url, static_prefix + os.environ["OPENPYPE_WEBSERVER_URL"], static_prefix ) def _add_listeners(self): @@ -81,6 +79,8 @@ class WebServerModule(PypeModule, ITrayService): self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) + webserver_url = "http://localhost:{}".format(self.port) + os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url @staticmethod def find_free_port( diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 7c47d8c613..6ccf10e8ce 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -40,6 +40,13 @@ class PypeCommands: ) return run_event_server(*args) + @staticmethod + def launch_webpublisher_webservercli(*args): + from openpype.modules.webserver.webserver_cli import ( + run_webserver + ) + return run_webserver(*args) + @staticmethod def launch_standalone_publisher(): from openpype.tools import standalonepublish From e4cc3033057c4e33a9a535efd79eb7c74d196f12 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Sun, 25 Jul 2021 15:50:47 +0200 Subject: [PATCH 02/88] Webpublisher backend - implemented context endopoint --- openpype/modules/webserver/webserver_cli.py | 153 ++++++++------------ 1 file changed, 59 insertions(+), 94 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index f3f2fc73d1..3ebbc86358 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -13,6 +13,7 @@ from openpype.api import get_hierarchy class WebpublisherProjectsEndpoint(_RestApiEndpoint): + """Returns list of project names.""" async def get(self) -> Response: output = [] for project_name in self.dbcon.database.collection_names(): @@ -32,23 +33,44 @@ class WebpublisherProjectsEndpoint(_RestApiEndpoint): ) -@attr.s -class AssetItem(object): - """Data class for Render Layer metadata.""" - id = attr.ib() - name = attr.ib() +class Node(dict): + """Node element in context tree.""" - # Render Products - children = attr.ib(init=False, default=attr.Factory(list)) + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + """Returns dictionary with context tree from assets.""" async def get(self, project_name) -> Response: - output = [] query_projection = { "_id": 1, "data.tasks": 1, "data.visualParent": 1, + "data.entityType": 1, "name": 1, "type": 1, } @@ -62,106 +84,51 @@ class WebpublisherHiearchyEndpoint(_RestApiEndpoint): for asset_doc in asset_docs } - asset_ids = list(asset_docs_by_id.keys()) - result = [] - if asset_ids: - result = self.dbcon.database[project_name].aggregate([ - { - "$match": { - "type": "subset", - "parent": {"$in": asset_ids} - } - }, - { - "$group": { - "_id": "$parent", - "count": {"$sum": 1} - } - } - ]) - asset_docs_by_parent_id = collections.defaultdict(list) for asset_doc in asset_docs_by_id.values(): parent_id = asset_doc["data"].get("visualParent") asset_docs_by_parent_id[parent_id].append(asset_doc) - appending_queue = collections.deque() - appending_queue.append((None, "root")) + assets = collections.defaultdict(list) - asset_items_by_id = {} - non_modifiable_items = set() - assets = {} + for parent_id, children in asset_docs_by_parent_id.items(): + for child in children: + node = assets.get(child["_id"]) + if not node: + node = Node(child["_id"], + child["data"]["entityType"], + child["name"]) + assets[child["_id"]] = node - # # # while appending_queue: - # # assets = self._recur_hiearchy(asset_docs_by_parent_id, - # # appending_queue, - # # assets, None) - # while asset_docs_by_parent_id: - # for parent_id, asset_docs in asset_items_by_id.items(): - # asset_docs = asset_docs_by_parent_id.get(parent_id) or [] + tasks = child["data"].get("tasks", {}) + for t_name, t_con in tasks.items(): + task_node = TaskNode("task", t_name) + task_node["attributes"]["type"] = t_con.get("type") - while appending_queue: - parent_id, parent_item_name = appending_queue.popleft() + task_node.parent = node - asset_docs = asset_docs_by_parent_id.get(parent_id) or [] - - asset_item = assets.get(parent_id) - if not asset_item: - asset_item = AssetItem(str(parent_id), parent_item_name) - - for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): - child_item = AssetItem(str(asset_doc["_id"]), - asset_doc["name"]) - asset_item.children.append(child_item) - if not asset_doc["data"]["tasks"]: - appending_queue.append((asset_doc["_id"], - child_item.name)) - - else: - asset_item = child_item - for task_name, _ in asset_doc["data"]["tasks"].items(): - child_item = AssetItem(str(asset_doc["_id"]), - task_name) - asset_item.children.append(child_item) - assets[parent_id] = attr.asdict(asset_item) + parent_node = assets.get(parent_id) + if not parent_node: + asset_doc = asset_docs_by_id.get(parent_id) + if asset_doc: # regular node + parent_node = Node(parent_id, + asset_doc["data"]["entityType"], + asset_doc["name"]) + else: # root + parent_node = Node(parent_id, + "project", + project_name) + assets[parent_id] = parent_node + node.parent = parent_node + roots = [x for x in assets.values() if x.parent is None] return Response( status=200, - body=self.resource.encode(assets), + body=self.resource.encode(roots[0]), content_type="application/json" ) - def _recur_hiearchy(self, asset_docs_by_parent_id, - appending_queue, assets, asset_item): - parent_id, parent_item_name = appending_queue.popleft() - - asset_docs = asset_docs_by_parent_id.get(parent_id) or [] - - if not asset_item: - asset_item = assets.get(parent_id) - if not asset_item: - asset_item = AssetItem(str(parent_id), parent_item_name) - - for asset_doc in sorted(asset_docs, key=lambda item: item["name"]): - child_item = AssetItem(str(asset_doc["_id"]), - asset_doc["name"]) - asset_item.children.append(child_item) - if not asset_doc["data"]["tasks"]: - appending_queue.append((asset_doc["_id"], - child_item.name)) - asset_item = child_item - assets = self._recur_hiearchy(asset_docs_by_parent_id, appending_queue, - assets, asset_item) - else: - asset_item = child_item - for task_name, _ in asset_doc["data"]["tasks"].items(): - child_item = AssetItem(str(asset_doc["_id"]), - task_name) - asset_item.children.append(child_item) - assets[asset_item.id] = attr.asdict(asset_item) - - return assets class RestApiResource: def __init__(self, server_manager): @@ -172,7 +139,6 @@ class RestApiResource: @staticmethod def json_dump_handler(value): - print("valuetype:: {}".format(type(value))) if isinstance(value, datetime.datetime): return value.isoformat() if isinstance(value, ObjectId): @@ -189,7 +155,6 @@ class RestApiResource: def run_webserver(): - print("webserver") from openpype.modules import ModulesManager manager = ModulesManager() From 6c32a8e6a36d11e1988933be3ed2d1c4a7c2e51e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 26 Jul 2021 10:15:23 +0200 Subject: [PATCH 03/88] Webpublisher backend - changed uri to api --- openpype/modules/webserver/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 3ebbc86358..17b98cc1af 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -165,14 +165,14 @@ def run_webserver(): projects_endpoint = WebpublisherProjectsEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/webpublisher/projects", + "/api/projects", projects_endpoint.dispatch ) hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/webpublisher/hiearchy/{project_name}", + "/api/hiearchy/{project_name}", hiearchy_endpoint.dispatch ) From 622ff2a797bcf9b5954f5b0f80ad06482576521a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 26 Jul 2021 11:25:52 +0200 Subject: [PATCH 04/88] Webpublisher backend - changed uri to api --- openpype/modules/webserver/webserver_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 17b98cc1af..b6317a5675 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -172,7 +172,7 @@ def run_webserver(): hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) webserver_module.server_manager.add_route( "GET", - "/api/hiearchy/{project_name}", + "/api/hierarchy/{project_name}", hiearchy_endpoint.dispatch ) From 32a82b50f4386d6756c24bdf17f3e02f606dd5f1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jul 2021 18:47:15 +0200 Subject: [PATCH 05/88] Webpublisher - backend - added webpublisher host --- openpype/hosts/webpublisher/README.md | 6 + openpype/hosts/webpublisher/__init__.py | 0 .../plugins/collect_published_files.py | 159 ++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 openpype/hosts/webpublisher/README.md create mode 100644 openpype/hosts/webpublisher/__init__.py create mode 100644 openpype/hosts/webpublisher/plugins/collect_published_files.py diff --git a/openpype/hosts/webpublisher/README.md b/openpype/hosts/webpublisher/README.md new file mode 100644 index 0000000000..0826e44490 --- /dev/null +++ b/openpype/hosts/webpublisher/README.md @@ -0,0 +1,6 @@ +Webpublisher +------------- + +Plugins meant for processing of Webpublisher. + +Gets triggered by calling openpype.cli.remotepublish with appropriate arguments. \ No newline at end of file diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpype/hosts/webpublisher/plugins/collect_published_files.py b/openpype/hosts/webpublisher/plugins/collect_published_files.py new file mode 100644 index 0000000000..1cc0dfe83f --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/collect_published_files.py @@ -0,0 +1,159 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import os +import json + +import pyblish.api +from avalon import api + + +class CollectPublishedFiles(pyblish.api.ContextPlugin): + """ + This collector will try to find json files in provided + `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. + + """ + # must be really early, context values are only in json file + order = pyblish.api.CollectorOrder - 0.495 + label = "Collect rendered frames" + host = ["webpublisher"] + + _context = None + + def _load_json(self, path): + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + self.log.error( + "Error loading json: " + "{} - Exception: {}".format(path, exc) + ) + return data + + def _fill_staging_dir(self, data_object, anatomy): + staging_dir = data_object.get("stagingDir") + if staging_dir: + data_object["stagingDir"] = anatomy.fill_root(staging_dir) + + def _process_path(self, data, anatomy): + # validate basic necessary data + data_err = "invalid json file - missing data" + required = ["asset", "user", "comment", + "job", "instances", "session", "version"] + assert all(elem in data.keys() for elem in required), data_err + + # set context by first json file + ctx = self._context.data + + ctx["asset"] = ctx.get("asset") or data.get("asset") + ctx["intent"] = ctx.get("intent") or data.get("intent") + ctx["comment"] = ctx.get("comment") or data.get("comment") + ctx["user"] = ctx.get("user") or data.get("user") + ctx["version"] = ctx.get("version") or data.get("version") + + # basic sanity check to see if we are working in same context + # if some other json file has different context, bail out. + ctx_err = "inconsistent contexts in json files - %s" + assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" + assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" + assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" + assert ctx.get("user") == data.get("user"), ctx_err % "user" + assert ctx.get("version") == data.get("version"), ctx_err % "version" + + # ftrack credentials are passed as environment variables by Deadline + # to publish job, but Muster doesn't pass them. + if data.get("ftrack") and not os.environ.get("FTRACK_API_USER"): + ftrack = data.get("ftrack") + os.environ["FTRACK_API_USER"] = ftrack["FTRACK_API_USER"] + os.environ["FTRACK_API_KEY"] = ftrack["FTRACK_API_KEY"] + os.environ["FTRACK_SERVER"] = ftrack["FTRACK_SERVER"] + + # now we can just add instances from json file and we are done + for instance_data in data.get("instances"): + self.log.info(" - processing instance for {}".format( + instance_data.get("subset"))) + instance = self._context.create_instance( + instance_data.get("subset") + ) + self.log.info("Filling stagingDir...") + + self._fill_staging_dir(instance_data, anatomy) + instance.data.update(instance_data) + + # stash render job id for later validation + instance.data["render_job_id"] = data.get("job").get("_id") + + representations = [] + for repre_data in instance_data.get("representations") or []: + self._fill_staging_dir(repre_data, anatomy) + representations.append(repre_data) + + instance.data["representations"] = representations + + # add audio if in metadata data + if data.get("audio"): + instance.data.update({ + "audio": [{ + "filename": data.get("audio"), + "offset": 0 + }] + }) + self.log.info( + f"Adding audio to instance: {instance.data['audio']}") + + def process(self, context): + self._context = context + + assert os.environ.get("OPENPYPE_PUBLISH_DATA"), ( + "Missing `OPENPYPE_PUBLISH_DATA`") + paths = os.environ["OPENPYPE_PUBLISH_DATA"].split(os.pathsep) + + project_name = os.environ.get("AVALON_PROJECT") + if project_name is None: + raise AssertionError( + "Environment `AVALON_PROJECT` was not found." + "Could not set project `root` which may cause issues." + ) + + # TODO root filling should happen after collect Anatomy + self.log.info("Getting root setting for project \"{}\"".format( + project_name + )) + + anatomy = context.data["anatomy"] + self.log.info("anatomy: {}".format(anatomy.roots)) + try: + session_is_set = False + for path in paths: + path = anatomy.fill_root(path) + data = self._load_json(path) + assert data, "failed to load json file" + if not session_is_set: + session_data = data["session"] + remapped = anatomy.roots_obj.path_remapper( + session_data["AVALON_WORKDIR"] + ) + if remapped: + session_data["AVALON_WORKDIR"] = remapped + + self.log.info("Setting session using data from file") + api.Session.update(session_data) + os.environ.update(session_data) + session_is_set = True + self._process_path(data, anatomy) + except Exception as e: + self.log.error(e, exc_info=True) + raise Exception("Error") from e From ca1ad20506c99b00412091a42a5cfe8ef28af7bd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 27 Jul 2021 18:47:58 +0200 Subject: [PATCH 06/88] Webpublisher - backend - added task_finish endpoint Added scaffolding to run publish process --- openpype/cli.py | 27 ++++++++- openpype/lib/applications.py | 20 ++++--- openpype/modules/webserver/webserver_cli.py | 62 +++++++++++++++++++-- openpype/pype_commands.py | 62 ++++++++++++++++++++- 4 files changed, 151 insertions(+), 20 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 1065152adb..e56a572c9c 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -96,11 +96,16 @@ def eventserver(debug, @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") -def webpublisherwebserver(debug): +@click.option("-e", "--executable", help="Executable") +@click.option("-u", "--upload_dir", help="Upload dir") +def webpublisherwebserver(debug, executable, upload_dir): if debug: os.environ['OPENPYPE_DEBUG'] = "3" - PypeCommands().launch_webpublisher_webservercli() + PypeCommands().launch_webpublisher_webservercli( + upload_dir=upload_dir, + executable=executable + ) @main.command() @@ -140,6 +145,24 @@ def publish(debug, paths, targets): PypeCommands.publish(list(paths), targets) +@main.command() +@click.argument("paths", nargs=-1) +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("-h", "--host", help="Host") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets module", default=None, + multiple=True) +def remotepublish(debug, project, paths, host, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + if debug: + os.environ['OPENPYPE_DEBUG'] = '3' + PypeCommands.remotepublish(project, list(paths), host, targets=None) + + @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("-p", "--project", required=True, diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index fb86d06150..1d0d5dcbaa 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1004,7 +1004,7 @@ class EnvironmentPrepData(dict): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name, env=None + project_name, asset_name, task_name, app_name=None, env=None ): """Prepare environment variables by context. Args: @@ -1033,20 +1033,14 @@ def get_app_environments_for_context( "name": asset_name }) - # Prepare app object which can be obtained only from ApplciationManager - app_manager = ApplicationManager() - app = app_manager.applications[app_name] - # Project's anatomy anatomy = Anatomy(project_name) - data = EnvironmentPrepData({ + prep_dict = { "project_name": project_name, "asset_name": asset_name, "task_name": task_name, - "app": app, - "dbcon": dbcon, "project_doc": project_doc, "asset_doc": asset_doc, @@ -1054,7 +1048,15 @@ def get_app_environments_for_context( "anatomy": anatomy, "env": env - }) + } + + if app_name: + # Prepare app object which can be obtained only from ApplicationManager + app_manager = ApplicationManager() + app = app_manager.applications[app_name] + prep_dict["app"] = app + + data = EnvironmentPrepData(prep_dict) prepare_host_environments(data) prepare_context_environments(data) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index b6317a5675..00caa24d27 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,16 +1,15 @@ -import attr +import os import time import json import datetime from bson.objectid import ObjectId import collections from aiohttp.web_response import Response +import subprocess from avalon.api import AvalonMongoDB from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint -from openpype.api import get_hierarchy - class WebpublisherProjectsEndpoint(_RestApiEndpoint): """Returns list of project names.""" @@ -130,9 +129,51 @@ class WebpublisherHiearchyEndpoint(_RestApiEndpoint): ) +class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def post(self, request) -> Response: + output = {} + + print(request) + + json_path = os.path.join(self.resource.upload_dir, + "webpublisher.json") # temp - pull from request + + openpype_app = self.resource.executable + args = [ + openpype_app, + 'remotepublish', + json_path + ] + + if not openpype_app or not os.path.exists(openpype_app): + msg = "Non existent OpenPype executable {}".format(openpype_app) + raise RuntimeError(msg) + + add_args = { + "host": "webpublisher", + "project": request.query["project"] + } + + for key, value in add_args.items(): + args.append("--{}".format(key)) + args.append(value) + + print("args:: {}".format(args)) + + exit_code = subprocess.call(args, shell=True) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + class RestApiResource: - def __init__(self, server_manager): + def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager + self.upload_dir = upload_dir + self.executable = executable self.dbcon = AvalonMongoDB() self.dbcon.install() @@ -154,14 +195,16 @@ class RestApiResource: ).encode("utf-8") -def run_webserver(): +def run_webserver(*args, **kwargs): from openpype.modules import ModulesManager manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - resource = RestApiResource(webserver_module.server_manager) + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) projects_endpoint = WebpublisherProjectsEndpoint(resource) webserver_module.server_manager.add_route( "GET", @@ -176,6 +219,13 @@ def run_webserver(): hiearchy_endpoint.dispatch ) + task_finish_endpoint = WebpublisherTaskFinishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/task_finish", + task_finish_endpoint.dispatch + ) + webserver_module.start_server() while True: time.sleep(0.5) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 6ccf10e8ce..d2726fd2a6 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -41,11 +41,11 @@ class PypeCommands: return run_event_server(*args) @staticmethod - def launch_webpublisher_webservercli(*args): + def launch_webpublisher_webservercli(*args, **kwargs): from openpype.modules.webserver.webserver_cli import ( run_webserver ) - return run_webserver(*args) + return run_webserver(*args, **kwargs) @staticmethod def launch_standalone_publisher(): @@ -53,7 +53,7 @@ class PypeCommands: standalonepublish.main() @staticmethod - def publish(paths, targets=None): + def publish(paths, targets=None, host=None): """Start headless publishing. Publish use json from passed paths argument. @@ -111,6 +111,62 @@ class PypeCommands: log.info("Publish finished.") uninstall() + @staticmethod + def remotepublish(project, paths, host, targets=None): + """Start headless publishing. + + Publish use json from passed paths argument. + + Args: + paths (list): Paths to jsons. + targets (string): What module should be targeted + (to choose validator for example) + host (string) + + Raises: + RuntimeError: When there is no path to process. + """ + if not any(paths): + raise RuntimeError("No publish paths specified") + + from openpype import install, uninstall + from openpype.api import Logger + + # Register target and host + import pyblish.api + import pyblish.util + + log = Logger.get_logger() + + install() + + if host: + pyblish.api.register_host(host) + + if targets: + if isinstance(targets, str): + targets = [targets] + for target in targets: + pyblish.api.register_target(target) + + os.environ["OPENPYPE_PUBLISH_DATA"] = os.pathsep.join(paths) + os.environ["AVALON_PROJECT"] = project + os.environ["AVALON_APP"] = host # to trigger proper plugings + + log.info("Running publish ...") + + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + for result in pyblish.util.publish_iter(): + if result["error"]: + log.error(error_format.format(**result)) + uninstall() + sys.exit(1) + + log.info("Publish finished.") + uninstall() + def extractenvironments(output_json_path, project, asset, task, app): env = os.environ.copy() if all((project, asset, task, app)): From 52c6bdc0e5669d729402dee9221d3d1a44087109 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:01:09 +0200 Subject: [PATCH 07/88] Webpublisher - backend - skip version collect for webpublisher --- openpype/plugins/publish/collect_scene_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_scene_version.py b/openpype/plugins/publish/collect_scene_version.py index 669e6752f3..62969858c5 100644 --- a/openpype/plugins/publish/collect_scene_version.py +++ b/openpype/plugins/publish/collect_scene_version.py @@ -16,7 +16,8 @@ class CollectSceneVersion(pyblish.api.ContextPlugin): if "standalonepublisher" in context.data.get("host", []): return - if "unreal" in pyblish.api.registered_hosts(): + if "unreal" in pyblish.api.registered_hosts() or \ + "webpublisher" in pyblish.api.registered_hosts(): return assert context.data.get('currentFile'), "Cannot get current file" From f104d601319efcec086d6a3a44a11c37ec74832e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:03:02 +0200 Subject: [PATCH 08/88] Webpublisher - backend - updated command Added logging to DB for reports --- openpype/cli.py | 7 +++-- openpype/pype_commands.py | 64 +++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index e56a572c9c..8dc32b307a 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -146,13 +146,14 @@ def publish(debug, paths, targets): @main.command() -@click.argument("paths", nargs=-1) +@click.argument("path") @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("-h", "--host", help="Host") +@click.option("-u", "--user", help="User email address") @click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets module", default=None, multiple=True) -def remotepublish(debug, project, paths, host, targets=None): +def remotepublish(debug, project, path, host, targets=None, user=None): """Start CLI publishing. Publish collects json from paths provided as an argument. @@ -160,7 +161,7 @@ def remotepublish(debug, project, paths, host, targets=None): """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands.remotepublish(project, list(paths), host, targets=None) + PypeCommands.remotepublish(project, path, host, user, targets=targets) @main.command() diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d2726fd2a6..24becd2423 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -3,7 +3,7 @@ import os import sys import json -from pathlib import Path +from datetime import datetime from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context @@ -112,25 +112,30 @@ class PypeCommands: uninstall() @staticmethod - def remotepublish(project, paths, host, targets=None): + def remotepublish(project, batch_path, host, user, targets=None): """Start headless publishing. Publish use json from passed paths argument. Args: - paths (list): Paths to jsons. + project (str): project to publish (only single context is expected + per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) targets (string): What module should be targeted (to choose validator for example) host (string) + user (string): email address for webpublisher Raises: RuntimeError: When there is no path to process. """ - if not any(paths): + if not batch_path: raise RuntimeError("No publish paths specified") from openpype import install, uninstall from openpype.api import Logger + from openpype.lib import OpenPypeMongoConnection # Register target and host import pyblish.api @@ -149,20 +154,67 @@ class PypeCommands: for target in targets: pyblish.api.register_target(target) - os.environ["OPENPYPE_PUBLISH_DATA"] = os.pathsep.join(paths) + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host # to trigger proper plugings + os.environ["AVALON_APP_NAME"] = host # to trigger proper plugings + + # this should be more generic + from openpype.hosts.webpublisher.api import install as w_install + w_install() + pyblish.api.register_host(host) log.info("Running publish ...") # Error exit as soon as any error occurs. error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + _, batch_id = os.path.split(batch_path) + _id = dbcon.insert_one({ + "batch_id": batch_id, + "start_date": datetime.now(), + "user": user, + "status": "in_progress" + }).inserted_id + for result in pyblish.util.publish_iter(): if result["error"]: log.error(error_format.format(**result)) uninstall() + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": "error", + "msg": error_format.format(**result) + } + } + ) sys.exit(1) + else: + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "progress": result["progress"] + } + } + ) + + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "state": "finished_ok", + "progress": 1 + } + } + ) log.info("Publish finished.") uninstall() From a43837ca91983d7251d1bbb8232b302abc29c950 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:03:39 +0200 Subject: [PATCH 09/88] Webpublisher - backend - added collector plugin --- openpype/hosts/webpublisher/api/__init__.py | 36 +++ .../plugins/collect_published_files.py | 159 ---------- .../publish/collect_published_files.py | 292 ++++++++++++++++++ 3 files changed, 328 insertions(+), 159 deletions(-) create mode 100644 openpype/hosts/webpublisher/api/__init__.py delete mode 100644 openpype/hosts/webpublisher/plugins/collect_published_files.py create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_published_files.py diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py new file mode 100644 index 0000000000..908c9b10be --- /dev/null +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -0,0 +1,36 @@ +import os +import logging + +from avalon import api as avalon +from pyblish import api as pyblish +import openpype.hosts.webpublisher + +log = logging.getLogger("openpype.hosts.webpublisher") + +HOST_DIR = os.path.dirname(os.path.abspath( + openpype.hosts.webpublisher.__file__)) +PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") +LOAD_PATH = os.path.join(PLUGINS_DIR, "load") +CREATE_PATH = os.path.join(PLUGINS_DIR, "create") + + +def application_launch(): + pass + + +def install(): + print("Installing Pype config...") + + pyblish.register_plugin_path(PUBLISH_PATH) + avalon.register_plugin_path(avalon.Loader, LOAD_PATH) + avalon.register_plugin_path(avalon.Creator, CREATE_PATH) + log.info(PUBLISH_PATH) + + avalon.on("application.launched", application_launch) + +def uninstall(): + pyblish.deregister_plugin_path(PUBLISH_PATH) + avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) + avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) + diff --git a/openpype/hosts/webpublisher/plugins/collect_published_files.py b/openpype/hosts/webpublisher/plugins/collect_published_files.py deleted file mode 100644 index 1cc0dfe83f..0000000000 --- a/openpype/hosts/webpublisher/plugins/collect_published_files.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Loads publishing context from json and continues in publish process. - -Requires: - anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) - -Provides: - context, instances -> All data from previous publishing process. -""" - -import os -import json - -import pyblish.api -from avalon import api - - -class CollectPublishedFiles(pyblish.api.ContextPlugin): - """ - This collector will try to find json files in provided - `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. - - """ - # must be really early, context values are only in json file - order = pyblish.api.CollectorOrder - 0.495 - label = "Collect rendered frames" - host = ["webpublisher"] - - _context = None - - def _load_json(self, path): - path = path.strip('\"') - assert os.path.isfile(path), ( - "Path to json file doesn't exist. \"{}\"".format(path) - ) - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - self.log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data - - def _fill_staging_dir(self, data_object, anatomy): - staging_dir = data_object.get("stagingDir") - if staging_dir: - data_object["stagingDir"] = anatomy.fill_root(staging_dir) - - def _process_path(self, data, anatomy): - # validate basic necessary data - data_err = "invalid json file - missing data" - required = ["asset", "user", "comment", - "job", "instances", "session", "version"] - assert all(elem in data.keys() for elem in required), data_err - - # set context by first json file - ctx = self._context.data - - ctx["asset"] = ctx.get("asset") or data.get("asset") - ctx["intent"] = ctx.get("intent") or data.get("intent") - ctx["comment"] = ctx.get("comment") or data.get("comment") - ctx["user"] = ctx.get("user") or data.get("user") - ctx["version"] = ctx.get("version") or data.get("version") - - # basic sanity check to see if we are working in same context - # if some other json file has different context, bail out. - ctx_err = "inconsistent contexts in json files - %s" - assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" - assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" - assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" - assert ctx.get("user") == data.get("user"), ctx_err % "user" - assert ctx.get("version") == data.get("version"), ctx_err % "version" - - # ftrack credentials are passed as environment variables by Deadline - # to publish job, but Muster doesn't pass them. - if data.get("ftrack") and not os.environ.get("FTRACK_API_USER"): - ftrack = data.get("ftrack") - os.environ["FTRACK_API_USER"] = ftrack["FTRACK_API_USER"] - os.environ["FTRACK_API_KEY"] = ftrack["FTRACK_API_KEY"] - os.environ["FTRACK_SERVER"] = ftrack["FTRACK_SERVER"] - - # now we can just add instances from json file and we are done - for instance_data in data.get("instances"): - self.log.info(" - processing instance for {}".format( - instance_data.get("subset"))) - instance = self._context.create_instance( - instance_data.get("subset") - ) - self.log.info("Filling stagingDir...") - - self._fill_staging_dir(instance_data, anatomy) - instance.data.update(instance_data) - - # stash render job id for later validation - instance.data["render_job_id"] = data.get("job").get("_id") - - representations = [] - for repre_data in instance_data.get("representations") or []: - self._fill_staging_dir(repre_data, anatomy) - representations.append(repre_data) - - instance.data["representations"] = representations - - # add audio if in metadata data - if data.get("audio"): - instance.data.update({ - "audio": [{ - "filename": data.get("audio"), - "offset": 0 - }] - }) - self.log.info( - f"Adding audio to instance: {instance.data['audio']}") - - def process(self, context): - self._context = context - - assert os.environ.get("OPENPYPE_PUBLISH_DATA"), ( - "Missing `OPENPYPE_PUBLISH_DATA`") - paths = os.environ["OPENPYPE_PUBLISH_DATA"].split(os.pathsep) - - project_name = os.environ.get("AVALON_PROJECT") - if project_name is None: - raise AssertionError( - "Environment `AVALON_PROJECT` was not found." - "Could not set project `root` which may cause issues." - ) - - # TODO root filling should happen after collect Anatomy - self.log.info("Getting root setting for project \"{}\"".format( - project_name - )) - - anatomy = context.data["anatomy"] - self.log.info("anatomy: {}".format(anatomy.roots)) - try: - session_is_set = False - for path in paths: - path = anatomy.fill_root(path) - data = self._load_json(path) - assert data, "failed to load json file" - if not session_is_set: - session_data = data["session"] - remapped = anatomy.roots_obj.path_remapper( - session_data["AVALON_WORKDIR"] - ) - if remapped: - session_data["AVALON_WORKDIR"] = remapped - - self.log.info("Setting session using data from file") - api.Session.update(session_data) - os.environ.update(session_data) - session_is_set = True - self._process_path(data, anatomy) - except Exception as e: - self.log.error(e, exc_info=True) - raise Exception("Error") from e diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py new file mode 100644 index 0000000000..69d30e06e1 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -0,0 +1,292 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import os +import json +import clique + +import pyblish.api +from avalon import api + +FAMILY_SETTING = { # TEMP + "Animation": { + "workfile": { + "is_sequence": False, + "extensions": ["tvp"], + "families": [] + }, + "render": { + "is_sequence": True, + "extensions": [ + "png", "exr", "tiff", "tif" + ], + "families": ["review"] + } + }, + "Compositing": { + "workfile": { + "is_sequence": False, + "extensions": ["aep"], + "families": [] + }, + "render": { + "is_sequence": True, + "extensions": [ + "png", "exr", "tiff", "tif" + ], + "families": ["review"] + } + }, + "Layout": { + "workfile": { + "is_sequence": False, + "extensions": [ + ".psd" + ], + "families": [] + }, + "image": { + "is_sequence": False, + "extensions": [ + "png", + "jpg", + "jpeg", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + } +} + +class CollectPublishedFiles(pyblish.api.ContextPlugin): + """ + This collector will try to find json files in provided + `OPENPYPE_PUBLISH_DATA`. Those files _MUST_ share same context. + + """ + # must be really early, context values are only in json file + order = pyblish.api.CollectorOrder - 0.490 + label = "Collect rendered frames" + host = ["webpublisher"] + + _context = None + + def _load_json(self, path): + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + self.log.error( + "Error loading json: " + "{} - Exception: {}".format(path, exc) + ) + return data + + def _fill_staging_dir(self, data_object, anatomy): + staging_dir = data_object.get("stagingDir") + if staging_dir: + data_object["stagingDir"] = anatomy.fill_root(staging_dir) + + def _process_path(self, data): + # validate basic necessary data + data_err = "invalid json file - missing data" + # required = ["asset", "user", "comment", + # "job", "instances", "session", "version"] + # assert all(elem in data.keys() for elem in required), data_err + + # set context by first json file + ctx = self._context.data + + ctx["asset"] = ctx.get("asset") or data.get("asset") + ctx["intent"] = ctx.get("intent") or data.get("intent") + ctx["comment"] = ctx.get("comment") or data.get("comment") + ctx["user"] = ctx.get("user") or data.get("user") + ctx["version"] = ctx.get("version") or data.get("version") + + # basic sanity check to see if we are working in same context + # if some other json file has different context, bail out. + ctx_err = "inconsistent contexts in json files - %s" + assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" + assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" + assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" + assert ctx.get("user") == data.get("user"), ctx_err % "user" + assert ctx.get("version") == data.get("version"), ctx_err % "version" + + # now we can just add instances from json file and we are done + for instance_data in data.get("instances"): + self.log.info(" - processing instance for {}".format( + instance_data.get("subset"))) + instance = self._context.create_instance( + instance_data.get("subset") + ) + self.log.info("Filling stagingDir...") + + self._fill_staging_dir(instance_data, anatomy) + instance.data.update(instance_data) + + # stash render job id for later validation + instance.data["render_job_id"] = data.get("job").get("_id") + + representations = [] + for repre_data in instance_data.get("representations") or []: + self._fill_staging_dir(repre_data, anatomy) + representations.append(repre_data) + + instance.data["representations"] = representations + + # add audio if in metadata data + if data.get("audio"): + instance.data.update({ + "audio": [{ + "filename": data.get("audio"), + "offset": 0 + }] + }) + self.log.info( + f"Adding audio to instance: {instance.data['audio']}") + + def _process_batch(self, dir_url): + task_subfolders = [os.path.join(dir_url, o) + for o in os.listdir(dir_url) + if os.path.isdir(os.path.join(dir_url, o))] + self.log.info("task_sub:: {}".format(task_subfolders)) + for task_dir in task_subfolders: + task_data = self._load_json(os.path.join(task_dir, + "manifest.json")) + self.log.info("task_data:: {}".format(task_data)) + ctx = task_data["context"] + asset = subset = task = task_type = None + + subset = "Main" # temp + if ctx["type"] == "task": + items = ctx["path"].split('/') + asset = items[-2] + os.environ["AVALON_TASK"] = ctx["name"] + task_type = ctx["attributes"]["type"] + else: + asset = ctx["name"] + + is_sequence = len(task_data["files"]) > 1 + + instance = self._context.create_instance(subset) + _, extension = os.path.splitext(task_data["files"][0]) + self.log.info("asset:: {}".format(asset)) + family, families = self._get_family(FAMILY_SETTING, # todo + task_type, + is_sequence, + extension.replace(".", '')) + os.environ["AVALON_ASSET"] = asset + instance.data["asset"] = asset + instance.data["subset"] = subset + instance.data["family"] = family + instance.data["families"] = families + # instance.data["version"] = self._get_version(task_data["subset"]) + instance.data["stagingDir"] = task_dir + instance.data["source"] = "webpublisher" + + os.environ["FTRACK_API_USER"] = task_data["user"] + + if is_sequence: + instance.data["representations"] = self._process_sequence( + task_data["files"], task_dir + ) + else: + _, ext = os.path.splittext(task_data["files"][0]) + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": task_data["files"], + "stagingDir": task_dir + } + instance.data["representation"] = repre_data + + self.log.info("instance.data:: {}".format(instance.data)) + + def _process_sequence(self, files, task_dir): + """Prepare reprentations for sequence of files.""" + collections, remainder = clique.assemble(files) + assert len(collections) == 1, \ + "Too many collections in {}".format(files) + + frame_start = list(collections[0].indexes)[0] + frame_end = list(collections[0].indexes)[-1] + ext = collections[0].tail + repre_data = { + "frameStart": frame_start, + "frameEnd": frame_end, + "name": ext[1:], + "ext": ext[1:], + "files": files, + "stagingDir": task_dir + } + self.log.info("repre_data.data:: {}".format(repre_data)) + return [repre_data] + + def _get_family(self, settings, task_type, is_sequence, extension): + """Guess family based on input data. + + Args: + settings (dict): configuration per task_type + task_type (str): Animation|Art etc + is_sequence (bool): single file or sequence + extension (str): without '.' + + Returns: + (family, [families]) tuple + AssertionError if not matching family found + """ + task_obj = settings.get(task_type) + assert task_obj, "No family configuration for '{}'".format(task_type) + + found_family = None + for family, content in task_obj.items(): + if is_sequence != content["is_sequence"]: + continue + if extension in content["extensions"]: + found_family = family + break + + msg = "No family found for combination of " +\ + "task_type: {}, is_sequence:{}, extension: {}".format( + task_type, is_sequence, extension) + assert found_family, msg + + return found_family, content["families"] + + def _get_version(self, subset_name): + return 1 + + def process(self, context): + self._context = context + + batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") + + assert batch_dir, ( + "Missing `OPENPYPE_PUBLISH_DATA`") + + assert batch_dir, \ + "Folder {} doesn't exist".format(batch_dir) + + project_name = os.environ.get("AVALON_PROJECT") + if project_name is None: + raise AssertionError( + "Environment `AVALON_PROJECT` was not found." + "Could not set project `root` which may cause issues." + ) + + self._process_batch(batch_dir) + From 824714c2f898a6eb37569d88b875a3137c16d1f9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:04:20 +0200 Subject: [PATCH 10/88] Webpublisher - backend - added endpoints for reporting --- openpype/modules/webserver/webserver_cli.py | 126 ++++++++++++++------ 1 file changed, 88 insertions(+), 38 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 00caa24d27..04d0002787 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -8,6 +8,8 @@ from aiohttp.web_response import Response import subprocess from avalon.api import AvalonMongoDB + +from openpype.lib import OpenPypeMongoConnection from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint @@ -32,36 +34,6 @@ class WebpublisherProjectsEndpoint(_RestApiEndpoint): ) -class Node(dict): - """Node element in context tree.""" - - def __init__(self, uid, node_type, name): - self._parent = None # pointer to parent Node - self["type"] = node_type - self["name"] = name - self['id'] = uid # keep reference to id # - self['children'] = [] # collection of pointers to child Nodes - - @property - def parent(self): - return self._parent # simply return the object at the _parent pointer - - @parent.setter - def parent(self, node): - self._parent = node - # add this node to parent's list of children - node['children'].append(self) - - -class TaskNode(Node): - """Special node type only for Tasks.""" - def __init__(self, node_type, name): - self._parent = None - self["type"] = node_type - self["name"] = name - self["attributes"] = {} - - class WebpublisherHiearchyEndpoint(_RestApiEndpoint): """Returns dictionary with context tree from assets.""" async def get(self, project_name) -> Response: @@ -129,21 +101,52 @@ class WebpublisherHiearchyEndpoint(_RestApiEndpoint): ) -class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): +class Node(dict): + """Node element in context tree.""" + + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} + + +class WebpublisherPublishEndpoint(_RestApiEndpoint): """Returns list of project names.""" async def post(self, request) -> Response: output = {} print(request) - json_path = os.path.join(self.resource.upload_dir, - "webpublisher.json") # temp - pull from request + batch_path = os.path.join(self.resource.upload_dir, + request.query["batch_id"]) openpype_app = self.resource.executable args = [ openpype_app, 'remotepublish', - json_path + batch_path ] if not openpype_app or not os.path.exists(openpype_app): @@ -152,7 +155,8 @@ class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): add_args = { "host": "webpublisher", - "project": request.query["project"] + "project": request.query["project"], + "user": request.query["user"] } for key, value in add_args.items(): @@ -169,6 +173,30 @@ class WebpublisherTaskFinishEndpoint(_RestApiEndpoint): ) +class BatchStatusEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def get(self, batch_id) -> Response: + output = self.dbcon.find_one({"batch_id": batch_id}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class PublishesStatusEndpoint(_RestApiEndpoint): + """Returns list of project names.""" + async def get(self, user) -> Response: + output = self.dbcon.find({"user": user}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + class RestApiResource: def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager @@ -195,6 +223,13 @@ class RestApiResource: ).encode("utf-8") +class OpenPypeRestApiResource(RestApiResource): + def __init__(self, ): + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + self.dbcon = mongo_client[database_name]["webpublishes"] + + def run_webserver(*args, **kwargs): from openpype.modules import ModulesManager @@ -219,11 +254,26 @@ def run_webserver(*args, **kwargs): hiearchy_endpoint.dispatch ) - task_finish_endpoint = WebpublisherTaskFinishEndpoint(resource) + webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", - "/api/task_finish", - task_finish_endpoint.dispatch + "/api/webpublish/{batch_id}", + webpublisher_publish_endpoint.dispatch + ) + + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) + + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch ) webserver_module.start_server() From bfd2ad65cf2877f245192ba8d28548ae1edc0ad2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 28 Jul 2021 18:10:09 +0200 Subject: [PATCH 11/88] Webpublisher - backend - hound --- openpype/hosts/webpublisher/api/__init__.py | 2 +- .../plugins/publish/collect_published_files.py | 3 +-- openpype/modules/webserver/webserver_cli.py | 3 +-- openpype/pype_commands.py | 9 +++------ 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 908c9b10be..1b6edcf24d 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,8 +29,8 @@ def install(): avalon.on("application.launched", application_launch) + def uninstall(): pyblish.deregister_plugin_path(PUBLISH_PATH) avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) - diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 69d30e06e1..dde9713c7a 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -169,7 +169,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] - asset = subset = task = task_type = None + task_type = None subset = "Main" # temp if ctx["type"] == "task": @@ -289,4 +289,3 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): ) self._process_batch(batch_dir) - diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 04d0002787..484c25c6b3 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -165,7 +165,7 @@ class WebpublisherPublishEndpoint(_RestApiEndpoint): print("args:: {}".format(args)) - exit_code = subprocess.call(args, shell=True) + _exit_code = subprocess.call(args, shell=True) return Response( status=200, body=self.resource.encode(output), @@ -279,4 +279,3 @@ def run_webserver(*args, **kwargs): webserver_module.start_server() while True: time.sleep(0.5) - diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 24becd2423..01fa6b8d33 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -191,8 +191,7 @@ class PypeCommands: "finish_date": datetime.now(), "status": "error", "msg": error_format.format(**result) - } - } + }} ) sys.exit(1) else: @@ -201,8 +200,7 @@ class PypeCommands: {"$set": { "progress": result["progress"] - } - } + }} ) dbcon.update_one( @@ -212,8 +210,7 @@ class PypeCommands: "finish_date": datetime.now(), "state": "finished_ok", "progress": 1 - } - } + }} ) log.info("Publish finished.") From 7decf0aa911a3fd18d7a91688ae39fd6f098eb13 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 11:06:17 +0200 Subject: [PATCH 12/88] Webpublisher - backend - added settings and defaults --- .../project_settings/webpublisher.json | 72 +++++++++++++++++++ .../schemas/projects_schema/schema_main.json | 4 ++ .../schema_project_webpublisher.json | 60 ++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 openpype/settings/defaults/project_settings/webpublisher.json create mode 100644 openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json new file mode 100644 index 0000000000..69b6babc64 --- /dev/null +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -0,0 +1,72 @@ +{ + "publish": { + "CollectPublishedFiles": { + "task_type_to_family": { + "Animation": { + "workfile": { + "is_sequence": false, + "extensions": [ + "tvp" + ], + "families": [] + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + }, + "Compositing": { + "workfile": { + "is_sequence": false, + "extensions": [ + "aep" + ], + "families": [] + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + }, + "Layout": { + "workfile": { + "is_sequence": false, + "extensions": [ + "psd" + ], + "families": [] + }, + "image": { + "is_sequence": false, + "extensions": [ + "png", + "jpg", + "jpeg", + "tiff", + "tif" + ], + "families": [ + "review" + ] + } + } + } + } + } +} \ No newline at end of file diff --git a/openpype/settings/entities/schemas/projects_schema/schema_main.json b/openpype/settings/entities/schemas/projects_schema/schema_main.json index 4a8a9d496e..575cfc9e72 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_main.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_main.json @@ -118,6 +118,10 @@ "type": "schema", "name": "schema_project_standalonepublisher" }, + { + "type": "schema", + "name": "schema_project_webpublisher" + }, { "type": "schema", "name": "schema_project_unreal" diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json new file mode 100644 index 0000000000..6ae82e0561 --- /dev/null +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -0,0 +1,60 @@ +{ + "type": "dict", + "collapsible": true, + "key": "webpublisher", + "label": "Web Publisher", + "is_file": true, + "children": [ + { + "type": "dict", + "collapsible": true, + "key": "publish", + "label": "Publish plugins", + "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectPublishedFiles", + "label": "Collect Published Files", + "children": [ + { + "type": "dict-modifiable", + "collapsible": true, + "key": "task_type_to_family", + "label": "Task type to family mapping", + "collapsible_key": true, + "object_type": { + "type": "dict-modifiable", + "collapsible": false, + "key": "task_type", + "collapsible_key": false, + "object_type": { + "type": "dict", + "children": [ + { + "type": "boolean", + "key": "is_sequence", + "label": "Is Sequence" + }, + { + "type": "list", + "key": "extensions", + "label": "Extensions", + "object_type": "text" + }, + { + "type": "list", + "key": "families", + "label": "Families", + "object_type": "text" + } + ] + } + } + } + ] + } + ] + } + ] +} \ No newline at end of file From 4f63e3d21ffd0e62caef178f8acb5fe2e422f8c3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 17:30:34 +0200 Subject: [PATCH 13/88] Webpublisher - backend - updated settings --- .../project_settings/webpublisher.json | 44 ++++++++++++++++--- .../schema_project_webpublisher.json | 5 +++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index 69b6babc64..8364b6a39d 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -8,7 +8,8 @@ "extensions": [ "tvp" ], - "families": [] + "families": [], + "subset_template_name": "" }, "render": { "is_sequence": true, @@ -20,7 +21,8 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } }, "Compositing": { @@ -29,7 +31,8 @@ "extensions": [ "aep" ], - "families": [] + "families": [], + "subset_template_name": "" }, "render": { "is_sequence": true, @@ -41,7 +44,8 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } }, "Layout": { @@ -50,7 +54,8 @@ "extensions": [ "psd" ], - "families": [] + "families": [], + "subset_template_name": "" }, "image": { "is_sequence": false, @@ -63,8 +68,35 @@ ], "families": [ "review" - ] + ], + "subset_template_name": "" } + }, + "default_task_type": { + "workfile": { + "is_sequence": false, + "extensions": [ + "tvp" + ], + "families": [], + "subset_template_name": "{family}{Variant}" + }, + "render": { + "is_sequence": true, + "extensions": [ + "png", + "exr", + "tiff", + "tif" + ], + "families": [ + "review" + ], + "subset_template_name": "{family}{Variant}" + } + }, + "__dynamic_keys_labels__": { + "default_task_type": "Default task type" } } } diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json index 6ae82e0561..bf59cd030e 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -47,6 +47,11 @@ "key": "families", "label": "Families", "object_type": "text" + }, + { + "type": "text", + "key": "subset_template_name", + "label": "Subset template name" } ] } From dea843d851162624e10a810d431e5ed78c1e13cb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 29 Jul 2021 18:17:19 +0200 Subject: [PATCH 14/88] Webpublisher - backend - implemented version and subset name --- openpype/hosts/webpublisher/api/__init__.py | 2 + .../publish/collect_published_files.py | 233 ++++++++---------- openpype/modules/webserver/webserver_cli.py | 13 +- openpype/pype_commands.py | 3 +- 4 files changed, 110 insertions(+), 141 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 1b6edcf24d..76709bb2d7 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -2,6 +2,7 @@ import os import logging from avalon import api as avalon +from avalon import io from pyblish import api as pyblish import openpype.hosts.webpublisher @@ -27,6 +28,7 @@ def install(): avalon.register_plugin_path(avalon.Creator, CREATE_PATH) log.info(PUBLISH_PATH) + io.install() avalon.on("application.launched", application_launch) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index dde9713c7a..deadbb856b 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -12,60 +12,9 @@ import json import clique import pyblish.api -from avalon import api +from avalon import io +from openpype.lib import prepare_template_data -FAMILY_SETTING = { # TEMP - "Animation": { - "workfile": { - "is_sequence": False, - "extensions": ["tvp"], - "families": [] - }, - "render": { - "is_sequence": True, - "extensions": [ - "png", "exr", "tiff", "tif" - ], - "families": ["review"] - } - }, - "Compositing": { - "workfile": { - "is_sequence": False, - "extensions": ["aep"], - "families": [] - }, - "render": { - "is_sequence": True, - "extensions": [ - "png", "exr", "tiff", "tif" - ], - "families": ["review"] - } - }, - "Layout": { - "workfile": { - "is_sequence": False, - "extensions": [ - ".psd" - ], - "families": [] - }, - "image": { - "is_sequence": False, - "extensions": [ - "png", - "jpg", - "jpeg", - "tiff", - "tif" - ], - "families": [ - "review" - ] - } - } -} class CollectPublishedFiles(pyblish.api.ContextPlugin): """ @@ -80,6 +29,9 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): _context = None + # from Settings + task_type_to_family = {} + def _load_json(self, path): path = path.strip('\"') assert os.path.isfile(path), ( @@ -96,69 +48,6 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): ) return data - def _fill_staging_dir(self, data_object, anatomy): - staging_dir = data_object.get("stagingDir") - if staging_dir: - data_object["stagingDir"] = anatomy.fill_root(staging_dir) - - def _process_path(self, data): - # validate basic necessary data - data_err = "invalid json file - missing data" - # required = ["asset", "user", "comment", - # "job", "instances", "session", "version"] - # assert all(elem in data.keys() for elem in required), data_err - - # set context by first json file - ctx = self._context.data - - ctx["asset"] = ctx.get("asset") or data.get("asset") - ctx["intent"] = ctx.get("intent") or data.get("intent") - ctx["comment"] = ctx.get("comment") or data.get("comment") - ctx["user"] = ctx.get("user") or data.get("user") - ctx["version"] = ctx.get("version") or data.get("version") - - # basic sanity check to see if we are working in same context - # if some other json file has different context, bail out. - ctx_err = "inconsistent contexts in json files - %s" - assert ctx.get("asset") == data.get("asset"), ctx_err % "asset" - assert ctx.get("intent") == data.get("intent"), ctx_err % "intent" - assert ctx.get("comment") == data.get("comment"), ctx_err % "comment" - assert ctx.get("user") == data.get("user"), ctx_err % "user" - assert ctx.get("version") == data.get("version"), ctx_err % "version" - - # now we can just add instances from json file and we are done - for instance_data in data.get("instances"): - self.log.info(" - processing instance for {}".format( - instance_data.get("subset"))) - instance = self._context.create_instance( - instance_data.get("subset") - ) - self.log.info("Filling stagingDir...") - - self._fill_staging_dir(instance_data, anatomy) - instance.data.update(instance_data) - - # stash render job id for later validation - instance.data["render_job_id"] = data.get("job").get("_id") - - representations = [] - for repre_data in instance_data.get("representations") or []: - self._fill_staging_dir(repre_data, anatomy) - representations.append(repre_data) - - instance.data["representations"] = representations - - # add audio if in metadata data - if data.get("audio"): - instance.data.update({ - "audio": [{ - "filename": data.get("audio"), - "offset": 0 - }] - }) - self.log.info( - f"Adding audio to instance: {instance.data['audio']}") - def _process_batch(self, dir_url): task_subfolders = [os.path.join(dir_url, o) for o in os.listdir(dir_url) @@ -169,32 +58,41 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] - task_type = None + task_type = "default_task_type" + task_name = None subset = "Main" # temp if ctx["type"] == "task": items = ctx["path"].split('/') asset = items[-2] os.environ["AVALON_TASK"] = ctx["name"] + task_name = ctx["name"] task_type = ctx["attributes"]["type"] else: asset = ctx["name"] is_sequence = len(task_data["files"]) > 1 - instance = self._context.create_instance(subset) _, extension = os.path.splitext(task_data["files"][0]) self.log.info("asset:: {}".format(asset)) - family, families = self._get_family(FAMILY_SETTING, # todo - task_type, - is_sequence, - extension.replace(".", '')) + family, families, subset_template = self._get_family( + self.task_type_to_family, + task_type, + is_sequence, + extension.replace(".", '')) + + subset = self._get_subset_name(family, subset_template, task_name, + task_data["variant"]) + os.environ["AVALON_ASSET"] = asset + io.Session["AVALON_ASSET"] = asset + + instance = self._context.create_instance(subset) instance.data["asset"] = asset instance.data["subset"] = subset instance.data["family"] = family instance.data["families"] = families - # instance.data["version"] = self._get_version(task_data["subset"]) + instance.data["version"] = self._get_version(asset, subset) + 1 instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" @@ -205,17 +103,33 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): task_data["files"], task_dir ) else: - _, ext = os.path.splittext(task_data["files"][0]) - repre_data = { - "name": ext[1:], - "ext": ext[1:], - "files": task_data["files"], - "stagingDir": task_dir - } - instance.data["representation"] = repre_data + + instance.data["representation"] = self._get_single_repre( + task_dir, task_data["files"] + ) self.log.info("instance.data:: {}".format(instance.data)) + def _get_subset_name(self, family, subset_template, task_name, variant): + fill_pairs = { + "variant": variant, + "family": family, + "task": task_name + } + subset = subset_template.format(**prepare_template_data(fill_pairs)) + return subset + + def _get_single_repre(self, task_dir, files): + _, ext = os.path.splittext(files[0]) + repre_data = { + "name": ext[1:], + "ext": ext[1:], + "files": files, + "stagingDir": task_dir + } + + return repre_data + def _process_sequence(self, files, task_dir): """Prepare reprentations for sequence of files.""" collections, remainder = clique.assemble(files) @@ -246,7 +160,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): extension (str): without '.' Returns: - (family, [families]) tuple + (family, [families], subset_template_name) tuple AssertionError if not matching family found """ task_obj = settings.get(task_type) @@ -265,10 +179,59 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): task_type, is_sequence, extension) assert found_family, msg - return found_family, content["families"] + return found_family, \ + content["families"], \ + content["subset_template_name"] - def _get_version(self, subset_name): - return 1 + def _get_version(self, asset_name, subset_name): + """Returns version number or 0 for 'asset' and 'subset'""" + query = [ + { + "$match": {"type": "asset", "name": asset_name} + }, + { + "$lookup": + { + "from": os.environ["AVALON_PROJECT"], + "localField": "_id", + "foreignField": "parent", + "as": "subsets" + } + }, + { + "$unwind": "$subsets" + }, + { + "$match": {"subsets.type": "subset", + "subsets.name": subset_name}}, + { + "$lookup": + { + "from": os.environ["AVALON_PROJECT"], + "localField": "subsets._id", + "foreignField": "parent", + "as": "versions" + } + }, + { + "$unwind": "$versions" + }, + { + "$group": { + "_id": { + "asset_name": "$name", + "subset_name": "$subsets.name" + }, + 'version': {'$max': "$versions.name"} + } + } + ] + version = list(io.aggregate(query)) + + if version: + return version[0].get("version") or 0 + else: + return 0 def process(self, context): self._context = context diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 484c25c6b3..7773bde567 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -146,7 +146,8 @@ class WebpublisherPublishEndpoint(_RestApiEndpoint): args = [ openpype_app, 'remotepublish', - batch_path + batch_id, + task_id ] if not openpype_app or not os.path.exists(openpype_app): @@ -174,7 +175,7 @@ class WebpublisherPublishEndpoint(_RestApiEndpoint): class BatchStatusEndpoint(_RestApiEndpoint): - """Returns list of project names.""" + """Returns dict with info for batch_id.""" async def get(self, batch_id) -> Response: output = self.dbcon.find_one({"batch_id": batch_id}) @@ -186,9 +187,9 @@ class BatchStatusEndpoint(_RestApiEndpoint): class PublishesStatusEndpoint(_RestApiEndpoint): - """Returns list of project names.""" + """Returns list of dict with batch info for user (email address).""" async def get(self, user) -> Response: - output = self.dbcon.find({"user": user}) + output = list(self.dbcon.find({"user": user})) return Response( status=200, @@ -198,6 +199,7 @@ class PublishesStatusEndpoint(_RestApiEndpoint): class RestApiResource: + """Resource carrying needed info and Avalon DB connection for publish.""" def __init__(self, server_manager, executable, upload_dir): self.server_manager = server_manager self.upload_dir = upload_dir @@ -224,6 +226,7 @@ class RestApiResource: class OpenPypeRestApiResource(RestApiResource): + """Resource carrying OP DB connection for storing batch info into DB.""" def __init__(self, ): mongo_client = OpenPypeMongoConnection.get_mongo_client() database_name = os.environ["OPENPYPE_DATABASE_NAME"] @@ -254,6 +257,7 @@ def run_webserver(*args, **kwargs): hiearchy_endpoint.dispatch ) + # triggers publish webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", @@ -261,6 +265,7 @@ def run_webserver(*args, **kwargs): webpublisher_publish_endpoint.dispatch ) + # reporting openpype_resource = OpenPypeRestApiResource() batch_status_endpoint = BatchStatusEndpoint(openpype_resource) webserver_module.server_manager.add_route( diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 01fa6b8d33..1391c36661 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -161,7 +161,6 @@ class PypeCommands: # this should be more generic from openpype.hosts.webpublisher.api import install as w_install w_install() - pyblish.api.register_host(host) log.info("Running publish ...") @@ -199,7 +198,7 @@ class PypeCommands: {"_id": _id}, {"$set": { - "progress": result["progress"] + "progress": max(result["progress"], 0.95) }} ) From 3c7f6a89fe7e72fd808d23c975306a800126579a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 10:22:16 +0200 Subject: [PATCH 15/88] Webpublisher - backend - refactored routes --- .../modules/webserver/webpublish_routes.py | 242 ++++++++++++++++ openpype/modules/webserver/webserver_cli.py | 258 ++---------------- 2 files changed, 265 insertions(+), 235 deletions(-) create mode 100644 openpype/modules/webserver/webpublish_routes.py diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py new file mode 100644 index 0000000000..805ac11a54 --- /dev/null +++ b/openpype/modules/webserver/webpublish_routes.py @@ -0,0 +1,242 @@ +"""Routes and etc. for webpublisher API.""" +import os +import json +import datetime +from bson.objectid import ObjectId +import collections +from aiohttp.web_response import Response +import subprocess + +from avalon.api import AvalonMongoDB + +from openpype.lib import OpenPypeMongoConnection +from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint + + +class RestApiResource: + """Resource carrying needed info and Avalon DB connection for publish.""" + def __init__(self, server_manager, executable, upload_dir): + self.server_manager = server_manager + self.upload_dir = upload_dir + self.executable = executable + + self.dbcon = AvalonMongoDB() + self.dbcon.install() + + @staticmethod + def json_dump_handler(value): + if isinstance(value, datetime.datetime): + return value.isoformat() + if isinstance(value, ObjectId): + return str(value) + raise TypeError(value) + + @classmethod + def encode(cls, data): + return json.dumps( + data, + indent=4, + default=cls.json_dump_handler + ).encode("utf-8") + + +class OpenPypeRestApiResource(RestApiResource): + """Resource carrying OP DB connection for storing batch info into DB.""" + def __init__(self, ): + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + self.dbcon = mongo_client[database_name]["webpublishes"] + + +class WebpublisherProjectsEndpoint(_RestApiEndpoint): + """Returns list of dict with project info (id, name).""" + async def get(self) -> Response: + output = [] + for project_name in self.dbcon.database.collection_names(): + project_doc = self.dbcon.database[project_name].find_one({ + "type": "project" + }) + if project_doc: + ret_val = { + "id": project_doc["_id"], + "name": project_doc["name"] + } + output.append(ret_val) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class WebpublisherHiearchyEndpoint(_RestApiEndpoint): + """Returns dictionary with context tree from assets.""" + async def get(self, project_name) -> Response: + query_projection = { + "_id": 1, + "data.tasks": 1, + "data.visualParent": 1, + "data.entityType": 1, + "name": 1, + "type": 1, + } + + asset_docs = self.dbcon.database[project_name].find( + {"type": "asset"}, + query_projection + ) + asset_docs_by_id = { + asset_doc["_id"]: asset_doc + for asset_doc in asset_docs + } + + asset_docs_by_parent_id = collections.defaultdict(list) + for asset_doc in asset_docs_by_id.values(): + parent_id = asset_doc["data"].get("visualParent") + asset_docs_by_parent_id[parent_id].append(asset_doc) + + assets = collections.defaultdict(list) + + for parent_id, children in asset_docs_by_parent_id.items(): + for child in children: + node = assets.get(child["_id"]) + if not node: + node = Node(child["_id"], + child["data"]["entityType"], + child["name"]) + assets[child["_id"]] = node + + tasks = child["data"].get("tasks", {}) + for t_name, t_con in tasks.items(): + task_node = TaskNode("task", t_name) + task_node["attributes"]["type"] = t_con.get("type") + + task_node.parent = node + + parent_node = assets.get(parent_id) + if not parent_node: + asset_doc = asset_docs_by_id.get(parent_id) + if asset_doc: # regular node + parent_node = Node(parent_id, + asset_doc["data"]["entityType"], + asset_doc["name"]) + else: # root + parent_node = Node(parent_id, + "project", + project_name) + assets[parent_id] = parent_node + node.parent = parent_node + + roots = [x for x in assets.values() if x.parent is None] + + return Response( + status=200, + body=self.resource.encode(roots[0]), + content_type="application/json" + ) + + +class Node(dict): + """Node element in context tree.""" + + def __init__(self, uid, node_type, name): + self._parent = None # pointer to parent Node + self["type"] = node_type + self["name"] = name + self['id'] = uid # keep reference to id # + self['children'] = [] # collection of pointers to child Nodes + + @property + def parent(self): + return self._parent # simply return the object at the _parent pointer + + @parent.setter + def parent(self, node): + self._parent = node + # add this node to parent's list of children + node['children'].append(self) + + +class TaskNode(Node): + """Special node type only for Tasks.""" + + def __init__(self, node_type, name): + self._parent = None + self["type"] = node_type + self["name"] = name + self["attributes"] = {} + + +class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): + """Triggers headless publishing of batch.""" + async def post(self, request) -> Response: + output = {} + + print(request) + + batch_path = os.path.join(self.resource.upload_dir, + request.query["batch_id"]) + + openpype_app = self.resource.executable + args = [ + openpype_app, + 'remotepublish', + batch_path + ] + + if not openpype_app or not os.path.exists(openpype_app): + msg = "Non existent OpenPype executable {}".format(openpype_app) + raise RuntimeError(msg) + + add_args = { + "host": "webpublisher", + "project": request.query["project"], + "user": request.query["user"] + } + + for key, value in add_args.items(): + args.append("--{}".format(key)) + args.append(value) + + print("args:: {}".format(args)) + + _exit_code = subprocess.call(args, shell=True) + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class WebpublisherTaskPublishEndpoint(_RestApiEndpoint): + """Prepared endpoint triggered after each task - for future development.""" + async def post(self, request) -> Response: + return Response( + status=200, + body=self.resource.encode([]), + content_type="application/json" + ) + + +class BatchStatusEndpoint(_RestApiEndpoint): + """Returns dict with info for batch_id.""" + async def get(self, batch_id) -> Response: + output = self.dbcon.find_one({"batch_id": batch_id}) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) + + +class PublishesStatusEndpoint(_RestApiEndpoint): + """Returns list of dict with batch info for user (email address).""" + async def get(self, user) -> Response: + output = list(self.dbcon.find({"user": user})) + + return Response( + status=200, + body=self.resource.encode(output), + content_type="application/json" + ) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 7773bde567..0812bfa372 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,239 +1,18 @@ -import os import time -import json -import datetime -from bson.objectid import ObjectId -import collections -from aiohttp.web_response import Response -import subprocess - -from avalon.api import AvalonMongoDB - -from openpype.lib import OpenPypeMongoConnection -from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint - - -class WebpublisherProjectsEndpoint(_RestApiEndpoint): - """Returns list of project names.""" - async def get(self) -> Response: - output = [] - for project_name in self.dbcon.database.collection_names(): - project_doc = self.dbcon.database[project_name].find_one({ - "type": "project" - }) - if project_doc: - ret_val = { - "id": project_doc["_id"], - "name": project_doc["name"] - } - output.append(ret_val) - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class WebpublisherHiearchyEndpoint(_RestApiEndpoint): - """Returns dictionary with context tree from assets.""" - async def get(self, project_name) -> Response: - query_projection = { - "_id": 1, - "data.tasks": 1, - "data.visualParent": 1, - "data.entityType": 1, - "name": 1, - "type": 1, - } - - asset_docs = self.dbcon.database[project_name].find( - {"type": "asset"}, - query_projection - ) - asset_docs_by_id = { - asset_doc["_id"]: asset_doc - for asset_doc in asset_docs - } - - asset_docs_by_parent_id = collections.defaultdict(list) - for asset_doc in asset_docs_by_id.values(): - parent_id = asset_doc["data"].get("visualParent") - asset_docs_by_parent_id[parent_id].append(asset_doc) - - assets = collections.defaultdict(list) - - for parent_id, children in asset_docs_by_parent_id.items(): - for child in children: - node = assets.get(child["_id"]) - if not node: - node = Node(child["_id"], - child["data"]["entityType"], - child["name"]) - assets[child["_id"]] = node - - tasks = child["data"].get("tasks", {}) - for t_name, t_con in tasks.items(): - task_node = TaskNode("task", t_name) - task_node["attributes"]["type"] = t_con.get("type") - - task_node.parent = node - - parent_node = assets.get(parent_id) - if not parent_node: - asset_doc = asset_docs_by_id.get(parent_id) - if asset_doc: # regular node - parent_node = Node(parent_id, - asset_doc["data"]["entityType"], - asset_doc["name"]) - else: # root - parent_node = Node(parent_id, - "project", - project_name) - assets[parent_id] = parent_node - node.parent = parent_node - - roots = [x for x in assets.values() if x.parent is None] - - return Response( - status=200, - body=self.resource.encode(roots[0]), - content_type="application/json" - ) - - -class Node(dict): - """Node element in context tree.""" - - def __init__(self, uid, node_type, name): - self._parent = None # pointer to parent Node - self["type"] = node_type - self["name"] = name - self['id'] = uid # keep reference to id # - self['children'] = [] # collection of pointers to child Nodes - - @property - def parent(self): - return self._parent # simply return the object at the _parent pointer - - @parent.setter - def parent(self, node): - self._parent = node - # add this node to parent's list of children - node['children'].append(self) - - -class TaskNode(Node): - """Special node type only for Tasks.""" - - def __init__(self, node_type, name): - self._parent = None - self["type"] = node_type - self["name"] = name - self["attributes"] = {} - - -class WebpublisherPublishEndpoint(_RestApiEndpoint): - """Returns list of project names.""" - async def post(self, request) -> Response: - output = {} - - print(request) - - batch_path = os.path.join(self.resource.upload_dir, - request.query["batch_id"]) - - openpype_app = self.resource.executable - args = [ - openpype_app, - 'remotepublish', - batch_id, - task_id - ] - - if not openpype_app or not os.path.exists(openpype_app): - msg = "Non existent OpenPype executable {}".format(openpype_app) - raise RuntimeError(msg) - - add_args = { - "host": "webpublisher", - "project": request.query["project"], - "user": request.query["user"] - } - - for key, value in add_args.items(): - args.append("--{}".format(key)) - args.append(value) - - print("args:: {}".format(args)) - - _exit_code = subprocess.call(args, shell=True) - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class BatchStatusEndpoint(_RestApiEndpoint): - """Returns dict with info for batch_id.""" - async def get(self, batch_id) -> Response: - output = self.dbcon.find_one({"batch_id": batch_id}) - - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class PublishesStatusEndpoint(_RestApiEndpoint): - """Returns list of dict with batch info for user (email address).""" - async def get(self, user) -> Response: - output = list(self.dbcon.find({"user": user})) - - return Response( - status=200, - body=self.resource.encode(output), - content_type="application/json" - ) - - -class RestApiResource: - """Resource carrying needed info and Avalon DB connection for publish.""" - def __init__(self, server_manager, executable, upload_dir): - self.server_manager = server_manager - self.upload_dir = upload_dir - self.executable = executable - - self.dbcon = AvalonMongoDB() - self.dbcon.install() - - @staticmethod - def json_dump_handler(value): - if isinstance(value, datetime.datetime): - return value.isoformat() - if isinstance(value, ObjectId): - return str(value) - raise TypeError(value) - - @classmethod - def encode(cls, data): - return json.dumps( - data, - indent=4, - default=cls.json_dump_handler - ).encode("utf-8") - - -class OpenPypeRestApiResource(RestApiResource): - """Resource carrying OP DB connection for storing batch info into DB.""" - def __init__(self, ): - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - self.dbcon = mongo_client[database_name]["webpublishes"] +from .webpublish_routes import ( + RestApiResource, + OpenPypeRestApiResource, + WebpublisherBatchPublishEndpoint, + WebpublisherTaskPublishEndpoint, + WebpublisherHiearchyEndpoint, + WebpublisherProjectsEndpoint, + BatchStatusEndpoint, + PublishesStatusEndpoint +) def run_webserver(*args, **kwargs): + """Runs webserver in command line, adds routes.""" from openpype.modules import ModulesManager manager = ModulesManager() @@ -258,11 +37,20 @@ def run_webserver(*args, **kwargs): ) # triggers publish - webpublisher_publish_endpoint = WebpublisherPublishEndpoint(resource) + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) webserver_module.server_manager.add_route( "POST", - "/api/webpublish/{batch_id}", - webpublisher_publish_endpoint.dispatch + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) + + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch ) # reporting From 349ddf6d915dff324827bf891b71f4a3026841ab Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 10:22:29 +0200 Subject: [PATCH 16/88] Webpublisher - backend - fix signature --- openpype/pype_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 1391c36661..a4a5cf7a4b 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -215,6 +215,7 @@ class PypeCommands: log.info("Publish finished.") uninstall() + @staticmethod def extractenvironments(output_json_path, project, asset, task, app): env = os.environ.copy() if all((project, asset, task, app)): From f12df9af7bbccc2487e9e491413c2df0b8b1be77 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:44:04 +0200 Subject: [PATCH 17/88] Webpublisher - backend - fix entityType as optional Fix payload for WebpublisherBatchPublishEndpoint --- openpype/modules/webserver/webpublish_routes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index 805ac11a54..cf6e4920b6 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -102,7 +102,7 @@ class WebpublisherHiearchyEndpoint(_RestApiEndpoint): node = assets.get(child["_id"]) if not node: node = Node(child["_id"], - child["data"]["entityType"], + child["data"].get("entityType", "Folder"), child["name"]) assets[child["_id"]] = node @@ -118,7 +118,8 @@ class WebpublisherHiearchyEndpoint(_RestApiEndpoint): asset_doc = asset_docs_by_id.get(parent_id) if asset_doc: # regular node parent_node = Node(parent_id, - asset_doc["data"]["entityType"], + asset_doc["data"].get("entityType", + "Folder"), asset_doc["name"]) else: # root parent_node = Node(parent_id, @@ -173,9 +174,10 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): output = {} print(request) + content = await request.json() batch_path = os.path.join(self.resource.upload_dir, - request.query["batch_id"]) + content["batch"]) openpype_app = self.resource.executable args = [ @@ -190,8 +192,8 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): add_args = { "host": "webpublisher", - "project": request.query["project"], - "user": request.query["user"] + "project": content["project_name"], + "user": content["user"] } for key, value in add_args.items(): From 61be1cbb14b82e986bdbb9f650c50b0bd279183d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:44:32 +0200 Subject: [PATCH 18/88] Webpublisher - backend - fix app name --- openpype/pype_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index a4a5cf7a4b..513d7d0865 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -156,7 +156,7 @@ class PypeCommands: os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP_NAME"] = host # to trigger proper plugings + os.environ["AVALON_APP"] = host # to trigger proper plugings # this should be more generic from openpype.hosts.webpublisher.api import install as w_install From 59ff9225d1a659ea2a84b019cddb93261c887143 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 30 Jul 2021 15:45:15 +0200 Subject: [PATCH 19/88] Webpublisher - backend - set to session for Ftrack family collector --- openpype/hosts/webpublisher/api/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 76709bb2d7..1bf1ef1a6f 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,6 +29,7 @@ def install(): log.info(PUBLISH_PATH) io.install() + avalon.Session["AVALON_APP"] = "webpublisher" # because of Ftrack collect avalon.on("application.launched", application_launch) From 276482e43520dd40fd15e880b86b3eac05fcc310 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 2 Aug 2021 17:16:23 +0200 Subject: [PATCH 20/88] Webpublisher - backend - fixes for single file publish --- .../plugins/publish/collect_published_files.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index deadbb856b..67d743278b 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -61,7 +61,6 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): task_type = "default_task_type" task_name = None - subset = "Main" # temp if ctx["type"] == "task": items = ctx["path"].split('/') asset = items[-2] @@ -74,7 +73,6 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): is_sequence = len(task_data["files"]) > 1 _, extension = os.path.splitext(task_data["files"][0]) - self.log.info("asset:: {}".format(asset)) family, families, subset_template = self._get_family( self.task_type_to_family, task_type, @@ -103,8 +101,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): task_data["files"], task_dir ) else: - - instance.data["representation"] = self._get_single_repre( + instance.data["representations"] = self._get_single_repre( task_dir, task_data["files"] ) @@ -120,15 +117,15 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): return subset def _get_single_repre(self, task_dir, files): - _, ext = os.path.splittext(files[0]) + _, ext = os.path.splitext(files[0]) repre_data = { "name": ext[1:], "ext": ext[1:], - "files": files, + "files": files[0], "stagingDir": task_dir } - - return repre_data + self.log.info("single file repre_data.data:: {}".format(repre_data)) + return [repre_data] def _process_sequence(self, files, task_dir): """Prepare reprentations for sequence of files.""" @@ -147,7 +144,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "files": files, "stagingDir": task_dir } - self.log.info("repre_data.data:: {}".format(repre_data)) + self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] def _get_family(self, settings, task_type, is_sequence, extension): @@ -170,7 +167,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): for family, content in task_obj.items(): if is_sequence != content["is_sequence"]: continue - if extension in content["extensions"]: + if extension in content["extensions"] or \ + '' in content["extensions"]: # all extensions setting found_family = family break From 63a0c66c881e47c42a1943635a0ed10b72f80a29 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 2 Aug 2021 17:18:07 +0200 Subject: [PATCH 21/88] Webpublisher - backend - fix - removed shell flag causing problems on Linux --- openpype/modules/webserver/webpublish_routes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index cf6e4920b6..5322802130 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -12,6 +12,10 @@ from avalon.api import AvalonMongoDB from openpype.lib import OpenPypeMongoConnection from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint +from openpype.lib import PypeLogger + +log = PypeLogger.get_logger("WebServer") + class RestApiResource: """Resource carrying needed info and Avalon DB connection for publish.""" @@ -172,8 +176,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): """Triggers headless publishing of batch.""" async def post(self, request) -> Response: output = {} - - print(request) + log.info("WebpublisherBatchPublishEndpoint called") content = await request.json() batch_path = os.path.join(self.resource.upload_dir, @@ -200,9 +203,9 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): args.append("--{}".format(key)) args.append(value) - print("args:: {}".format(args)) + log.info("args:: {}".format(args)) - _exit_code = subprocess.call(args, shell=True) + _exit_code = subprocess.call(args) return Response( status=200, body=self.resource.encode(output), From 40f44edd6f5a3f1234995eab51a9d8265d0430aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 3 Aug 2021 09:53:04 +0200 Subject: [PATCH 22/88] Webpublisher - backend - fix - wrong key in DB --- openpype/pype_commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 513d7d0865..17b6d58ffd 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -143,6 +143,8 @@ class PypeCommands: log = Logger.get_logger() + log.info("remotepublish command") + install() if host: @@ -207,7 +209,7 @@ class PypeCommands: {"$set": { "finish_date": datetime.now(), - "state": "finished_ok", + "status": "finished_ok", "progress": 1 }} ) From 8c5941dde81cc520c032cdba901c7fb5611b9dc9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 6 Aug 2021 12:28:50 +0200 Subject: [PATCH 23/88] Webpublisher - added webpublisher host to extract burnin and review --- openpype/plugins/publish/extract_burnin.py | 3 ++- openpype/plugins/publish/extract_review.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index ef52d51325..809cf438c8 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -44,7 +44,8 @@ class ExtractBurnin(openpype.api.Extractor): "harmony", "fusion", "aftereffects", - "tvpaint" + "tvpaint", + "webpublisher" # "resolve" ] optional = True diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index de54b554e3..07e40b0421 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -44,7 +44,8 @@ class ExtractReview(pyblish.api.InstancePlugin): "standalonepublisher", "fusion", "tvpaint", - "resolve" + "resolve", + "webpublisher" ] # Supported extensions From 64834df4003c04014dcec7e88fb2b58041ff82b2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 12 Aug 2021 16:43:55 +0200 Subject: [PATCH 24/88] Fix - Deadline publish on Linux started Tray instead of headless publishing --- vendor/deadline/custom/plugins/GlobalJobPreLoad.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py index 41df9d4dc9..8631b035cf 100644 --- a/vendor/deadline/custom/plugins/GlobalJobPreLoad.py +++ b/vendor/deadline/custom/plugins/GlobalJobPreLoad.py @@ -55,9 +55,9 @@ def inject_openpype_environment(deadlinePlugin): "AVALON_TASK, AVALON_APP_NAME" raise RuntimeError(msg) - print("args::{}".format(args)) + print("args:::{}".format(args)) - exit_code = subprocess.call(args, shell=True) + exit_code = subprocess.call(args, cwd=os.path.dirname(openpype_app)) if exit_code != 0: raise RuntimeError("Publishing failed, check worker's log") From 96021daebd7cbc7e13108a797e837134bcdc664c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Aug 2021 11:28:06 +0200 Subject: [PATCH 25/88] creating thumbnails from exr in webpublisher --- .../plugins/publish/{extract_jpeg.py => extract_jpeg_exr.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename openpype/plugins/publish/{extract_jpeg.py => extract_jpeg_exr.py} (98%) diff --git a/openpype/plugins/publish/extract_jpeg.py b/openpype/plugins/publish/extract_jpeg_exr.py similarity index 98% rename from openpype/plugins/publish/extract_jpeg.py rename to openpype/plugins/publish/extract_jpeg_exr.py index b1289217e6..8d9e48b634 100644 --- a/openpype/plugins/publish/extract_jpeg.py +++ b/openpype/plugins/publish/extract_jpeg_exr.py @@ -17,7 +17,7 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin): "imagesequence", "render", "render2d", "source", "plate", "take" ] - hosts = ["shell", "fusion", "resolve"] + hosts = ["shell", "fusion", "resolve", "webpublisher"] enabled = False # presetable attribute From 9c56eb3b53bc76ca68cefe2e1b9ed6975e3d02f1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 11:48:02 +0200 Subject: [PATCH 26/88] Webpublisher - added translation from email to username --- .../plugins/publish/collect_username.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_username.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py new file mode 100644 index 0000000000..25d6f190a3 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -0,0 +1,45 @@ +"""Loads publishing context from json and continues in publish process. + +Requires: + anatomy -> context["anatomy"] *(pyblish.api.CollectorOrder - 0.11) + +Provides: + context, instances -> All data from previous publishing process. +""" + +import ftrack_api +import os + +import pyblish.api + + +class CollectUsername(pyblish.api.ContextPlugin): + """ + Translates user email to Ftrack username. + + Emails in Ftrack are same as company's Slack, username is needed to + load data to Ftrack. + + """ + order = pyblish.api.CollectorOrder - 0.488 + label = "Collect ftrack username" + host = ["webpublisher"] + + _context = None + + def process(self, context): + os.environ["FTRACK_API_USER"] = "pype.club" + os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] + self.log.info("CollectUsername") + for instance in context: + email = instance.data["user_email"] + self.log.info("email:: {}".format(email)) + session = ftrack_api.Session(auto_connect_event_hub=False) + user = session.query("User where email like '{}'".format( + email)) + + if not user: + raise ValueError("Couldnt find user with {} email".format(email)) + + os.environ["FTRACK_API_USER"] = user[0].get("username") + break From 3d0b470e36f6dee5fc8b5f0160357f73f80254e6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 11:48:19 +0200 Subject: [PATCH 27/88] Webpublisher - added collector for fps --- .../plugins/publish/collect_fps.py | 28 +++++++++++++++++++ .../publish/collect_published_files.py | 13 +++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 openpype/hosts/webpublisher/plugins/publish/collect_fps.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_fps.py b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py new file mode 100644 index 0000000000..79fe53176a --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/collect_fps.py @@ -0,0 +1,28 @@ +""" +Requires: + Nothing + +Provides: + Instance +""" + +import pyblish.api +from pprint import pformat + + +class CollectFPS(pyblish.api.InstancePlugin): + """ + Adds fps from context to instance because of ExtractReview + """ + + label = "Collect fps" + order = pyblish.api.CollectorOrder + 0.49 + hosts = ["webpublisher"] + + def process(self, instance): + fps = instance.context.data["fps"] + + instance.data.update({ + "fps": fps + }) + self.log.debug(f"instance.data: {pformat(instance.data)}") diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 67d743278b..5bc13dff96 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -69,6 +69,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): task_type = ctx["attributes"]["type"] else: asset = ctx["name"] + os.environ["AVALON_TASK"] = "" is_sequence = len(task_data["files"]) > 1 @@ -94,12 +95,16 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" - os.environ["FTRACK_API_USER"] = task_data["user"] + instance.data["user_email"] = task_data["user"] if is_sequence: instance.data["representations"] = self._process_sequence( task_data["files"], task_dir ) + instance.data["frameStart"] = \ + instance.data["representations"][0]["frameStart"] + instance.data["frameEnd"] = \ + instance.data["representations"][0]["frameEnd"] else: instance.data["representations"] = self._get_single_repre( task_dir, task_data["files"] @@ -122,7 +127,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "name": ext[1:], "ext": ext[1:], "files": files[0], - "stagingDir": task_dir + "stagingDir": task_dir, + "tags": ["review"] } self.log.info("single file repre_data.data:: {}".format(repre_data)) return [repre_data] @@ -142,7 +148,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "name": ext[1:], "ext": ext[1:], "files": files, - "stagingDir": task_dir + "stagingDir": task_dir, + "tags": ["review"] } self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] From 45fbdcbb564606b59c5f96a8a1232cd2bf596974 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 14:23:45 +0200 Subject: [PATCH 28/88] Webpublisher - added storing full log to Mongo --- openpype/pype_commands.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 17b6d58ffd..19981d2a39 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -181,17 +181,26 @@ class PypeCommands: "status": "in_progress" }).inserted_id + log_lines = [] for result in pyblish.util.publish_iter(): + for record in result["records"]: + log_lines.append("{}: {}".format( + result["plugin"].label, record.msg)) + if result["error"]: log.error(error_format.format(**result)) uninstall() + log_lines.append(error_format.format(**result)) dbcon.update_one( {"_id": _id}, {"$set": { "finish_date": datetime.now(), "status": "error", - "msg": error_format.format(**result) + "msg": "Publishing failed > click here and paste " + "report to slack OpenPype support", + "log": os.linesep.join(log_lines) + }} ) sys.exit(1) @@ -200,7 +209,8 @@ class PypeCommands: {"_id": _id}, {"$set": { - "progress": max(result["progress"], 0.95) + "progress": max(result["progress"], 0.95), + "log": os.linesep.join(log_lines) }} ) @@ -210,7 +220,8 @@ class PypeCommands: { "finish_date": datetime.now(), "status": "finished_ok", - "progress": 1 + "progress": 1, + "log": os.linesep.join(log_lines) }} ) From f459791902877c5c6f3e2a13217e2fe52a5bf70d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 18 Aug 2021 16:59:13 +0200 Subject: [PATCH 29/88] Webpublisher - added reprocess functionality Added system settings to enable webpublish --- openpype/modules/webserver/webserver_cli.py | 151 ++++++++++++------ .../defaults/system_settings/modules.json | 3 + .../schemas/system_schema/schema_modules.json | 14 ++ 3 files changed, 121 insertions(+), 47 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 0812bfa372..dcaa0b4e7b 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -1,4 +1,10 @@ import time +import os +from datetime import datetime +import requests +import json + + from .webpublish_routes import ( RestApiResource, OpenPypeRestApiResource, @@ -10,6 +16,8 @@ from .webpublish_routes import ( PublishesStatusEndpoint ) +from openpype.api import get_system_settings + def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" @@ -19,56 +27,105 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - resource = RestApiResource(webserver_module.server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) - projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/projects", - projects_endpoint.dispatch - ) + is_webpublish_enabled = get_system_settings()["modules"]\ + ["webpublish_tool"]["enabled"] - hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/hierarchy/{project_name}", - hiearchy_endpoint.dispatch - ) + if is_webpublish_enabled: + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/projects", + projects_endpoint.dispatch + ) - # triggers publish - webpublisher_task_publish_endpoint = \ - WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/batch", - webpublisher_task_publish_endpoint.dispatch - ) + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/hierarchy/{project_name}", + hiearchy_endpoint.dispatch + ) - webpublisher_batch_publish_endpoint = \ - WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/task", - webpublisher_batch_publish_endpoint.dispatch - ) + # triggers publish + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) - # reporting - openpype_resource = OpenPypeRestApiResource() - batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/batch_status/{batch_id}", - batch_status_endpoint.dispatch - ) + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch + ) - user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/publishes/{user}", - user_status_endpoint.dispatch - ) + # reporting + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) - webserver_module.start_server() - while True: - time.sleep(0.5) + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch + ) + + webserver_module.start_server() + last_reprocessed = time.time() + while True: + if is_webpublish_enabled: + if time.time() - last_reprocessed > 60: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() + time.sleep(1.0) + + +def reprocess_failed(upload_dir): + print("reprocess_failed") + from openpype.lib import OpenPypeMongoConnection + + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + results = dbcon.find({"status": "reprocess"}) + + for batch in results: + print("batch:: {}".format(batch)) + batch_url = os.path.join(upload_dir, + batch["batch_id"], + "manifest.json") + if not os.path.exists(batch_url): + msg = "Manifest {} not found".format(batch_url) + print(msg) + dbcon.update_one( + {"_id": batch["_id"]}, + {"$set": + { + "finish_date": datetime.now(), + "status": "error", + "progress": 1, + "log": batch.get("log") + msg + }} + ) + continue + + server_url = "{}/api/webpublish/batch".format( + os.environ["OPENPYPE_WEBSERVER_URL"]) + + with open(batch_url) as f: + data = json.loads(f.read()) + + r = requests.post(server_url, json=data) + print(r.status_code) \ No newline at end of file diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1b74b4695c..3f9b098a96 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -165,6 +165,9 @@ "standalonepublish_tool": { "enabled": true }, + "webpublish_tool": { + "enabled": false + }, "project_manager": { "enabled": true }, diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 7d734ff4fd..f82c3632a9 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -195,6 +195,20 @@ } ] }, + { + "type": "dict", + "key": "webpublish_tool", + "label": "Web Publish", + "collapsible": true, + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + } + ] + }, { "type": "dict", "key": "project_manager", From d2a34a6c712b65de90e206616128a01ddfe82c4e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 12:52:44 +0200 Subject: [PATCH 30/88] Webpublisher - added reprocess functionality --- openpype/modules/webserver/webserver_cli.py | 46 +++++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index dcaa0b4e7b..2eee20f855 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -4,6 +4,7 @@ from datetime import datetime import requests import json +from openpype.lib import PypeLogger from .webpublish_routes import ( RestApiResource, @@ -18,6 +19,10 @@ from .webpublish_routes import ( from openpype.api import get_system_settings +SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost + +log = PypeLogger().get_logger("webserver_gui") + def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" @@ -27,9 +32,14 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - is_webpublish_enabled = get_system_settings()["modules"]\ - ["webpublish_tool"]["enabled"] + is_webpublish_enabled = False + webpublish_tool = get_system_settings()["modules"].\ + get("webpublish_tool") + if webpublish_tool and webpublish_tool["enabled"]: + is_webpublish_enabled = True + + log.debug("is_webpublish_enabled {}".format(is_webpublish_enabled)) if is_webpublish_enabled: resource = RestApiResource(webserver_module.server_manager, upload_dir=kwargs["upload_dir"], @@ -81,18 +91,18 @@ def run_webserver(*args, **kwargs): user_status_endpoint.dispatch ) - webserver_module.start_server() - last_reprocessed = time.time() - while True: - if is_webpublish_enabled: - if time.time() - last_reprocessed > 60: - reprocess_failed(kwargs["upload_dir"]) - last_reprocessed = time.time() - time.sleep(1.0) + webserver_module.start_server() + last_reprocessed = time.time() + while True: + if is_webpublish_enabled: + if time.time() - last_reprocessed > 20: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() + time.sleep(1.0) def reprocess_failed(upload_dir): - print("reprocess_failed") + # log.info("check_reprocesable_records") from openpype.lib import OpenPypeMongoConnection mongo_client = OpenPypeMongoConnection.get_mongo_client() @@ -100,12 +110,11 @@ def reprocess_failed(upload_dir): dbcon = mongo_client[database_name]["webpublishes"] results = dbcon.find({"status": "reprocess"}) - for batch in results: - print("batch:: {}".format(batch)) batch_url = os.path.join(upload_dir, batch["batch_id"], "manifest.json") + log.info("batch:: {} {}".format(os.path.exists(batch_url), batch_url)) if not os.path.exists(batch_url): msg = "Manifest {} not found".format(batch_url) print(msg) @@ -120,12 +129,13 @@ def reprocess_failed(upload_dir): }} ) continue - - server_url = "{}/api/webpublish/batch".format( - os.environ["OPENPYPE_WEBSERVER_URL"]) + server_url = "{}/api/webpublish/batch".format(SERVER_URL) with open(batch_url) as f: data = json.loads(f.read()) - r = requests.post(server_url, json=data) - print(r.status_code) \ No newline at end of file + try: + r = requests.post(server_url, json=data) + log.info("response{}".format(r)) + except: + log.info("exception", exc_info=True) From 932ae5fbb4014e9886a8e679ce8c2b7859439199 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 13:03:25 +0200 Subject: [PATCH 31/88] Hound --- .../plugins/publish/collect_published_files.py | 11 ++++++----- .../webpublisher/plugins/publish/collect_username.py | 6 +++++- openpype/modules/webserver/webpublish_routes.py | 2 +- openpype/pype_commands.py | 2 -- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 5bc13dff96..cd231a0efc 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -49,9 +49,10 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): return data def _process_batch(self, dir_url): - task_subfolders = [os.path.join(dir_url, o) - for o in os.listdir(dir_url) - if os.path.isdir(os.path.join(dir_url, o))] + task_subfolders = [ + os.path.join(dir_url, o) + for o in os.listdir(dir_url) + if os.path.isdir(os.path.join(dir_url, o))] self.log.info("task_sub:: {}".format(task_subfolders)) for task_dir in task_subfolders: task_data = self._load_json(os.path.join(task_dir, @@ -185,8 +186,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): assert found_family, msg return found_family, \ - content["families"], \ - content["subset_template_name"] + content["families"], \ + content["subset_template_name"] def _get_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py index 25d6f190a3..0c2c6310f4 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_username.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -20,6 +20,9 @@ class CollectUsername(pyblish.api.ContextPlugin): Emails in Ftrack are same as company's Slack, username is needed to load data to Ftrack. + Expects "pype.club" user created on Ftrack and FTRACK_BOT_API_KEY env + var set up. + """ order = pyblish.api.CollectorOrder - 0.488 label = "Collect ftrack username" @@ -39,7 +42,8 @@ class CollectUsername(pyblish.api.ContextPlugin): email)) if not user: - raise ValueError("Couldnt find user with {} email".format(email)) + raise ValueError( + "Couldnt find user with {} email".format(email)) os.environ["FTRACK_API_USER"] = user[0].get("username") break diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/modules/webserver/webpublish_routes.py index 5322802130..32feb276ed 100644 --- a/openpype/modules/webserver/webpublish_routes.py +++ b/openpype/modules/webserver/webpublish_routes.py @@ -205,7 +205,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): log.info("args:: {}".format(args)) - _exit_code = subprocess.call(args) + subprocess.call(args) return Response( status=200, body=self.resource.encode(output), diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 19981d2a39..d288e9f2a3 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -197,8 +197,6 @@ class PypeCommands: { "finish_date": datetime.now(), "status": "error", - "msg": "Publishing failed > click here and paste " - "report to slack OpenPype support", "log": os.linesep.join(log_lines) }} From c6dad89b3478f2c59b60fdcc6ffadbba61d2c1c2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 14:22:42 +0200 Subject: [PATCH 32/88] Webpublisher - added configurable tags + defaults --- .../publish/collect_published_files.py | 19 ++++++++++--------- .../project_settings/webpublisher.json | 16 ++++++++++++++++ .../schema_project_webpublisher.json | 4 ++++ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index cd231a0efc..0c89bde8a5 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -75,7 +75,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): is_sequence = len(task_data["files"]) > 1 _, extension = os.path.splitext(task_data["files"][0]) - family, families, subset_template = self._get_family( + family, families, subset_template, tags = self._get_family( self.task_type_to_family, task_type, is_sequence, @@ -100,7 +100,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): if is_sequence: instance.data["representations"] = self._process_sequence( - task_data["files"], task_dir + task_data["files"], task_dir, tags ) instance.data["frameStart"] = \ instance.data["representations"][0]["frameStart"] @@ -108,7 +108,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["representations"][0]["frameEnd"] else: instance.data["representations"] = self._get_single_repre( - task_dir, task_data["files"] + task_dir, task_data["files"], tags ) self.log.info("instance.data:: {}".format(instance.data)) @@ -122,19 +122,19 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): subset = subset_template.format(**prepare_template_data(fill_pairs)) return subset - def _get_single_repre(self, task_dir, files): + def _get_single_repre(self, task_dir, files, tags): _, ext = os.path.splitext(files[0]) repre_data = { "name": ext[1:], "ext": ext[1:], "files": files[0], "stagingDir": task_dir, - "tags": ["review"] + "tags": tags } self.log.info("single file repre_data.data:: {}".format(repre_data)) return [repre_data] - def _process_sequence(self, files, task_dir): + def _process_sequence(self, files, task_dir, tags): """Prepare reprentations for sequence of files.""" collections, remainder = clique.assemble(files) assert len(collections) == 1, \ @@ -150,7 +150,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "ext": ext[1:], "files": files, "stagingDir": task_dir, - "tags": ["review"] + "tags": tags } self.log.info("sequences repre_data.data:: {}".format(repre_data)) return [repre_data] @@ -165,7 +165,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): extension (str): without '.' Returns: - (family, [families], subset_template_name) tuple + (family, [families], subset_template_name, tags) tuple AssertionError if not matching family found """ task_obj = settings.get(task_type) @@ -187,7 +187,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): return found_family, \ content["families"], \ - content["subset_template_name"] + content["subset_template_name"], \ + content["tags"] def _get_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index 8364b6a39d..a6916de144 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -9,6 +9,7 @@ "tvp" ], "families": [], + "tags": [], "subset_template_name": "" }, "render": { @@ -22,6 +23,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -32,6 +36,7 @@ "aep" ], "families": [], + "tags": [], "subset_template_name": "" }, "render": { @@ -45,6 +50,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -55,6 +63,7 @@ "psd" ], "families": [], + "tags": [], "subset_template_name": "" }, "image": { @@ -69,6 +78,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "" } }, @@ -79,6 +91,7 @@ "tvp" ], "families": [], + "tags": [], "subset_template_name": "{family}{Variant}" }, "render": { @@ -92,6 +105,9 @@ "families": [ "review" ], + "tags": [ + "ftrackreview" + ], "subset_template_name": "{family}{Variant}" } }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json index bf59cd030e..91337da2b2 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json @@ -48,6 +48,10 @@ "label": "Families", "object_type": "text" }, + { + "type": "schema", + "name": "schema_representation_tags" + }, { "type": "text", "key": "subset_template_name", From b2c06b937bc8d148ec8e81485232d9ecc93ac030 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Aug 2021 16:53:25 +0200 Subject: [PATCH 33/88] added collector for avalon host name --- openpype/plugins/publish/collect_host_name.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 openpype/plugins/publish/collect_host_name.py diff --git a/openpype/plugins/publish/collect_host_name.py b/openpype/plugins/publish/collect_host_name.py new file mode 100644 index 0000000000..897c50e4d8 --- /dev/null +++ b/openpype/plugins/publish/collect_host_name.py @@ -0,0 +1,21 @@ +""" +Requires: + None +Provides: + context -> host (str) +""" +import os +import pyblish.api + + +class CollectHostName(pyblish.api.ContextPlugin): + """Collect avalon host name to context.""" + + label = "Collect Host Name" + order = pyblish.api.CollectorOrder + + def process(self, context): + # Don't override value if is already set + host_name = context.data.get("host") + if not host_name: + context.data["host"] = os.environ.get("AVALON_APP") From 64a186b437844711e0501a4a848c45ff13fa7d06 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Aug 2021 17:00:18 +0200 Subject: [PATCH 34/88] moved host collection from collect anatomy context data --- .../publish/collect_anatomy_context_data.py | 17 ++-------------- openpype/plugins/publish/collect_host_name.py | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/openpype/plugins/publish/collect_anatomy_context_data.py b/openpype/plugins/publish/collect_anatomy_context_data.py index f121760e27..33db00636a 100644 --- a/openpype/plugins/publish/collect_anatomy_context_data.py +++ b/openpype/plugins/publish/collect_anatomy_context_data.py @@ -62,23 +62,10 @@ class CollectAnatomyContextData(pyblish.api.ContextPlugin): "asset": asset_entity["name"], "hierarchy": hierarchy.replace("\\", "/"), "task": task_name, - "username": context.data["user"] + "username": context.data["user"], + "app": context.data["host"] } - # Use AVALON_APP as first if available it is the same as host name - # - only if is not defined use AVALON_APP_NAME (e.g. on Farm) and - # set it back to AVALON_APP env variable - host_name = os.environ.get("AVALON_APP") - if not host_name: - app_manager = ApplicationManager() - app_name = os.environ.get("AVALON_APP_NAME") - if app_name: - app = app_manager.applications.get(app_name) - if app: - host_name = app.host_name - os.environ["AVALON_APP"] = host_name - context_data["app"] = host_name - datetime_data = context.data.get("datetimeData") or {} context_data.update(datetime_data) diff --git a/openpype/plugins/publish/collect_host_name.py b/openpype/plugins/publish/collect_host_name.py index 897c50e4d8..17af9253c3 100644 --- a/openpype/plugins/publish/collect_host_name.py +++ b/openpype/plugins/publish/collect_host_name.py @@ -7,6 +7,8 @@ Provides: import os import pyblish.api +from openpype.lib import ApplicationManager + class CollectHostName(pyblish.api.ContextPlugin): """Collect avalon host name to context.""" @@ -15,7 +17,21 @@ class CollectHostName(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder def process(self, context): - # Don't override value if is already set host_name = context.data.get("host") + # Don't override value if is already set + if host_name: + return + + # Use AVALON_APP as first if available it is the same as host name + # - only if is not defined use AVALON_APP_NAME (e.g. on Farm) and + # set it back to AVALON_APP env variable + host_name = os.environ.get("AVALON_APP") if not host_name: - context.data["host"] = os.environ.get("AVALON_APP") + app_name = os.environ.get("AVALON_APP_NAME") + if app_name: + app_manager = ApplicationManager() + app = app_manager.applications.get(app_name) + if app: + host_name = app.host_name + + context.data["host"] = host_name From 46fd7ee628afaee99c71c8157180a60f3d861ea6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Aug 2021 17:00:34 +0200 Subject: [PATCH 35/88] use context[host] in extract review and burnin --- openpype/plugins/publish/extract_burnin.py | 2 +- openpype/plugins/publish/extract_review.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 91e0a0f3ec..b0c6136694 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -96,7 +96,7 @@ class ExtractBurnin(openpype.api.Extractor): def main_process(self, instance): # TODO get these data from context - host_name = os.environ["AVALON_APP"] + host_name = instance.context["host"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index bdcd3b8e60..3b373bc1d6 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -89,7 +89,7 @@ class ExtractReview(pyblish.api.InstancePlugin): instance.data["representations"].remove(repre) def main_process(self, instance): - host_name = os.environ["AVALON_APP"] + host_name = instance.context["host"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) From 4a09a18275ec6b218f62f53a4f5b8e6cfd408d62 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 19 Aug 2021 17:37:33 +0200 Subject: [PATCH 36/88] Webpublisher - fix - status wasn't changed for reprocessed batches --- openpype/modules/webserver/webserver_cli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 2eee20f855..8e4dfd229d 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -139,3 +139,13 @@ def reprocess_failed(upload_dir): log.info("response{}".format(r)) except: log.info("exception", exc_info=True) + + dbcon.update_one( + {"_id": batch["_id"]}, + {"$set": + { + "finish_date": datetime.now(), + "status": "sent_for_reprocessing", + "progress": 1 + }} + ) From 080069ca9c18643276285904124f7508eca44d8a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Aug 2021 16:00:34 +0200 Subject: [PATCH 37/88] Webpublisher - added review to enum, changed defaults This defaults result in creating working review --- .../settings/defaults/project_settings/webpublisher.json | 8 ++++---- .../schemas/schema_representation_tags.json | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/settings/defaults/project_settings/webpublisher.json b/openpype/settings/defaults/project_settings/webpublisher.json index a6916de144..f57b79a609 100644 --- a/openpype/settings/defaults/project_settings/webpublisher.json +++ b/openpype/settings/defaults/project_settings/webpublisher.json @@ -24,7 +24,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -51,7 +51,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -79,7 +79,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "" } @@ -106,7 +106,7 @@ "review" ], "tags": [ - "ftrackreview" + "review" ], "subset_template_name": "{family}{Variant}" } diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json index b65de747e5..7607e1a8c1 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_representation_tags.json @@ -8,7 +8,10 @@ "burnin": "Add burnins" }, { - "ftrackreview": "Add to Ftrack" + "review": "Create review" + }, + { + "ftrackreview": "Add review to Ftrack" }, { "delete": "Delete output" From 92ef09444695e16427d24b381ba0f513c90b3903 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 20 Aug 2021 17:18:44 +0200 Subject: [PATCH 38/88] Webpublisher - added path field to log documents --- .../publish/collect_published_files.py | 4 ++ .../publish/integrate_context_to_log.py | 39 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 0c89bde8a5..8861190003 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -96,6 +96,10 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" + instance.data["ctx_path"] = ctx["path"] # to store for logging + instance.data["batch_id"] = task_data["batch"] + + instance.data["user_email"] = task_data["user"] if is_sequence: diff --git a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py new file mode 100644 index 0000000000..1dd57ffff9 --- /dev/null +++ b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py @@ -0,0 +1,39 @@ +import os + +from avalon import io +import pyblish.api +from openpype.lib import OpenPypeMongoConnection + + +class IntegrateContextToLog(pyblish.api.ContextPlugin): + """ Adds context information to log document for displaying in front end""" + + label = "Integrate Context to Log" + order = pyblish.api.IntegratorOrder - 0.1 + hosts = ["webpublisher"] + + def process(self, context): + self.log.info("Integrate Context to Log") + + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + dbcon = mongo_client[database_name]["webpublishes"] + + for instance in context: + self.log.info("ctx_path: {}".format(instance.data.get("ctx_path"))) + self.log.info("batch_id: {}".format(instance.data.get("batch_id"))) + if instance.data.get("ctx_path") and instance.data.get("batch_id"): + self.log.info("Updating log record") + dbcon.update_one( + { + "batch_id": instance.data.get("batch_id"), + "status": "in_progress" + }, + {"$set": + { + "path": instance.data.get("ctx_path") + + }} + ) + + return \ No newline at end of file From 10fa591e683eb785dea76c9ea300fe6567bdd033 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 23 Aug 2021 10:45:33 +0200 Subject: [PATCH 39/88] Webpublisher - added documentation --- openpype/cli.py | 7 ++ .../docs/admin_webserver_for_webpublisher.md | 82 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 website/docs/admin_webserver_for_webpublisher.md diff --git a/openpype/cli.py b/openpype/cli.py index 8dc32b307a..28195008cc 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -99,6 +99,13 @@ def eventserver(debug, @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") def webpublisherwebserver(debug, executable, upload_dir): + """Starts webserver for communication with Webpublish FR via command line + + OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND + FTRACK_BOT_API_KEY provided with api key from Ftrack. + + Expect "pype.club" user created on Ftrack. + """ if debug: os.environ['OPENPYPE_DEBUG'] = "3" diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md new file mode 100644 index 0000000000..748b269ad7 --- /dev/null +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -0,0 +1,82 @@ +--- +id: admin_webserver_for_webpublisher +title: Webserver for webpublisher +sidebar_label: Webserver for webpublisher +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Running Openpype webserver is needed as a backend part for Web publishing. +Any OS supported by Openpype could be used as a host server. + +Webpublishing consists of two sides, Front end (FE) and Openpype backend. This documenation is only targeted on OP side. + +It is expected that FE and OP will live on two separate servers, FE publicly available, OP safely in customer network. + +# Requirements for servers +- OP server allows access to its `8079` port for FE. (It is recommended to whitelist only FE IP.) +- have shared folder for published resources (images, workfiles etc) on both servers + +# Prepare Ftrack +Current webpublish process expects authentication via Slack. It is expected that customer has users created on a Ftrack +with same email addresses as on Slack. As some customer might have usernames different from emails, conversion from email to username is needed. + +For this "pype.club" user needs to be present on Ftrack, creation of this user should be standard part of Ftrack preparation for Openpype. +Next create API key on Ftrack, store this information temporarily as you won't have access to this key after creation. + + +# Prepare Openpype + +Deploy OP build distribution (Openpype Igniter) on an OS of your choice. + +##Run webserver as a Linux service: + +(This expects that OP Igniter is deployed to `opt/openpype` and log should be stored in `/tmp/openpype.log`) + +- create file `sudo vi /opt/openpype/webpublisher_webserver.sh` + +- paste content +```sh +#!/usr/bin/env bash +export OPENPYPE_DEBUG=3 +export FTRACK_BOT_API_KEY=YOUR_API_KEY +export PYTHONDONTWRITEBYTECODE=1 +export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION + +pushd /opt/openpype +./openpype_console webpublisherwebserver --upload_dir YOUR_SHARED_FOLDER_ON_HOST --executable /opt/openpype/openpype_console > /tmp/openpype.log 2>&1 +``` + +1. create service file `sudo vi /etc/systemd/system/openpye-webserver.service` + +2. paste content +```sh +[Unit] +Description=Run OpenPype Ftrack Webserver Service +After=network.target + +[Service] +Type=idle +ExecStart=/opt/openpype/webpublisher_webserver.sh +Restart=on-failure +RestartSec=10s +StandardOutput=append:/tmp/openpype.log +StandardError=append:/tmp/openpype.log + +[Install] +WantedBy=multi-user.target +``` + +5. change file permission: + `sudo chmod 0755 /etc/systemd/system/openpype-webserver.service` + +6. enable service: + `sudo systemctl enable openpype-webserver` + +7. start service: + `sudo systemctl start openpype-webserver` + +8. Check `/tmp/openpype.log` if OP got started + +(Note: service could be restarted by `service openpype-webserver restart` - this will result in purge of current log file!) \ No newline at end of file From eabfc473acc0297ec8c2b7af355acea607b98f10 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 23 Aug 2021 10:48:54 +0200 Subject: [PATCH 40/88] Hound --- .../webpublisher/plugins/publish/collect_published_files.py | 5 +++-- .../webpublisher/plugins/publish/integrate_context_to_log.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 8861190003..59c315861e 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -96,10 +96,11 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" - instance.data["ctx_path"] = ctx["path"] # to store for logging + # to store logging info into DB openpype.webpublishes + instance.data["ctx_path"] = ctx["path"] instance.data["batch_id"] = task_data["batch"] - + # to convert from email provided into Ftrack username instance.data["user_email"] = task_data["user"] if is_sequence: diff --git a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py index 1dd57ffff9..419c065e16 100644 --- a/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py +++ b/openpype/hosts/webpublisher/plugins/publish/integrate_context_to_log.py @@ -1,6 +1,5 @@ import os -from avalon import io import pyblish.api from openpype.lib import OpenPypeMongoConnection @@ -36,4 +35,4 @@ class IntegrateContextToLog(pyblish.api.ContextPlugin): }} ) - return \ No newline at end of file + return From 5d182faae26797764b2cdb98ee369c6600f5679c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 23 Aug 2021 12:25:44 +0200 Subject: [PATCH 41/88] changed context key from "host" to "hostName" --- openpype/plugins/publish/collect_anatomy_context_data.py | 2 +- openpype/plugins/publish/collect_host_name.py | 4 ++-- openpype/plugins/publish/extract_burnin.py | 2 +- openpype/plugins/publish/extract_review.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/plugins/publish/collect_anatomy_context_data.py b/openpype/plugins/publish/collect_anatomy_context_data.py index 33db00636a..ec88d5669d 100644 --- a/openpype/plugins/publish/collect_anatomy_context_data.py +++ b/openpype/plugins/publish/collect_anatomy_context_data.py @@ -63,7 +63,7 @@ class CollectAnatomyContextData(pyblish.api.ContextPlugin): "hierarchy": hierarchy.replace("\\", "/"), "task": task_name, "username": context.data["user"], - "app": context.data["host"] + "app": context.data["hostName"] } datetime_data = context.data.get("datetimeData") or {} diff --git a/openpype/plugins/publish/collect_host_name.py b/openpype/plugins/publish/collect_host_name.py index 17af9253c3..e1b7eb17c3 100644 --- a/openpype/plugins/publish/collect_host_name.py +++ b/openpype/plugins/publish/collect_host_name.py @@ -17,7 +17,7 @@ class CollectHostName(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder def process(self, context): - host_name = context.data.get("host") + host_name = context.data.get("hostName") # Don't override value if is already set if host_name: return @@ -34,4 +34,4 @@ class CollectHostName(pyblish.api.ContextPlugin): if app: host_name = app.host_name - context.data["host"] = host_name + context.data["hostName"] = host_name diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index b0c6136694..8fef5eaacb 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -96,7 +96,7 @@ class ExtractBurnin(openpype.api.Extractor): def main_process(self, instance): # TODO get these data from context - host_name = instance.context["host"] + host_name = instance.context["hostName"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index 3b373bc1d6..cdd40af027 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -89,7 +89,7 @@ class ExtractReview(pyblish.api.InstancePlugin): instance.data["representations"].remove(repre) def main_process(self, instance): - host_name = instance.context["host"] + host_name = instance.context["hostName"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) From 72c3bab5ec5ec06167afb1705a0d4fa7c47f31a4 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 23 Aug 2021 12:31:55 +0200 Subject: [PATCH 42/88] enhancement branches don't bump minor version --- .github/workflows/prerelease.yml | 2 +- tools/ci_tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index d0853e74d6..82f9a6ae9d 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -47,7 +47,7 @@ jobs: enhancementLabel: '**πŸš€ Enhancements**' bugsLabel: '**πŸ› Bug fixes**' deprecatedLabel: '**⚠️ Deprecations**' - addSections: '{"documentation":{"prefix":"### πŸ“– Documentation","labels":["documentation"]},"tests":{"prefix":"### βœ… Testing","labels":["tests"]}}' + addSections: '{"documentation":{"prefix":"### πŸ“– Documentation","labels":["documentation"]},"tests":{"prefix":"### βœ… Testing","labels":["tests"]},"feature":{"prefix":"### πŸ†• New features","labels":["feature"]},}' issues: false issuesWoLabels: false sinceTag: "3.0.0" diff --git a/tools/ci_tools.py b/tools/ci_tools.py index 436551c243..3c1aaae991 100644 --- a/tools/ci_tools.py +++ b/tools/ci_tools.py @@ -36,7 +36,7 @@ def get_log_since_tag(version): def release_type(log): regex_minor = ["feature/", "(feat)"] - regex_patch = ["bugfix/", "fix/", "(fix)"] + regex_patch = ["bugfix/", "fix/", "(fix)", "enhancement/"] for reg in regex_minor: if re.search(reg, log): return "minor" From a3f106736456755f7e7a9d4399eba68cc54d321a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 24 Aug 2021 11:08:44 +0200 Subject: [PATCH 43/88] Webpublisher - webserver ip is configurable In some cases webserver needs to listen on specific ip (because of Docker) --- openpype/modules/webserver/server.py | 5 ++++- openpype/modules/webserver/webserver_cli.py | 5 +++-- openpype/modules/webserver/webserver_module.py | 5 ++++- website/docs/admin_webserver_for_webpublisher.md | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/openpype/modules/webserver/server.py b/openpype/modules/webserver/server.py index 65c5795995..9d99e1c7a3 100644 --- a/openpype/modules/webserver/server.py +++ b/openpype/modules/webserver/server.py @@ -1,5 +1,6 @@ import threading import asyncio +import os from aiohttp import web @@ -110,7 +111,9 @@ class WebServerThread(threading.Thread): """ Starts runner and TCPsite """ self.runner = web.AppRunner(self.manager.app) await self.runner.setup() - self.site = web.TCPSite(self.runner, 'localhost', self.port) + host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' + log.info("host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) + self.site = web.TCPSite(self.runner, host_ip, self.port) await self.site.start() def stop(self): diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/modules/webserver/webserver_cli.py index 8e4dfd229d..24bd28ba7d 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/modules/webserver/webserver_cli.py @@ -19,7 +19,7 @@ from .webpublish_routes import ( from openpype.api import get_system_settings -SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost +# SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost log = PypeLogger().get_logger("webserver_gui") @@ -129,7 +129,8 @@ def reprocess_failed(upload_dir): }} ) continue - server_url = "{}/api/webpublish/batch".format(SERVER_URL) + server_url = "{}/api/webpublish/batch".format( + os.environ["OPENPYPE_WEBSERVER_URL"]) with open(batch_url) as f: data = json.loads(f.read()) diff --git a/openpype/modules/webserver/webserver_module.py b/openpype/modules/webserver/webserver_module.py index 4832038575..10508265da 100644 --- a/openpype/modules/webserver/webserver_module.py +++ b/openpype/modules/webserver/webserver_module.py @@ -79,7 +79,10 @@ class WebServerModule(PypeModule, ITrayService): self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) - webserver_url = "http://localhost:{}".format(self.port) + # in a case that webserver should listen on specific ip (webpublisher) + self.log.info("module host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) + host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' + webserver_url = "http://{}:{}".format(host_ip, self.port) os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url @staticmethod diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md index 748b269ad7..2b23033595 100644 --- a/website/docs/admin_webserver_for_webpublisher.md +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -40,6 +40,7 @@ Deploy OP build distribution (Openpype Igniter) on an OS of your choice. ```sh #!/usr/bin/env bash export OPENPYPE_DEBUG=3 +export WEBSERVER_HOST_IP=localhost export FTRACK_BOT_API_KEY=YOUR_API_KEY export PYTHONDONTWRITEBYTECODE=1 export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION From c67b647dc78614779075774e5180261d2f6dc7f6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 10:16:10 +0200 Subject: [PATCH 44/88] Fix context.data --- openpype/plugins/publish/extract_burnin.py | 2 +- openpype/plugins/publish/extract_review.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/extract_burnin.py b/openpype/plugins/publish/extract_burnin.py index 8fef5eaacb..607d2cbff7 100644 --- a/openpype/plugins/publish/extract_burnin.py +++ b/openpype/plugins/publish/extract_burnin.py @@ -96,7 +96,7 @@ class ExtractBurnin(openpype.api.Extractor): def main_process(self, instance): # TODO get these data from context - host_name = instance.context["hostName"] + host_name = instance.context.data["hostName"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) diff --git a/openpype/plugins/publish/extract_review.py b/openpype/plugins/publish/extract_review.py index cdd40af027..a9235c3ffa 100644 --- a/openpype/plugins/publish/extract_review.py +++ b/openpype/plugins/publish/extract_review.py @@ -89,7 +89,7 @@ class ExtractReview(pyblish.api.InstancePlugin): instance.data["representations"].remove(repre) def main_process(self, instance): - host_name = instance.context["hostName"] + host_name = instance.context.data["hostName"] task_name = os.environ["AVALON_TASK"] family = self.main_family_from_instance(instance) From 47f529bdccf00aebccf83971403b4ca91dc9bdb8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 10:50:35 +0200 Subject: [PATCH 45/88] Webpublisher - rename to last version --- .../webpublisher/plugins/publish/collect_published_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 59c315861e..6584120d97 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -92,7 +92,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): instance.data["subset"] = subset instance.data["family"] = family instance.data["families"] = families - instance.data["version"] = self._get_version(asset, subset) + 1 + instance.data["version"] = \ + self._get_last_version(asset, subset) + 1 instance.data["stagingDir"] = task_dir instance.data["source"] = "webpublisher" @@ -195,7 +196,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): content["subset_template_name"], \ content["tags"] - def _get_version(self, asset_name, subset_name): + def _get_last_version(self, asset_name, subset_name): """Returns version number or 0 for 'asset' and 'subset'""" query = [ { From 5164481b367bcfb1a7fc768d5b83a3f7b896ac0b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 10:57:22 +0200 Subject: [PATCH 46/88] Webpublisher - introduced FTRACK_BOT_API_USER --- openpype/hosts/webpublisher/plugins/publish/collect_username.py | 2 +- website/docs/admin_webserver_for_webpublisher.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/hosts/webpublisher/plugins/publish/collect_username.py index 0c2c6310f4..7a303a1608 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_username.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_username.py @@ -31,7 +31,7 @@ class CollectUsername(pyblish.api.ContextPlugin): _context = None def process(self, context): - os.environ["FTRACK_API_USER"] = "pype.club" + os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] self.log.info("CollectUsername") for instance in context: diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md index 2b23033595..dced825bdc 100644 --- a/website/docs/admin_webserver_for_webpublisher.md +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -41,6 +41,7 @@ Deploy OP build distribution (Openpype Igniter) on an OS of your choice. #!/usr/bin/env bash export OPENPYPE_DEBUG=3 export WEBSERVER_HOST_IP=localhost +export FTRACK_BOT_API_USER=YOUR_API_USER export FTRACK_BOT_API_KEY=YOUR_API_KEY export PYTHONDONTWRITEBYTECODE=1 export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION From bcea6fbf0d2b19ae92fc946ff1501704ce024308 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:09:57 +0200 Subject: [PATCH 47/88] Webpublisher - removed is_webpublish_enabled as unneeded run_server gets triggered only for webpublisher, doesn't make sense to double check Moved webpublish dependent classes under webpublish host Cleaned up setting --- .../webserver_service}/webpublish_routes.py | 0 .../webserver_service}/webserver_cli.py | 105 ++++++++---------- openpype/pype_commands.py | 7 +- .../defaults/system_settings/modules.json | 3 - .../schemas/system_schema/schema_modules.json | 14 --- 5 files changed, 50 insertions(+), 79 deletions(-) rename openpype/{modules/webserver => hosts/webpublisher/webserver_service}/webpublish_routes.py (100%) rename openpype/{modules/webserver => hosts/webpublisher/webserver_service}/webserver_cli.py (52%) diff --git a/openpype/modules/webserver/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py similarity index 100% rename from openpype/modules/webserver/webpublish_routes.py rename to openpype/hosts/webpublisher/webserver_service/webpublish_routes.py diff --git a/openpype/modules/webserver/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py similarity index 52% rename from openpype/modules/webserver/webserver_cli.py rename to openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 24bd28ba7d..b1c14260e9 100644 --- a/openpype/modules/webserver/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -19,7 +19,6 @@ from .webpublish_routes import ( from openpype.api import get_system_settings -# SERVER_URL = "http://172.17.0.1:8079" # machine is not listening on localhost log = PypeLogger().get_logger("webserver_gui") @@ -32,72 +31,62 @@ def run_webserver(*args, **kwargs): webserver_module = manager.modules_by_name["webserver"] webserver_module.create_server_manager() - is_webpublish_enabled = False - webpublish_tool = get_system_settings()["modules"].\ - get("webpublish_tool") + resource = RestApiResource(webserver_module.server_manager, + upload_dir=kwargs["upload_dir"], + executable=kwargs["executable"]) + projects_endpoint = WebpublisherProjectsEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/projects", + projects_endpoint.dispatch + ) - if webpublish_tool and webpublish_tool["enabled"]: - is_webpublish_enabled = True + hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) + webserver_module.server_manager.add_route( + "GET", + "/api/hierarchy/{project_name}", + hiearchy_endpoint.dispatch + ) - log.debug("is_webpublish_enabled {}".format(is_webpublish_enabled)) - if is_webpublish_enabled: - resource = RestApiResource(webserver_module.server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) - projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/projects", - projects_endpoint.dispatch - ) + # triggers publish + webpublisher_task_publish_endpoint = \ + WebpublisherBatchPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/batch", + webpublisher_task_publish_endpoint.dispatch + ) - hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( - "GET", - "/api/hierarchy/{project_name}", - hiearchy_endpoint.dispatch - ) + webpublisher_batch_publish_endpoint = \ + WebpublisherTaskPublishEndpoint(resource) + webserver_module.server_manager.add_route( + "POST", + "/api/webpublish/task", + webpublisher_batch_publish_endpoint.dispatch + ) - # triggers publish - webpublisher_task_publish_endpoint = \ - WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/batch", - webpublisher_task_publish_endpoint.dispatch - ) + # reporting + openpype_resource = OpenPypeRestApiResource() + batch_status_endpoint = BatchStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/batch_status/{batch_id}", + batch_status_endpoint.dispatch + ) - webpublisher_batch_publish_endpoint = \ - WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( - "POST", - "/api/webpublish/task", - webpublisher_batch_publish_endpoint.dispatch - ) - - # reporting - openpype_resource = OpenPypeRestApiResource() - batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/batch_status/{batch_id}", - batch_status_endpoint.dispatch - ) - - user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( - "GET", - "/api/publishes/{user}", - user_status_endpoint.dispatch - ) + user_status_endpoint = PublishesStatusEndpoint(openpype_resource) + webserver_module.server_manager.add_route( + "GET", + "/api/publishes/{user}", + user_status_endpoint.dispatch + ) webserver_module.start_server() last_reprocessed = time.time() while True: - if is_webpublish_enabled: - if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"]) - last_reprocessed = time.time() + if time.time() - last_reprocessed > 20: + reprocess_failed(kwargs["upload_dir"]) + last_reprocessed = time.time() time.sleep(1.0) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d288e9f2a3..e0cab962f6 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -42,9 +42,8 @@ class PypeCommands: @staticmethod def launch_webpublisher_webservercli(*args, **kwargs): - from openpype.modules.webserver.webserver_cli import ( - run_webserver - ) + from openpype.hosts.webpublisher.webserver_service.webserver_cli \ + import (run_webserver) return run_webserver(*args, **kwargs) @staticmethod @@ -53,7 +52,7 @@ class PypeCommands: standalonepublish.main() @staticmethod - def publish(paths, targets=None, host=None): + def publish(paths, targets=None): """Start headless publishing. Publish use json from passed paths argument. diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json index 1005f8d16b..3a70b90590 100644 --- a/openpype/settings/defaults/system_settings/modules.json +++ b/openpype/settings/defaults/system_settings/modules.json @@ -167,9 +167,6 @@ "standalonepublish_tool": { "enabled": true }, - "webpublish_tool": { - "enabled": false - }, "project_manager": { "enabled": true }, diff --git a/openpype/settings/entities/schemas/system_schema/schema_modules.json b/openpype/settings/entities/schemas/system_schema/schema_modules.json index 8cd729d2a1..75c08b2cd9 100644 --- a/openpype/settings/entities/schemas/system_schema/schema_modules.json +++ b/openpype/settings/entities/schemas/system_schema/schema_modules.json @@ -197,20 +197,6 @@ } ] }, - { - "type": "dict", - "key": "webpublish_tool", - "label": "Web Publish", - "collapsible": true, - "checkbox_key": "enabled", - "children": [ - { - "type": "boolean", - "key": "enabled", - "label": "Enabled" - } - ] - }, { "type": "dict", "key": "project_manager", From 9a9acc119ae5a95b117cca0ad6ffad65554faa71 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:23:26 +0200 Subject: [PATCH 48/88] Webpublisher - introduced command line arguments for host and port --- openpype/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 28195008cc..0b6d41b060 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -96,9 +96,11 @@ def eventserver(debug, @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") -def webpublisherwebserver(debug, executable, upload_dir): +def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None): """Starts webserver for communication with Webpublish FR via command line OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND @@ -111,7 +113,9 @@ def webpublisherwebserver(debug, executable, upload_dir): PypeCommands().launch_webpublisher_webservercli( upload_dir=upload_dir, - executable=executable + executable=executable, + host=host, + port=port ) From 91f9362288b1a6a4ca5d894812e6e32621f5874c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:47:52 +0200 Subject: [PATCH 49/88] Webpublisher - proper merge --- .../default_modules/webserver/server.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/modules/default_modules/webserver/server.py b/openpype/modules/default_modules/webserver/server.py index 9d99e1c7a3..83a29e074e 100644 --- a/openpype/modules/default_modules/webserver/server.py +++ b/openpype/modules/default_modules/webserver/server.py @@ -1,6 +1,5 @@ import threading import asyncio -import os from aiohttp import web @@ -11,8 +10,9 @@ log = PypeLogger.get_logger("WebServer") class WebServerManager: """Manger that care about web server thread.""" - def __init__(self, module): - self.module = module + def __init__(self, port=None, host=None): + self.port = port or 8079 + self.host = host or "localhost" self.client = None self.handlers = {} @@ -25,8 +25,8 @@ class WebServerManager: self.webserver_thread = WebServerThread(self) @property - def port(self): - return self.module.port + def url(self): + return "http://{}:{}".format(self.host, self.port) def add_route(self, *args, **kwargs): self.app.router.add_route(*args, **kwargs) @@ -79,6 +79,10 @@ class WebServerThread(threading.Thread): def port(self): return self.manager.port + @property + def host(self): + return self.manager.host + def run(self): self.is_running = True @@ -111,9 +115,7 @@ class WebServerThread(threading.Thread): """ Starts runner and TCPsite """ self.runner = web.AppRunner(self.manager.app) await self.runner.setup() - host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' - log.info("host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) - self.site = web.TCPSite(self.runner, host_ip, self.port) + self.site = web.TCPSite(self.runner, self.host, self.port) await self.site.start() def stop(self): From f7cb778470fbfbab3b0ad7209912670f0992cca2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:48:43 +0200 Subject: [PATCH 50/88] Webpublisher - proper merge --- .../webserver/webserver_module.py | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index c000d5ce10..bdb0010118 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -1,24 +1,34 @@ import os import socket +from abc import ABCMeta, abstractmethod + +import six from openpype import resources -from openpype.modules import OpenPypeModule -from openpype_interfaces import ( - ITrayService, - IWebServerRoutes -) +from .. import PypeModule, ITrayService -class WebServerModule(OpenPypeModule, ITrayService): +@six.add_metaclass(ABCMeta) +class IWebServerRoutes: + """Other modules interface to register their routes.""" + @abstractmethod + def webserver_initialization(self, server_manager): + pass + + +class WebServerModule(PypeModule, ITrayService): name = "webserver" label = "WebServer" + webserver_url_env = "OPENPYPE_WEBSERVER_URL" + def initialize(self, _module_settings): self.enabled = True self.server_manager = None self._host_listener = None self.port = self.find_free_port() + self.webserver_url = None def connect_with_modules(self, enabled_modules): if not self.server_manager: @@ -44,7 +54,7 @@ class WebServerModule(OpenPypeModule, ITrayService): self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( - os.environ["OPENPYPE_WEBSERVER_URL"], static_prefix + self.webserver_url, static_prefix ) def _add_listeners(self): @@ -62,21 +72,33 @@ class WebServerModule(OpenPypeModule, ITrayService): if self.server_manager: self.server_manager.stop_server() + @staticmethod + def create_new_server_manager(port=None, host=None): + """Create webserver manager for passed port and host. + + Args: + port(int): Port on which wil webserver listen. + host(str): Host name or IP address. Default is 'localhost'. + + Returns: + WebServerManager: Prepared manager. + """ + from .server import WebServerManager + + return WebServerManager(port, host) + def create_server_manager(self): if self.server_manager: return - from .server import WebServerManager - - self.server_manager = WebServerManager(self) + self.server_manager = self.create_new_server_manager(self.port) self.server_manager.on_stop_callbacks.append( self.set_service_failed_icon ) - # in a case that webserver should listen on specific ip (webpublisher) - self.log.info("module host_ip:: {}".format(os.environ.get("WEBSERVER_HOST_IP"))) - host_ip = os.environ.get("WEBSERVER_HOST_IP") or 'localhost' - webserver_url = "http://{}:{}".format(host_ip, self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url + + webserver_url = self.server_manager.url + os.environ[self.webserver_url_env] = str(webserver_url) + self.webserver_url = webserver_url @staticmethod def find_free_port( From b235068a3eee5858be38cd8c70ed9bb4d824d2ae Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:55:32 +0200 Subject: [PATCH 51/88] Webpublisher - proper merge --- .../webserver_service/webpublish_routes.py | 2 +- .../default_modules/webserver/webserver_module.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 32feb276ed..0014d1b344 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -10,7 +10,7 @@ import subprocess from avalon.api import AvalonMongoDB from openpype.lib import OpenPypeMongoConnection -from openpype.modules.avalon_apps.rest_api import _RestApiEndpoint +from openpype_modules.avalon_apps.rest_api import _RestApiEndpoint from openpype.lib import PypeLogger diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index bdb0010118..d8e54632b5 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -5,7 +5,11 @@ from abc import ABCMeta, abstractmethod import six from openpype import resources -from .. import PypeModule, ITrayService +from openpype.modules import OpenPypeModule +from openpype_interfaces import ( + ITrayService, + IWebServerRoutes +) @six.add_metaclass(ABCMeta) @@ -16,7 +20,7 @@ class IWebServerRoutes: pass -class WebServerModule(PypeModule, ITrayService): +class WebServerModule(OpenPypeModule, ITrayService): name = "webserver" label = "WebServer" @@ -53,6 +57,8 @@ class WebServerModule(PypeModule, ITrayService): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) + webserver_url = "http://localhost:{}".format(self.port) + os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( self.webserver_url, static_prefix ) From e666fad275a018aad670418605e041614eb85e1c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 11:57:58 +0200 Subject: [PATCH 52/88] Webpublisher - updated help label --- openpype/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/cli.py b/openpype/cli.py index 0b6d41b060..c446d5e443 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -162,7 +162,7 @@ def publish(debug, paths, targets): @click.option("-h", "--host", help="Host") @click.option("-u", "--user", help="User email address") @click.option("-p", "--project", help="Project") -@click.option("-t", "--targets", help="Targets module", default=None, +@click.option("-t", "--targets", help="Targets", default=None, multiple=True) def remotepublish(debug, project, path, host, targets=None, user=None): """Start CLI publishing. From 1387b75d5f4d1242c2ab7f35f8e406564013b848 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 12:36:03 +0200 Subject: [PATCH 53/88] Webpublisher - revert mixed up commit --- openpype/lib/applications.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 19208ff173..71ab2eac61 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -1009,7 +1009,7 @@ class EnvironmentPrepData(dict): def get_app_environments_for_context( - project_name, asset_name, task_name, app_name=None, env=None + project_name, asset_name, task_name, app_name, env=None ): """Prepare environment variables by context. Args: @@ -1038,14 +1038,20 @@ def get_app_environments_for_context( "name": asset_name }) + # Prepare app object which can be obtained only from ApplciationManager + app_manager = ApplicationManager() + app = app_manager.applications[app_name] + # Project's anatomy anatomy = Anatomy(project_name) - prep_dict = { + data = EnvironmentPrepData({ "project_name": project_name, "asset_name": asset_name, "task_name": task_name, + "app": app, + "dbcon": dbcon, "project_doc": project_doc, "asset_doc": asset_doc, @@ -1053,15 +1059,7 @@ def get_app_environments_for_context( "anatomy": anatomy, "env": env - } - - if app_name: - # Prepare app object which can be obtained only from ApplicationManager - app_manager = ApplicationManager() - app = app_manager.applications[app_name] - prep_dict["app"] = app - - data = EnvironmentPrepData(prep_dict) + }) prepare_host_environments(data) prepare_context_environments(data) From 647f9779acb8cbbfa82848097dc47bfba0cb2bde Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 12:53:11 +0200 Subject: [PATCH 54/88] moved host name collector earlier --- openpype/plugins/publish/collect_host_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_host_name.py b/openpype/plugins/publish/collect_host_name.py index e1b7eb17c3..41d9cc3a5a 100644 --- a/openpype/plugins/publish/collect_host_name.py +++ b/openpype/plugins/publish/collect_host_name.py @@ -14,7 +14,7 @@ class CollectHostName(pyblish.api.ContextPlugin): """Collect avalon host name to context.""" label = "Collect Host Name" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder - 1 def process(self, context): host_name = context.data.get("hostName") From 3dfe3513c3e44e445042c086ea94740c542a8e3b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 13:10:45 +0200 Subject: [PATCH 55/88] Webpublisher - fixed host install --- openpype/hosts/webpublisher/__init__.py | 3 +++ openpype/hosts/webpublisher/api/__init__.py | 1 - openpype/pype_commands.py | 9 +++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index e69de29bb2..d47bab580b 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -0,0 +1,3 @@ +# to have required methods for interface +def ls(): + pass \ No newline at end of file diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 1bf1ef1a6f..76709bb2d7 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -29,7 +29,6 @@ def install(): log.info(PUBLISH_PATH) io.install() - avalon.Session["AVALON_APP"] = "webpublisher" # because of Ftrack collect avalon.on("application.launched", application_launch) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 7774a010a6..656f864229 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -157,11 +157,12 @@ class PypeCommands: os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host # to trigger proper plugings + os.environ["AVALON_APP"] = host - # this should be more generic - from openpype.hosts.webpublisher.api import install as w_install - w_install() + import avalon.api + from openpype.hosts import webpublisher + + avalon.api.install(webpublisher) log.info("Running publish ...") From c7c45ecf879ffe54ee7f942a7490350baec0c862 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 13:24:42 +0200 Subject: [PATCH 56/88] Webpublisher - removed unwanted folder --- openpype/modules/sync_server/__init__.py | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 openpype/modules/sync_server/__init__.py diff --git a/openpype/modules/sync_server/__init__.py b/openpype/modules/sync_server/__init__.py deleted file mode 100644 index a814f0db62..0000000000 --- a/openpype/modules/sync_server/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from openpype.modules.sync_server.sync_server_module import SyncServerModule - - -def tray_init(tray_widget, main_widget): - return SyncServerModule() From a65f0e15d71632ff94dc90d3a8b13222149f494d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:00:09 +0200 Subject: [PATCH 57/88] fixed webserver module --- .../default_modules/webserver/webserver_module.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index d8e54632b5..cfbb0c1ee0 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -12,14 +12,6 @@ from openpype_interfaces import ( ) -@six.add_metaclass(ABCMeta) -class IWebServerRoutes: - """Other modules interface to register their routes.""" - @abstractmethod - def webserver_initialization(self, server_manager): - pass - - class WebServerModule(OpenPypeModule, ITrayService): name = "webserver" label = "WebServer" @@ -57,8 +49,6 @@ class WebServerModule(OpenPypeModule, ITrayService): static_prefix = "/res" self.server_manager.add_static(static_prefix, resources.RESOURCES_DIR) - webserver_url = "http://localhost:{}".format(self.port) - os.environ["OPENPYPE_WEBSERVER_URL"] = webserver_url os.environ["OPENPYPE_STATICS_SERVER"] = "{}{}".format( self.webserver_url, static_prefix ) From 5dbc7ab36d1eb3c601bbb6cffd92a6a0624d4852 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:02:55 +0200 Subject: [PATCH 58/88] recommit changes --- .../webserver_service/webserver_cli.py | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b1c14260e9..b733cc260f 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,20 +29,23 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - webserver_module.create_server_manager() + host = os.environ.get("WEBSERVER_HOST_IP") + port = 8079 + server_manager = webserver_module.create_new_server_manager(port, host) + webserver_url = server_manager.url - resource = RestApiResource(webserver_module.server_manager, + resource = RestApiResource(server_manager, upload_dir=kwargs["upload_dir"], executable=kwargs["executable"]) projects_endpoint = WebpublisherProjectsEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/projects", projects_endpoint.dispatch ) hiearchy_endpoint = WebpublisherHiearchyEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/hierarchy/{project_name}", hiearchy_endpoint.dispatch @@ -51,7 +54,7 @@ def run_webserver(*args, **kwargs): # triggers publish webpublisher_task_publish_endpoint = \ WebpublisherBatchPublishEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "POST", "/api/webpublish/batch", webpublisher_task_publish_endpoint.dispatch @@ -59,7 +62,7 @@ def run_webserver(*args, **kwargs): webpublisher_batch_publish_endpoint = \ WebpublisherTaskPublishEndpoint(resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "POST", "/api/webpublish/task", webpublisher_batch_publish_endpoint.dispatch @@ -68,29 +71,29 @@ def run_webserver(*args, **kwargs): # reporting openpype_resource = OpenPypeRestApiResource() batch_status_endpoint = BatchStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/batch_status/{batch_id}", batch_status_endpoint.dispatch ) user_status_endpoint = PublishesStatusEndpoint(openpype_resource) - webserver_module.server_manager.add_route( + server_manager.add_route( "GET", "/api/publishes/{user}", user_status_endpoint.dispatch ) - webserver_module.start_server() + server_manager.start_server() last_reprocessed = time.time() while True: if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"]) + reprocess_failed(kwargs["upload_dir"], webserver_url) last_reprocessed = time.time() time.sleep(1.0) -def reprocess_failed(upload_dir): +def reprocess_failed(upload_dir, webserver_url): # log.info("check_reprocesable_records") from openpype.lib import OpenPypeMongoConnection @@ -118,8 +121,7 @@ def reprocess_failed(upload_dir): }} ) continue - server_url = "{}/api/webpublish/batch".format( - os.environ["OPENPYPE_WEBSERVER_URL"]) + server_url = "{}/api/webpublish/batch".format(webserver_url) with open(batch_url) as f: data = json.loads(f.read()) From cd5659248bc6db9c4a4602b24ddb467415f188c2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 14:10:34 +0200 Subject: [PATCH 59/88] use host port from kwargs --- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b733cc260f..06d78e2fca 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,8 +29,8 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = os.environ.get("WEBSERVER_HOST_IP") - port = 8079 + host = kwargs.get("host") or "localhost" + port = kwargs.get("port") or 8079 server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url From 66b710116447774fee71f088d34c8eb5e34ab798 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:11:08 +0200 Subject: [PATCH 60/88] Webpublisher - fix propagation of host --- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index b733cc260f..723762003d 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -29,8 +29,8 @@ def run_webserver(*args, **kwargs): manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = os.environ.get("WEBSERVER_HOST_IP") - port = 8079 + host = kwargs["host"] + port = kwargs["port"] server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url From 2d55233c6b2ab4c8ed8129f052b08eda70a3bffe Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:33:16 +0200 Subject: [PATCH 61/88] Hound --- openpype/hosts/webpublisher/__init__.py | 2 +- .../hosts/webpublisher/webserver_service/webserver_cli.py | 4 +--- .../modules/default_modules/webserver/webserver_module.py | 3 --- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index d47bab580b..3de2e3434b 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -1,3 +1,3 @@ # to have required methods for interface def ls(): - pass \ No newline at end of file + pass diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 06d78e2fca..d00d269059 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -17,8 +17,6 @@ from .webpublish_routes import ( PublishesStatusEndpoint ) -from openpype.api import get_system_settings - log = PypeLogger().get_logger("webserver_gui") @@ -129,7 +127,7 @@ def reprocess_failed(upload_dir, webserver_url): try: r = requests.post(server_url, json=data) log.info("response{}".format(r)) - except: + except Exception: log.info("exception", exc_info=True) dbcon.update_one( diff --git a/openpype/modules/default_modules/webserver/webserver_module.py b/openpype/modules/default_modules/webserver/webserver_module.py index cfbb0c1ee0..5bfb2d6390 100644 --- a/openpype/modules/default_modules/webserver/webserver_module.py +++ b/openpype/modules/default_modules/webserver/webserver_module.py @@ -1,8 +1,5 @@ import os import socket -from abc import ABCMeta, abstractmethod - -import six from openpype import resources from openpype.modules import OpenPypeModule From 430801da30a7fa259bcd1b815e643c5433a41651 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 14:36:37 +0200 Subject: [PATCH 62/88] Webpublisher - move plugin to Ftrack --- .../default_modules/ftrack}/plugins/publish/collect_username.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openpype/{hosts/webpublisher => modules/default_modules/ftrack}/plugins/publish/collect_username.py (100%) diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py similarity index 100% rename from openpype/hosts/webpublisher/plugins/publish/collect_username.py rename to openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py From 8c9f20bbc483d0facc9e8a40e6789dbf7fa2e9e9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 15:06:27 +0200 Subject: [PATCH 63/88] Webpublisher - moved dummy ls to api --- openpype/hosts/webpublisher/__init__.py | 3 --- openpype/hosts/webpublisher/api/__init__.py | 5 +++++ openpype/pype_commands.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index 3de2e3434b..e69de29bb2 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -1,3 +0,0 @@ -# to have required methods for interface -def ls(): - pass diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 76709bb2d7..e40d46d662 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -36,3 +36,8 @@ def uninstall(): pyblish.deregister_plugin_path(PUBLISH_PATH) avalon.deregister_plugin_path(avalon.Loader, LOAD_PATH) avalon.deregister_plugin_path(avalon.Creator, CREATE_PATH) + + +# to have required methods for interface +def ls(): + pass diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 656f864229..c18fe36667 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -160,7 +160,7 @@ class PypeCommands: os.environ["AVALON_APP"] = host import avalon.api - from openpype.hosts import webpublisher + from openpype.hosts.webpublisher import api as webpublisher avalon.api.install(webpublisher) From cb229e9185308fc08603c88341c8792fd63f652b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 25 Aug 2021 15:11:08 +0200 Subject: [PATCH 64/88] Merge back to develop --- repos/avalon-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/avalon-core b/repos/avalon-core index 82d5b8137e..52e24a9993 160000 --- a/repos/avalon-core +++ b/repos/avalon-core @@ -1 +1 @@ -Subproject commit 82d5b8137eea3b49d4781a4af51d7f375bb9f628 +Subproject commit 52e24a9993e5223b0a719786e77a4b87e936e556 From 053e5d9750aee23feb12670b44cd8b28f34b1d5b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 18:48:49 +0200 Subject: [PATCH 65/88] launch blender like a python application on windows --- .../blender/hooks/pre_windows_console.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 openpype/hosts/blender/hooks/pre_windows_console.py diff --git a/openpype/hosts/blender/hooks/pre_windows_console.py b/openpype/hosts/blender/hooks/pre_windows_console.py new file mode 100644 index 0000000000..d6be45b225 --- /dev/null +++ b/openpype/hosts/blender/hooks/pre_windows_console.py @@ -0,0 +1,28 @@ +import subprocess +from openpype.lib import PreLaunchHook + + +class BlenderConsoleWindows(PreLaunchHook): + """Foundry applications have specific way how to launch them. + + Blender is executed "like" python process so it is required to pass + `CREATE_NEW_CONSOLE` flag on windows to trigger creation of new console. + At the same time the newly created console won't create it's own stdout + and stderr handlers so they should not be redirected to DEVNULL. + """ + + # Should be as last hook because must change launch arguments to string + order = 1000 + app_groups = ["blender"] + platforms = ["windows"] + + def execute(self): + # Change `creationflags` to CREATE_NEW_CONSOLE + # - on Windows will blender create new window using it's console + # Set `stdout` and `stderr` to None so new created console does not + # have redirected output to DEVNULL in build + self.launch_context.kwargs.update({ + "creationflags": subprocess.CREATE_NEW_CONSOLE, + "stdout": None, + "stderr": None + }) From 5aabef9f9bcb314aaf68a40d2de8dc0a85ad0b12 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 18:53:21 +0200 Subject: [PATCH 66/88] push new defaults --- openpype/settings/defaults/project_settings/global.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index aab8c2196c..0c87c915f9 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -151,6 +151,7 @@ "template_name_profiles": [ { "families": [], + "hosts": [], "tasks": [], "template_name": "publish" }, @@ -160,6 +161,7 @@ "render", "prerender" ], + "hosts": [], "tasks": [], "template_name": "render" } From 1323ad175329535063ce2ab42fef47ded33b85c8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 18:55:55 +0200 Subject: [PATCH 67/88] added margins to footer layout --- openpype/tools/settings/settings/categories.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/tools/settings/settings/categories.py b/openpype/tools/settings/settings/categories.py index d1babd7fdb..c420a8cdc5 100644 --- a/openpype/tools/settings/settings/categories.py +++ b/openpype/tools/settings/settings/categories.py @@ -203,6 +203,7 @@ class SettingsCategoryWidget(QtWidgets.QWidget): refresh_btn.setIcon(refresh_icon) footer_layout = QtWidgets.QHBoxLayout() + footer_layout.setContentsMargins(5, 5, 5, 5) if self.user_role == "developer": self._add_developer_ui(footer_layout) From 946e7da7f8aaa54d6e67b971b9ed57eecd156f49 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 18:58:57 +0200 Subject: [PATCH 68/88] don't go to root with mouse click --- openpype/tools/settings/settings/breadcrumbs_widget.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/tools/settings/settings/breadcrumbs_widget.py b/openpype/tools/settings/settings/breadcrumbs_widget.py index b625a7bb07..d25cbdc8cb 100644 --- a/openpype/tools/settings/settings/breadcrumbs_widget.py +++ b/openpype/tools/settings/settings/breadcrumbs_widget.py @@ -325,7 +325,9 @@ class BreadcrumbsButton(QtWidgets.QToolButton): self.setSizePolicy(size_policy) menu.triggered.connect(self._on_menu_click) - self.clicked.connect(self._on_click) + # Don't allow to go to root with mouse click + if path: + self.clicked.connect(self._on_click) self._path = path self._path_prefix = path_prefix From 75f3dca1ced1dcd4ac4c676ecc5a684178727b0c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 19:03:38 +0200 Subject: [PATCH 69/88] mouse click on checkbox changes path --- openpype/tools/settings/settings/item_widgets.py | 4 ++++ openpype/tools/settings/settings/widgets.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/openpype/tools/settings/settings/item_widgets.py b/openpype/tools/settings/settings/item_widgets.py index d29fa6f42b..a808caa465 100644 --- a/openpype/tools/settings/settings/item_widgets.py +++ b/openpype/tools/settings/settings/item_widgets.py @@ -312,8 +312,12 @@ class BoolWidget(InputWidget): self.setFocusProxy(self.input_field) + self.input_field.focused_in.connect(self._on_input_focus) self.input_field.stateChanged.connect(self._on_value_change) + def _on_input_focus(self): + self.focused_in() + def _on_entity_change(self): if self.entity.value != self.input_field.isChecked(): self.set_entity_value() diff --git a/openpype/tools/settings/settings/widgets.py b/openpype/tools/settings/settings/widgets.py index 34b222dd8e..d49057e1e8 100644 --- a/openpype/tools/settings/settings/widgets.py +++ b/openpype/tools/settings/settings/widgets.py @@ -459,6 +459,7 @@ class NiceCheckbox(QtWidgets.QFrame): stateChanged = QtCore.Signal(int) checked_bg_color = QtGui.QColor(69, 128, 86) unchecked_bg_color = QtGui.QColor(170, 80, 80) + focused_in = QtCore.Signal() def set_bg_color(self, color): self._bg_color = color @@ -583,6 +584,10 @@ class NiceCheckbox(QtWidgets.QFrame): self._on_checkstate_change() + def mousePressEvent(self, event): + self.focused_in.emit() + super(NiceCheckbox, self).mousePressEvent(event) + def mouseReleaseEvent(self, event): if event.button() == QtCore.Qt.LeftButton: self.setCheckState() From 7b0b6a50e82f465d5e8fa4bc86c6d74ca5935fbe Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 19:57:17 +0200 Subject: [PATCH 70/88] removed unused variable --- openpype/tools/launcher/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 846a07e081..09bdc3f961 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -122,7 +122,6 @@ class ActionModel(QtGui.QStandardItemModel): self.application_manager = ApplicationManager() - self._groups = {} self.default_icon = qtawesome.icon("fa.cube", color="white") # Cache of available actions self._registered_actions = list() @@ -186,8 +185,6 @@ class ActionModel(QtGui.QStandardItemModel): self.clear() self.items_by_id.clear() - self._groups.clear() - actions = self.filter_compatible_actions(self._registered_actions) self.beginResetModel() From 7458c049ccd2edfc47dc7055f5bc531fd01d3899 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 19:57:41 +0200 Subject: [PATCH 71/88] don't clear items by id on discover --- openpype/tools/launcher/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 09bdc3f961..398d8aad3d 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -137,7 +137,6 @@ class ActionModel(QtGui.QStandardItemModel): actions.extend(app_actions) self._registered_actions = actions - self.items_by_id.clear() def get_application_actions(self): actions = [] From 6db6969e1b8896f7c54444b2af60727ddbfab2d9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 19:57:52 +0200 Subject: [PATCH 72/88] added projection to project document --- openpype/tools/launcher/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 398d8aad3d..82d118c5b7 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -143,7 +143,10 @@ class ActionModel(QtGui.QStandardItemModel): if not self.dbcon.Session.get("AVALON_PROJECT"): return actions - project_doc = self.dbcon.find_one({"type": "project"}) + project_doc = self.dbcon.find_one( + {"type": "project"}, + {"config.apps": True} + ) if not project_doc: return actions From 3a2deeb7c48acce8bd94a59a04ea40673b39f3cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 20:00:21 +0200 Subject: [PATCH 73/88] reorganized filter actions --- openpype/tools/launcher/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 82d118c5b7..3ceccf439f 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -183,14 +183,12 @@ class ActionModel(QtGui.QStandardItemModel): return icon def filter_actions(self): + self.items_by_id.clear() # Validate actions based on compatibility self.clear() - self.items_by_id.clear() actions = self.filter_compatible_actions(self._registered_actions) - self.beginResetModel() - single_actions = [] varianted_actions = collections.defaultdict(list) grouped_actions = collections.defaultdict(list) @@ -273,12 +271,17 @@ class ActionModel(QtGui.QStandardItemModel): items_by_order[order].append(item) + self.beginResetModel() + + items = [] for order in sorted(items_by_order.keys()): for item in items_by_order[order]: item_id = str(uuid.uuid4()) item.setData(item_id, ACTION_ID_ROLE) self.items_by_id[item_id] = item - self.appendRow(item) + items.append(item) + + self.invisibleRootItem().appendRows(items) self.endResetModel() From bac16b26ac5e29925d136a9e7e59192bc514df97 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Aug 2021 20:00:39 +0200 Subject: [PATCH 74/88] stop animation before filtering --- openpype/tools/launcher/widgets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/tools/launcher/widgets.py b/openpype/tools/launcher/widgets.py index 048210115c..0cdd129070 100644 --- a/openpype/tools/launcher/widgets.py +++ b/openpype/tools/launcher/widgets.py @@ -158,6 +158,8 @@ class ActionBar(QtWidgets.QWidget): self.model.discover() def filter_actions(self): + if self._animation_timer.isActive(): + self._animation_timer.stop() self.model.filter_actions() def set_row_height(self, rows): From 64d0fba55f4febe1daa0c96c749389758709b2e8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 11:55:16 +0200 Subject: [PATCH 75/88] enhanced project handler --- openpype/tools/launcher/lib.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/openpype/tools/launcher/lib.py b/openpype/tools/launcher/lib.py index 65d40cd0df..d6374f49d2 100644 --- a/openpype/tools/launcher/lib.py +++ b/openpype/tools/launcher/lib.py @@ -44,9 +44,12 @@ class ProjectHandler(QtCore.QObject): # Signal emmited when project has changed project_changed = QtCore.Signal(str) + projects_refreshed = QtCore.Signal() + timer_timeout = QtCore.Signal() def __init__(self, dbcon, model): super(ProjectHandler, self).__init__() + self._active = False # Store project model for usage self.model = model # Store dbcon @@ -54,6 +57,28 @@ class ProjectHandler(QtCore.QObject): self.current_project = dbcon.Session.get("AVALON_PROJECT") + refresh_timer = QtCore.QTimer() + refresh_timer.setInterval(self.refresh_interval) + refresh_timer.timeout.connect(self._on_timeout) + + self.refresh_timer = refresh_timer + + def _on_timeout(self): + if self._active: + self.timer_timeout.emit() + self.refresh_model() + + def set_active(self, active): + self._active = active + + def start_timer(self, trigger=False): + self.refresh_timer.start() + if trigger: + self._on_timeout() + + def stop_timer(self): + self.refresh_timer.stop() + def set_project(self, project_name): # Change current project of this handler self.current_project = project_name @@ -66,6 +91,7 @@ class ProjectHandler(QtCore.QObject): def refresh_model(self): self.model.refresh() + self.projects_refreshed.emit() def get_action_icon(action): From e0df3e92d26fcb088d58871c638fd7e59afbab56 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 11:56:02 +0200 Subject: [PATCH 76/88] use single refresh timer across all widgets --- openpype/tools/launcher/constants.py | 4 +- openpype/tools/launcher/widgets.py | 33 +++++++---------- openpype/tools/launcher/window.py | 55 +++++++--------------------- 3 files changed, 29 insertions(+), 63 deletions(-) diff --git a/openpype/tools/launcher/constants.py b/openpype/tools/launcher/constants.py index e6dbbb6e19..7f394cb5ac 100644 --- a/openpype/tools/launcher/constants.py +++ b/openpype/tools/launcher/constants.py @@ -8,5 +8,5 @@ ACTION_ID_ROLE = QtCore.Qt.UserRole + 3 ANIMATION_START_ROLE = QtCore.Qt.UserRole + 4 ANIMATION_STATE_ROLE = QtCore.Qt.UserRole + 5 - -ANIMATION_LEN = 10 +# Animation length in seconds +ANIMATION_LEN = 7 diff --git a/openpype/tools/launcher/widgets.py b/openpype/tools/launcher/widgets.py index 0cdd129070..35c7d98be1 100644 --- a/openpype/tools/launcher/widgets.py +++ b/openpype/tools/launcher/widgets.py @@ -40,16 +40,11 @@ class ProjectBar(QtWidgets.QWidget): QtWidgets.QSizePolicy.Maximum ) - refresh_timer = QtCore.QTimer() - refresh_timer.setInterval(project_handler.refresh_interval) - self.project_handler = project_handler self.project_delegate = project_delegate self.project_combobox = project_combobox - self.refresh_timer = refresh_timer # Signals - refresh_timer.timeout.connect(self._on_refresh_timeout) self.project_combobox.currentIndexChanged.connect(self.on_index_change) project_handler.project_changed.connect(self._on_project_change) @@ -58,20 +53,6 @@ class ProjectBar(QtWidgets.QWidget): if project_name: self.set_project(project_name) - def showEvent(self, event): - if not self.refresh_timer.isActive(): - self.refresh_timer.start() - super(ProjectBar, self).showEvent(event) - - def _on_refresh_timeout(self): - if not self.isVisible(): - # Stop timer if widget is not visible - self.refresh_timer.stop() - - elif self.isActiveWindow(): - # Refresh projects if window is active - self.project_handler.refresh_model() - def _on_project_change(self, project_name): if self.get_current_project() == project_name: return @@ -103,9 +84,10 @@ class ActionBar(QtWidgets.QWidget): action_clicked = QtCore.Signal(object) - def __init__(self, dbcon, parent=None): + def __init__(self, project_handler, dbcon, parent=None): super(ActionBar, self).__init__(parent) + self.project_handler = project_handler self.dbcon = dbcon layout = QtWidgets.QHBoxLayout(self) @@ -152,9 +134,12 @@ class ActionBar(QtWidgets.QWidget): self.set_row_height(1) + project_handler.projects_refreshed.connect(self._on_projects_refresh) view.clicked.connect(self.on_clicked) def discover_actions(self): + if self._animation_timer.isActive(): + self._animation_timer.stop() self.model.discover() def filter_actions(self): @@ -165,6 +150,9 @@ class ActionBar(QtWidgets.QWidget): def set_row_height(self, rows): self.setMinimumHeight(rows * 75) + def _on_projects_refresh(self): + self.discover_actions() + def _on_animation(self): time_now = time.time() for action_id in tuple(self._animated_items): @@ -184,6 +172,8 @@ class ActionBar(QtWidgets.QWidget): self.update() def _start_animation(self, index): + # Offset refresh timout + self.project_handler.start_timer() action_id = index.data(ACTION_ID_ROLE) item = self.model.items_by_id.get(action_id) if item: @@ -204,6 +194,9 @@ class ActionBar(QtWidgets.QWidget): self.action_clicked.emit(action) return + # Offset refresh timout + self.project_handler.start_timer() + actions = index.data(ACTION_ROLE) menu = QtWidgets.QMenu(self) diff --git a/openpype/tools/launcher/window.py b/openpype/tools/launcher/window.py index 979aab42cf..bd37a9b89c 100644 --- a/openpype/tools/launcher/window.py +++ b/openpype/tools/launcher/window.py @@ -103,14 +103,9 @@ class ProjectsPanel(QtWidgets.QWidget): layout.addWidget(view) - refresh_timer = QtCore.QTimer() - refresh_timer.setInterval(project_handler.refresh_interval) - - refresh_timer.timeout.connect(self._on_refresh_timeout) view.clicked.connect(self.on_clicked) self.view = view - self.refresh_timer = refresh_timer self.project_handler = project_handler def on_clicked(self, index): @@ -118,21 +113,6 @@ class ProjectsPanel(QtWidgets.QWidget): project_name = index.data(QtCore.Qt.DisplayRole) self.project_handler.set_project(project_name) - def showEvent(self, event): - self.project_handler.refresh_model() - if not self.refresh_timer.isActive(): - self.refresh_timer.start() - super(ProjectsPanel, self).showEvent(event) - - def _on_refresh_timeout(self): - if not self.isVisible(): - # Stop timer if widget is not visible - self.refresh_timer.stop() - - elif self.isActiveWindow(): - # Refresh projects if window is active - self.project_handler.refresh_model() - class AssetsPanel(QtWidgets.QWidget): """Assets page""" @@ -268,8 +248,6 @@ class AssetsPanel(QtWidgets.QWidget): class LauncherWindow(QtWidgets.QDialog): """Launcher interface""" - # Refresh actions each 10000msecs - actions_refresh_timeout = 10000 def __init__(self, parent=None): super(LauncherWindow, self).__init__(parent) @@ -304,7 +282,7 @@ class LauncherWindow(QtWidgets.QDialog): page_slider.addWidget(asset_panel) # actions - actions_bar = ActionBar(self.dbcon, self) + actions_bar = ActionBar(project_handler, self.dbcon, self) # statusbar statusbar = QtWidgets.QWidget() @@ -342,10 +320,6 @@ class LauncherWindow(QtWidgets.QDialog): layout.setSpacing(0) layout.setContentsMargins(0, 0, 0, 0) - actions_refresh_timer = QtCore.QTimer() - actions_refresh_timer.setInterval(self.actions_refresh_timeout) - - self.actions_refresh_timer = actions_refresh_timer self.project_handler = project_handler self.message_label = message_label @@ -357,22 +331,31 @@ class LauncherWindow(QtWidgets.QDialog): self._page = 0 # signals - actions_refresh_timer.timeout.connect(self._on_action_timer) actions_bar.action_clicked.connect(self.on_action_clicked) action_history.trigger_history.connect(self.on_history_action) project_handler.project_changed.connect(self.on_project_change) + project_handler.timer_timeout.connect(self._on_refresh_timeout) asset_panel.back_clicked.connect(self.on_back_clicked) asset_panel.session_changed.connect(self.on_session_changed) self.resize(520, 740) def showEvent(self, event): - if not self.actions_refresh_timer.isActive(): - self.actions_refresh_timer.start() - self.discover_actions() + self.project_handler.set_active(True) + self.project_handler.start_timer(True) super(LauncherWindow, self).showEvent(event) + def _on_refresh_timeout(self): + # Stop timer if widget is not visible + if not self.isVisible(): + self.project_handler.stop_timer() + + def changeEvent(self, event): + if event.type() == QtCore.QEvent.ActivationChange: + self.project_handler.set_active(self.isActiveWindow()) + super(LauncherWindow, self).changeEvent(event) + def set_page(self, page): current = self.page_slider.currentIndex() if current == page and self._page == page: @@ -392,20 +375,10 @@ class LauncherWindow(QtWidgets.QDialog): def discover_actions(self): self.actions_bar.discover_actions() - self.filter_actions() def filter_actions(self): self.actions_bar.filter_actions() - def _on_action_timer(self): - if not self.isVisible(): - # Stop timer if widget is not visible - self.actions_refresh_timer.stop() - - elif self.isActiveWindow(): - # Refresh projects if window is active - self.discover_actions() - def on_project_change(self, project_name): # Update the Action plug-ins available for the current project self.set_page(1) From dfa9132ac3275dc4e9ab8abd58ee4246c485cc4f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 11:58:04 +0200 Subject: [PATCH 77/88] trigger filtering after discover --- openpype/tools/launcher/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/tools/launcher/models.py b/openpype/tools/launcher/models.py index 3ceccf439f..4988829c11 100644 --- a/openpype/tools/launcher/models.py +++ b/openpype/tools/launcher/models.py @@ -138,6 +138,8 @@ class ActionModel(QtGui.QStandardItemModel): self._registered_actions = actions + self.filter_actions() + def get_application_actions(self): actions = [] if not self.dbcon.Session.get("AVALON_PROJECT"): From 5578784360a8b15a12be2ccd5d9cae222975c41e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 12:15:24 +0200 Subject: [PATCH 78/88] fixed set focus for dictionary widget --- .../tools/settings/settings/item_widgets.py | 19 +++++++++++++++++++ openpype/tools/settings/settings/widgets.py | 2 ++ 2 files changed, 21 insertions(+) diff --git a/openpype/tools/settings/settings/item_widgets.py b/openpype/tools/settings/settings/item_widgets.py index a808caa465..e3372ac2c4 100644 --- a/openpype/tools/settings/settings/item_widgets.py +++ b/openpype/tools/settings/settings/item_widgets.py @@ -89,6 +89,25 @@ class DictImmutableKeysWidget(BaseWidget): self._prepare_entity_layouts(child["children"], wrapper) + def set_focus(self, scroll_to=False): + """Set focus of a widget. + + Args: + scroll_to(bool): Also scroll to widget in category widget. + """ + if self.body_widget: + if scroll_to: + self.scroll_to(self.body_widget.top_part) + self.body_widget.top_part.setFocus() + + else: + if scroll_to: + if not self.input_fields: + self.scroll_to(self) + else: + self.scroll_to(self.input_fields[0]) + self.setFocus() + def _ui_item_base(self): self.setObjectName("DictInvisible") diff --git a/openpype/tools/settings/settings/widgets.py b/openpype/tools/settings/settings/widgets.py index d49057e1e8..b821c3bb2c 100644 --- a/openpype/tools/settings/settings/widgets.py +++ b/openpype/tools/settings/settings/widgets.py @@ -221,6 +221,8 @@ class ExpandingWidget(QtWidgets.QWidget): self.main_layout.setSpacing(0) self.main_layout.addWidget(top_part) + self.top_part = top_part + def hide_toolbox(self, hide_content=False): self.button_toggle.setArrowType(QtCore.Qt.NoArrow) self.toolbox_hidden = True From 472ae6bd4623ab8493f1db123b16eaeda453fd9c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 14:52:46 +0200 Subject: [PATCH 79/88] stretch second column in dictionary widget --- openpype/tools/settings/settings/item_widgets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/tools/settings/settings/item_widgets.py b/openpype/tools/settings/settings/item_widgets.py index e3372ac2c4..b2b129da86 100644 --- a/openpype/tools/settings/settings/item_widgets.py +++ b/openpype/tools/settings/settings/item_widgets.py @@ -48,6 +48,10 @@ class DictImmutableKeysWidget(BaseWidget): self._ui_item_base() label = self.entity.label + # Set stretch of second column to 1 + if isinstance(self.content_layout, QtWidgets.QGridLayout): + self.content_layout.setColumnStretch(1, 1) + self._direct_children_widgets = [] self._parent_widget_by_entity_id = {} self._added_wrapper_ids = set() From 79314142bbb2e481d716b71608bdffa41e99d657 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 26 Aug 2021 14:53:19 +0200 Subject: [PATCH 80/88] fixed single selection of deadline url --- openpype/settings/entities/enum_entity.py | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py index 5db31959a5..ed5da5bd9a 100644 --- a/openpype/settings/entities/enum_entity.py +++ b/openpype/settings/entities/enum_entity.py @@ -458,27 +458,19 @@ class DeadlineUrlEnumEntity(BaseEnumEntity): self.valid_value_types = (list,) self.value_on_not_set = [] else: - for key in self.valid_keys: - if self.value_on_not_set is NOT_SET: - self.value_on_not_set = key - break - self.valid_value_types = (STRING_TYPE,) + self.value_on_not_set = "" # GUI attribute self.placeholder = self.schema_data.get("placeholder") def _get_enum_values(self): - system_settings_entity = self.get_entity_from_path("system_settings") + deadline_urls_entity = self.get_entity_from_path( + "system_settings/modules/deadline/deadline_urls" + ) valid_keys = set() enum_items_list = [] - deadline_urls_entity = ( - system_settings_entity - ["modules"] - ["deadline"] - ["deadline_urls"] - ) for server_name, url_entity in deadline_urls_entity.items(): enum_items_list.append( {server_name: "{}: {}".format(server_name, url_entity.value)}) @@ -489,8 +481,16 @@ class DeadlineUrlEnumEntity(BaseEnumEntity): super(DeadlineUrlEnumEntity, self).set_override_state(*args, **kwargs) self.enum_items, self.valid_keys = self._get_enum_values() - new_value = [] - for key in self._current_value: - if key in self.valid_keys: - new_value.append(key) - self._current_value = new_value + if self.multiselection: + new_value = [] + for key in self._current_value: + if key in self.valid_keys: + new_value.append(key) + self._current_value = new_value + + else: + if not self.valid_keys: + self._current_value = "" + + elif self._current_value not in self.valid_keys: + self._current_value = tuple(self.valid_keys)[0] From a8fd971b06a6f417599d7874d596c1a31a3eb1ae Mon Sep 17 00:00:00 2001 From: Petr Dvorak Date: Fri, 27 Aug 2021 10:58:36 +0200 Subject: [PATCH 81/88] Removed deprecated submodules --- openpype/modules/ftrack/python2_vendor/arrow | 1 - openpype/modules/ftrack/python2_vendor/ftrack-python-api | 1 - 2 files changed, 2 deletions(-) delete mode 160000 openpype/modules/ftrack/python2_vendor/arrow delete mode 160000 openpype/modules/ftrack/python2_vendor/ftrack-python-api diff --git a/openpype/modules/ftrack/python2_vendor/arrow b/openpype/modules/ftrack/python2_vendor/arrow deleted file mode 160000 index b746fedf72..0000000000 --- a/openpype/modules/ftrack/python2_vendor/arrow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b746fedf7286c3755a46f07ab72f4c414cd41fc0 diff --git a/openpype/modules/ftrack/python2_vendor/ftrack-python-api b/openpype/modules/ftrack/python2_vendor/ftrack-python-api deleted file mode 160000 index d277f474ab..0000000000 --- a/openpype/modules/ftrack/python2_vendor/ftrack-python-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d277f474ab016e7b53479c36af87cb861d0cc53e From 412dccc6182d456f660aca01b9f7444627f31686 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Fri, 27 Aug 2021 11:08:38 +0100 Subject: [PATCH 82/88] Fix PyQt5 on Windows build. --- tools/build_dependencies.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/build_dependencies.py b/tools/build_dependencies.py index 3898450471..e5a430e220 100644 --- a/tools/build_dependencies.py +++ b/tools/build_dependencies.py @@ -135,6 +135,16 @@ progress_bar.close() # iterate over frozen libs and create list to delete libs_dir = build_dir / "lib" +# On Windows "python3.dll" is needed for PyQt5 from the build. +if platform.system().lower() == "windows": + src = Path(libs_dir / "PyQt5" / "python3.dll") + dst = Path(deps_dir / "PyQt5" / "python3.dll") + if src.exists(): + shutil.copyfile(src, dst) + else: + _print("Could not find {}".format(src), 1) + sys.exit(1) + to_delete = [] # _print("Finding duplicates ...") deps_items = list(deps_dir.iterdir()) From d8b62cd2dd129fdf7307e782c95879ce74aec6b8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 27 Aug 2021 14:17:05 +0200 Subject: [PATCH 83/88] fixed hierarchy position of houdini submit to deadline plugins --- .../deadline/plugins/publish/submit_houdini_remote_publish.py | 0 .../deadline/plugins/publish/submit_houdini_render_deadline.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename openpype/modules/{ => default_modules}/deadline/plugins/publish/submit_houdini_remote_publish.py (100%) rename openpype/modules/{ => default_modules}/deadline/plugins/publish/submit_houdini_render_deadline.py (100%) diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py similarity index 100% rename from openpype/modules/deadline/plugins/publish/submit_houdini_remote_publish.py rename to openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_remote_publish.py diff --git a/openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py b/openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py similarity index 100% rename from openpype/modules/deadline/plugins/publish/submit_houdini_render_deadline.py rename to openpype/modules/default_modules/deadline/plugins/publish/submit_houdini_render_deadline.py From ca1dfbd98c24f9b8bc22961a1cb91eac5436cdbf Mon Sep 17 00:00:00 2001 From: OpenPype Date: Sat, 28 Aug 2021 03:38:52 +0000 Subject: [PATCH 84/88] [Automated] Bump version --- CHANGELOG.md | 27 +++++++++++++++------------ openpype/version.py | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c55be842a..4259a0f725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,26 @@ # Changelog -## [3.4.0-nightly.2](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.4.0-nightly.3](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.3.1...HEAD) **Merged pull requests:** +- Removed deprecated submodules [\#1967](https://github.com/pypeclub/OpenPype/pull/1967) +- Launcher: Fix crashes on action click [\#1964](https://github.com/pypeclub/OpenPype/pull/1964) +- Settings: Minor fixes in UI and missing default values [\#1963](https://github.com/pypeclub/OpenPype/pull/1963) +- Blender: Toggle system console works on windows [\#1962](https://github.com/pypeclub/OpenPype/pull/1962) +- Resolve path when adding to zip [\#1960](https://github.com/pypeclub/OpenPype/pull/1960) +- Bump url-parse from 1.5.1 to 1.5.3 in /website [\#1958](https://github.com/pypeclub/OpenPype/pull/1958) +- Global: Avalon Host name collector [\#1949](https://github.com/pypeclub/OpenPype/pull/1949) +- Global: Define hosts in CollectSceneVersion [\#1948](https://github.com/pypeclub/OpenPype/pull/1948) - Maya: Add Xgen family support [\#1947](https://github.com/pypeclub/OpenPype/pull/1947) - Add face sets to exported alembics [\#1942](https://github.com/pypeclub/OpenPype/pull/1942) -- Environments: Tool environments in alphabetical order [\#1910](https://github.com/pypeclub/OpenPype/pull/1910) +- Bump path-parse from 1.0.6 to 1.0.7 in /website [\#1933](https://github.com/pypeclub/OpenPype/pull/1933) +- \#1894 - adds host to template\_name\_profiles for filtering [\#1915](https://github.com/pypeclub/OpenPype/pull/1915) +- Disregard publishing time. [\#1888](https://github.com/pypeclub/OpenPype/pull/1888) - Dynamic modules [\#1872](https://github.com/pypeclub/OpenPype/pull/1872) +- Houdini: add Camera, Point Cache, Composite, Redshift ROP and VDB Cache support [\#1821](https://github.com/pypeclub/OpenPype/pull/1821) ## [3.3.1](https://github.com/pypeclub/OpenPype/tree/3.3.1) (2021-08-20) @@ -45,6 +56,7 @@ - Nuke: update video file crassing [\#1916](https://github.com/pypeclub/OpenPype/pull/1916) - Fix - texture validators for workfiles triggers only for textures workfiles [\#1914](https://github.com/pypeclub/OpenPype/pull/1914) - submodules: avalon-core update [\#1911](https://github.com/pypeclub/OpenPype/pull/1911) +- Environments: Tool environments in alphabetical order [\#1910](https://github.com/pypeclub/OpenPype/pull/1910) - Settings UI: List order works as expected [\#1906](https://github.com/pypeclub/OpenPype/pull/1906) - Add support for multiple Deadline β˜ οΈβž– servers [\#1905](https://github.com/pypeclub/OpenPype/pull/1905) - Hiero: loaded clip was not set colorspace from version data [\#1904](https://github.com/pypeclub/OpenPype/pull/1904) @@ -63,6 +75,7 @@ - TVPaint: Increment workfile [\#1885](https://github.com/pypeclub/OpenPype/pull/1885) - Allow Multiple Notes to run on tasks. [\#1882](https://github.com/pypeclub/OpenPype/pull/1882) - Normalize path returned from Workfiles. [\#1880](https://github.com/pypeclub/OpenPype/pull/1880) +- Feature/webpublisher backend [\#1876](https://github.com/pypeclub/OpenPype/pull/1876) - Prepare for pyside2 [\#1869](https://github.com/pypeclub/OpenPype/pull/1869) - Filter hosts in settings host-enum [\#1868](https://github.com/pypeclub/OpenPype/pull/1868) - Local actions with process identifier [\#1867](https://github.com/pypeclub/OpenPype/pull/1867) @@ -70,23 +83,13 @@ - Maya: add support for `RedshiftNormalMap` node, fix `tx` linear space πŸš€ [\#1863](https://github.com/pypeclub/OpenPype/pull/1863) - Workfiles tool event arguments fix [\#1862](https://github.com/pypeclub/OpenPype/pull/1862) - Maya: support for configurable `dirmap` πŸ—ΊοΈ [\#1859](https://github.com/pypeclub/OpenPype/pull/1859) -- Maya: don't add reference members as connections to the container set πŸ“¦ [\#1855](https://github.com/pypeclub/OpenPype/pull/1855) - Settings list can use template or schema as object type [\#1815](https://github.com/pypeclub/OpenPype/pull/1815) - Maya: expected files -\> render products βš™οΈ overhaul [\#1812](https://github.com/pypeclub/OpenPype/pull/1812) -- Settings error dialog on show [\#1798](https://github.com/pypeclub/OpenPype/pull/1798) ## [3.2.0](https://github.com/pypeclub/OpenPype/tree/3.2.0) (2021-07-13) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/CI/3.2.0-nightly.7...3.2.0) -**Merged pull requests:** - -- Build: don't add Poetry to `PATH` [\#1808](https://github.com/pypeclub/OpenPype/pull/1808) -- Nuke: ftrack family plugin settings preset [\#1805](https://github.com/pypeclub/OpenPype/pull/1805) -- nuke: fixing wrong name of family folder when `used existing frames` [\#1803](https://github.com/pypeclub/OpenPype/pull/1803) -- Collect ftrack family bugs [\#1801](https://github.com/pypeclub/OpenPype/pull/1801) -- Standalone publisher last project [\#1799](https://github.com/pypeclub/OpenPype/pull/1799) - ## [2.18.4](https://github.com/pypeclub/OpenPype/tree/2.18.4) (2021-06-24) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/2.18.3...2.18.4) diff --git a/openpype/version.py b/openpype/version.py index 5fd6520953..2e769a1b62 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.4.0-nightly.2" +__version__ = "3.4.0-nightly.3" From 6533934577e22ec53c2f4a93811d696a07054b9e Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 28 Aug 2021 15:12:56 +0200 Subject: [PATCH 85/88] Added Moonrock Animation Studio to OpenPype users --- website/src/pages/index.js | 7 ++++++- website/static/img/moonrock_logo.png | Bin 0 -> 22947 bytes 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 website/static/img/moonrock_logo.png diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 6a233ddb66..00cf002aec 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -120,7 +120,12 @@ const studios = [ title: "Bad Clay", image: "/img/badClay_logo.png", infoLink: "https://www.bad-clay.com/", - } + }, + { + title: "Moonrock Animation Studio", + image: "/img/moonrock_logo.png", + infoLink: "https://www.moonrock.eu/", + } ]; function Service({imageUrl, title, description}) { diff --git a/website/static/img/moonrock_logo.png b/website/static/img/moonrock_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..249db7c2470b37bc24cd4649feb418238a690c34 GIT binary patch literal 22947 zcmdqJgNJyh7NFyO7B8a4fboWS?Gztg?3P`7P4?Q9hA}P%PLnA{o zFu)Ms=J~zf`$xQ97neG-&p!L?z1F?%b+2`ZefCuK8aWF&1OmCHuBNODf#98jpA}MK z@Co}TITG*}nY)^i7X-rNdi8@>qel81e0alK#nAh?o2|FcD^DAUkB^VAgR7I5)hl-! zVK+~^v~3v{2;?S2UHOroZ~Eq}Pdek+`Q_e(_~&YdB%YfigpZt=H(4I!+}+TQ7h(y> zVRm{V#zL?4J8V{ib_NF4GQNtfeY^>E&?WOW{wq!*RdKS`Wqjiqd9C@{PQBT|&I<)16yBKA zg}G3rWq*5x;lVuQZ5R=vBfgdharU5$=Z^$Gbm2}F)lDe67{RzB6xzn>V6s5&OD9BJ zfaeuD<#f?n97HZy^Oi@qMQ>fj`6Gb|!3>^s>B$;R7E`a#a%3t&AZcJ_L2LVJyMa*@ zU9S|Qf71F3yk>~>BKheN!Xhi*rYOQ5Qm`0zfrE=cDhLh)m13=9$y?Qqn+T){0`LRy zeuiEzh#j$`+NImwPYr3KxSz^Hr_KlF>G+89U&lcLP;t1=1+mo*w7dMfRP;nRZ2rC5T%tL5zzM=@-pL%#lFl5&Sx$di;;D8W= zcU*=b9KpBHWbYM37=j(~5_z!d_|LvCGz8MURNfX769aMe{Ms5iTm-kvFC)CTmQGo# zqO2S@(DW-v{3udgs7e3lbk006Ep2aYFv2{vqEbY`ajzjCK69`678Lq*82=%$FAMfb zJ*-Mc^D&MYPTCM08WWQ@q=3>uIc>g;GxnU}NA19Q1u)9VF=`#32;>o3<(8%jqazaq z5GMRB8rme~Vtippft9)7=*ax%MPkPzMP=nDFaqIjtxt}oY@^i^8y>fhe88$K>j>Nh zw*4H?mRb62Tihs7Z%%V%F5PAbeWm=r5=OTG&WCi2fwW1_uKkFAH@@WIsP3?{#4)4y zgNc?8VPPRS6nY+Os^wy}^#I8o1fJ*-CsOv(#zu2opNA^Z((=)N7 zeha_@PQX?UUGrteaePWO)}0POh$=&KI9faN2_N1#W~!|5xSquIk35~&yQ4qX+=*3H znak%q)UI+zvbdPLHHeB5e88;{W}fqDWpvcjAs(@%zzDrH1f`_=_gmB%$TI_uSp;l^mG1x3E~1+kA##CkrZK`Re8_zk4l=MqSRd zDJ77NS|%ZFSX0?q55F72?m@5Xpx5U75lkceJ4}9YeC&GC^0A0Jo9Lia3|As~E8*A; zU(!kNpBU%S6u9O|g$YTL^{}iznWZ#JNN-ssxs#f@;eS*JO^Ul=fVwHnX;n^CI(6aJls>`3k*OsuAVD{`&I)lym1}f;@^XwF! zS@#1wCkLY>=kLUA(P=u`aWNbf^r2=5ilUyk)1FX=F>07; zh&Ch|Jj?O=UelsDsljrKIzDdP{n2|bG!xQbOhTK~;X*Cgd2U&fUv&8I zZ+G){6T0QsTc&{iud)I4KE2#2zFRX55`(3mzmG%CXlT{z@DOa9+IJi!Ym<}dcK(PB zVm=F*j5~+T552n~)*_2V{*`Gr{%$PmP6KdYh2ShzQVXV%T&9tF z{mxZ?QNPUCV>wCnaeYv4tx~Ad)t7qfHCApGf~n&V>!@TzPn(5sn>E_Yxc7ff`P|!J zq`lwBgTyfe97OzO;c6FA)SMhAa`p4UxE}1Sudms&`V{$c=gd#*h|Idnghcr2Rz92D z^IHAWgRSvg(U`7x(Nt1WTE->+F~-{@eo{{b{ZXWv>B^0mE$S(p1NOq&7i?VTsIoKH zYBiN&tnN0{(bU@`)l0;cabLkbaH;oKNw^; z08K-e0QbkB{12hG9#T$By^naFtzB3aN;2eC_LbHy+ikW9<_vC*BXF;Tkve3M2c6b- z##ioGvnzM#=<0T4J=(igV?R}Gb2UhSXq)8Jm?TLH%#mpl`mIb!&DS5h&5jN_u<85! zt6qeUJPnVUpo#b6GjMy)->Ez|=+~4;89%A}t>z-}PWR z-PqVL#PX^vu^}O{>dcI%-gmlZNu2D!?F%_tt(D2#d58L?VoJ3m<=BOZUN=zP8jd-* zaoW@*rC(}DJ9Ck@?oE6Vl~JM1z<9dM8kO;)zjS0ow|>etZSttF?}#ouJk+a9smGWf z)2koZY;FFET{&mC!EN4;_cfBJLdR){keyEJqe@#-M{2;c9`Xpq9NYy zBY2pAE|(yAtMKgG?4Yo}rAiF9!Gqi(*j6OakBjri=th*|H5o_W5eKW-wZf2U4C^GC z#SPgaGQm(*M*o1Vp*%X3!knBpWeo`-BT9$s+;~q3+HBZ!z7!43G*b$p4ZVpenHAn% z7qXnM@W&)W@)7p;MMc1T^{04i;8~B2Ue=QsR$IsR2CIlylDNNv#aL!5x;ZgyPxi*Rkyg=Szr(Wnj*=gmF5TX#8Qj%nBAjeTZhq~0EU{=ao7lzE7~(TLDnUz(shjt5iK^o!3t zGc)7B1&`&XL`acoowM^Xo*sWA!qigTdR_V$S)wXJ_Ja6qKuY;@@lbckwOuDPjb-2W zp{vhdv$?iX$A*#zwG2EukYSPq#L(@IO6zG)(NK3G`EetdzfcnkWjzeC_yUU&)Xh#D zj8tF(*O$B~(Z>LQww~V@*6jEMX|JP7QsKw!*(VBMbeD;*u_zlEeeG=nOJaCPIoU6z zJPu}D!bk4E(_JnVo(Ts#%Ca#Q{UlP({}W;g*vJh*idOI;9SOnf_B%K76ggX;532Dv6OFGb}~PY|XBioUN56j@huO z304gXk`>S|6`g8ORDk!Dl>OKkJYR<$6oJ7`wcwc=IBkL+o*+9E%CQs!{ZN z>)~7&R1!^aCXc3ZC{*N(=gfA?uEslDSK0TBf`!&T&!#r#1NM+~=`c~wE*o%kf2JL* z+>!*}#3))#jPZ0gh>87Ub(l-T>}L?aYR`P9P<&}p!<=6Dwyes`W=ksXThq3 zn{=P8$_M~k3wr2{>g6#iHzuhJJn?jNa?%ouiCBM#hB)LZn!`0vqTs5Rp*<1^8@9`i&3_sA*QD!>CVZAHUZDT0{%Tj zBDv{+;l47??K>KwME4zK0nF4jx3B|8k*#R{g&>Uo>JDICz(OChyRhFB0`nipq9Mu7 z|K~%8q_j}*nb&(5M7{n;IEjfWn7Z0AAS)UcbN^ZP;C#J;$kiMmPUM0Wjo;N<=PuEB zW+y_}5WeSRlyy|MdiDPciQ%PIkV@-bIy7l=G%(rM(+<3C*w&q@ z8Q13Fwx9sp!HoV0GJ0kSS2jTO#%Vtk&`I3cd4Rr|92|g`k!tK!W>8^frYm!{`*Srj z+ISaCy2$R)NjtaP`$6JFENG_{+maz0kZPx8fg`HcY4XlwHLYhdS8sHt2LI-najv;m z(Xvl|dA25>CL0BqMnHz|BAIcmJ*`!f-q493z23T4E`fR=ckz&usF_r{_lSDS$!gjr zAQNF`?LVoc-MensDW~(pFv_DmMMG=0cEBD2)%|LzMsu~B_V+zqo43Tf64*5XdwLpz zRh2*0Q7+akp|8w`#c%<0clF9@hLNhvfg^~CizDaoMTNBltV(agtt@TLpS<5HBj8FR z4|10_BzeCqAS9&qCkB|STrt0ZfFEX9Rq}K__h9a^x;hyF2^Hnt7d*~(>^EaEzNYS{ z1$Wv2od5vlQfhUU=#S}ruCL#fn<)4DRy3RLZhf+MDR%_<`Qsv0&UC7qd!O>I>^+Fi z)Mpl!&EOXbh+Aa!H%gso=i7Sg;eYYC+LHBw6>5LZJX_{X2YfNxW$>>QF)IscZ7o&O zeSXZ3k9fU?FJIDWMVg7*9Pci?3^_l{cJ=CdcO%?GLMRs#I@L+J~oM&BmW)V-2FNYrKbe|1(Qbuwqu=VZe%OAQ!)C7w3ZR5?Yp!d zj_c{`8+RKf^7K_zfnD_4@8wmkKkxm(E9Nkg*HfQK3kGrl`9?lSjKN&JS)Q67a~+uW z51|I6qPn`eH4ct0Z-&mij#_VN{lD`T4^bcnLoN=~GTVxZix=hscH*6JH%4Zp6q1XE zaA*er9U`%jX4v(mx-or|y|no=o{=DAFm6lGu=3_ekhQW18ls+~xKmLBP*9H1zi}ao z6>7@aR$iU}?j~HlW%rJw!X%o!wY?uaM)eCly-JX}nRFi$n>Hm5>Zr0a_TFzmvq--# z)i0qazfCi%5c^Rl8(ZiWHBuTzK%%RY)WRz|mYb+LQ}1%)fumj?&feZ0^}VD#&YSbb zbE?_~sw4(}%moNQw%mKwvi9Ui_}K-vz0hvZK6+o~XwoY1^j~4p_U-UdbrYq?Hots? zEeO(6$%gB!EG&ji-o>zYe*pMwJ;WltZqE<)B;CF}uS#6}p+1T{=s*fpfVe_ftssi+ zi*ppT|^i9j%`eFMj&?QPh6uW)v0s z@1bm^8#2D4CeBqv-=uvCKokI+uFC(=@w$lFpJZ;${6pRV7dRa(jN|&l$Bizk_Wdd@Byi6 z+9lhfE3iPf^~{y7^pJ~_)tv{wwh~FRG_k?PMkJNG0DIowQkrxuDFacMB0oQ0(6~0{ z(Teb9kpM~BA%FmyUH~4N=b^C&iczEzf?CU)njb&Wz zRW7*qlk-A!|6I?&zzgh@YFj3y6R^sW1;WktPhHbZK0Dj9fd~C&Fc_>xHlB#$x-UKN zQw%_&CH5f9@SJU$i*3ei&iOy>khpxi6ygUD9?TipeY3d`i>-d4ukX(pa^@kJx%R@q z)aD}i`CPdwSLokLc|$8F0I?Xb*0c%r;Nx;Oju&Q&j*0RYzCDFl zOf!fNVgcqSBbp-dD`KT(WrzMpn+{iVxvFr5(oS-gR@JR#hRD^@kA38O-ZFo2)L@n= z7xZV|;Nw=!pd7`PSYNWoQkNgtrB(S%Q4>JbTT)X~4dvQ#+s%V#=eAk1@oJE@S0R;M zBjWRcUKLeUcQN$(R#vhj+9!gSBLi5wM*1MCf3r`e{uiJ>8yQ@hWM6ED5uPX_P& z5r5SYF)7!atrVTwK2KTe5kt63u}M8{IE@G+xGnG3pCLQbyqv7rlE5wv28d1%Nbj(n z*nNI|`|&%`OUcOL1j*GelX5k&^t|BfceScCWGK3;>y4XVZ&^ww37NMcoAh>;Ccw8ffL{laDX7|<7Fm-xN9bKcAc83AW&|%;arVt)pjaI7!6)N!fwx6 zO-+E^WR?#O6vXl4|C3!;EKr}*Ij0QxBa zU5!9EDzjR)Ss%(y_ZpC_+HxB$(z!1#-c{fJt^|)Zx6Y#*{h9LfPGjFco!|zZ1>oV` z7t6?$LPI=KCB1l-y5g5UK7V?5Dg=;IW-(hz-f!M?b{T#=fU{(%C)~UeA%Me_TID?k zo(ezRD3Js#2M-7c+~LI6a!eXr^^*cga;?84m_CJv=n)NiiZrq2@n%UQ$J^qYh~#vW)Mzt6;`3}8+dtv)9!DZGxxV@~CbU8h!j z#w7+7N862S92Gu0b4h2;)DKaihV$CM6C|vAXb!BT!ng30@sDo+JX2>CwFqqpJlQwY z_6_EFn!XO(nVUVU3G=m6{}69d=lIFz7o$Oh)RHB@QVE;i9}VZbKWNK21k2df&W5W& zRK$q3yQcpYW^$m(M6Q4MMwsJ2D#7cqba*?aVxWD+}zwRVPh}O4l6?P^YXZk z&Z#+MCav469Y7F}r002J(SCl^7r-hp>rF3YQcrpJQ+v{F)kQzEQk)>PZX6HD$8CO^ zr38j1RhC5EY+jTA2gh~V77W@?8Lhe#;)DFhuC%Zwd9ngPB4sX)mp0)`36htyONsIs z3qPY(-#h+Y?N93trbCnrB)Li ziB?eBOP87YmA!6GJ0LSqocT;QxcwJlpVWC+b-jBv_WecGPD6}fCDnTn!}|qJHL+Si zu@wdc+6hbsfR^R(AhKy ztsRmI&HMRzlKPJNlalY>_vHXzHY5>Pu{=B8GgDDfSDML~a$Nx!$`B5`OcV6pB~-g-^!A)!iL{&YAI zzMk_II@2aRQ>eNI0*mbR0XdxWYaqrl(@Sd0>=9-LpE_*fgcJ_|QSyKC^CHb*(U>l? z!h8bw==ENl9J;iobUt-C3#hT^;oNSC-~`}@Z(+l!FvioCc4O2yE#7D@wPgk@luKjr z;1Jz(w%^yd|64$5zT6-4Z{+(6f#AKj=7CF>i5#-2RP0in8Eab&^Q!>S-UYR*6wwyd zQ{DQv$9y#ZgnlZD@<#GH?l5D`Phvi>#=zK^WjY|^p-0Z4NitiRVO0kj*?4dn90f2U z%jok>0>Uvx2me}E_Kjg2Zdyt~S1Uv2D}^u|A@#UsKng9m-HJ8i=jT_P$^Z-b+XQnO zkj1`)fUKtR;uNjr83_Db2AmZSPFRs2bH$D&CIJ7O_2_0t0zVnQn+oe7{x3!`=-K1AJw87nU5fff+XH|pHx=gxfUVy4S_E{jxm+?rsa zdH?wfyAPXy0Z)MAzaZ>vN@0dSf8jr!i3h&BsQ2{h=>zhYXOm224xVe@e5$S0hef{8 z@*b{FWGntR!Tn=YI5k~JJBI}zFKb}0^8&Ejl9#4JwjW$=Y;1UNF81~of4gY#4F6!jKo?yZ*qw$hhK!l-vXoVJL3P1zdlB_s`;Pf z0378SplCcGTvR74I<5ZsO35iFvnyyrab3{jpkE^K;V0{U0L24N6|J`Hxsw>ewraqh zH-<^XW~CI2OAV{uSS8AeC!r_Yia?s(a=iF9Y+Pg>IGWxvx1C5UCOIzf%rQ(7$PdaO zs~RjN$m<1pcjImcMVS>Lh~-RRe|tCr?zdk=3eRZD0qSauCjDj4(=^+}!FXzsEIr|* zctuNPF3%3e@8bU($|lg3J!BwZb4T!)=PHUco&s+Hh3OIYL*|DvXkZ~<4 zJDOv$_U}_%S0<2Fgs(_9;9nAG=3m28F^T_wFMvj5Fafys34y*zv>NN+!Jk#eL6C~x z0C_L3Jb;=B+-0hus;37)nJX;{T)ja6lAhdd^cx&x=pCP=8by8 zom00QZqtgL+YYczrZVHYoPvTVdTp6usf4`FbL{-!WR+E!6+_8j7>1H64rrJ^ObK5) zutVePCoL)3ju$C_LzU$&+f3IwNDk)k{&E8RrUj75@w;(j@?ksR;aFyKskdE#hW5a2 zfDMQbcxYz7=SeA989_%zjD998F%7g}BUNUXme&-r}6aMEmUm3`bOxn zCg-Zl{&KGj;05D%m7!OJJ|l2n9qfr~GQb_(?7?{9c_vg7y1WKxd0#;c8ksEmYH&>m zQeYgd1_iQF3&^sQzC%jQ7?AjTR$Y`$ueTi8Rk#=cM<@XF z;LkVd2`P;#ocQjWE9E6oPU3-mrRpR73bX#QhB-%Y-}@l!y+~>>AZY<|dGToO;H(lMj8vdsS4%bZ>VMG0|A}%3kLB7F(A&<6hN4k`T`vPm02*C# z%~Xxu0l*gHZkn~rQUWeeNejN9D6OYTA_B1$D5=RccdfdfnV2}SE2S4+4dAHgn^(3l z2LdOs?p^@I|HjMGU<^qLj*gD&VBq zDp4RBU;mRSpNZRPpO=b?sp#emd8s02Vpd)zqrtH)=BJodC){$9b^MXmnX_%mMchMVT~Rj6Wta;E$BT z=PLzkoL0H*6=r>yf4_I1lz%_0Sk)Xpj*!XvEv0td9-m!36_o0#Kn=3`O2E8RJ)VIO z0E<HS4 zdyuNZ82CI74-YZuiNvpBcE+knb>$?&K=kF=7>-`zv#5f_lo|qM62y&67$q%7@bN3g zljUK199yv}XU0qzr%7~A+@o;fy@hB_UELfdz0FPFb}b+o8^3E^CK7s~kPCZ%Unfyc zCiUk7Wsv9I1vC+Cgo^XVXfb$1FA4sANl;=K3<(=7Q?iaNU}e-yl*tk3B>PK>v;wdK ziULb`Az&A|!J!IXr718 zS1PJ>#}&X#z9MINMd;<#6x1@i5RkQ0f%~t59J3ajl?kB?s8p2wE+w!Y;zt>xX^a4x zD*!!UD&+FKELSbbu*LW8RkXd5f_1FRq=!4L{hlYv1zr-b8b|je1Lpe`Ff_yN_fQXD z9@YP0bG4jly-27(X2A(EjgMnl>v{J{z+j}tPCM!vt1Y|Ay>cK-eRwJro3EL2wc~Sy z7Z#veGMby%RTf%~i(Cg=m|LZjIH*y};lF;%wm;Q|U1{7W{=aHWVi((eEvurdq!K7* zoM`4`kKD)c)!h3*5GKl(pBU105)dy9Wa{2LT+bbY z6j89`fSRvs(?1??vs+hoaCyCuS?)QhX6fM^5P$y8O^tx6#2`Z24I?{Zd-+bnJJ0bzs=sCS#7({rV zS;Q;_F9mo(GN9mZ!M9UVQF(th#!7D7x2G`=+>3-U7&$>%`WVO#*Oy@IsSv2%#+as@ zD|~_Y%8^=6Vt^dp{x(eh%8`IzAa{A@Rgd)41kXPhpL7g*p;i+D_+;CJs3lwApjFs`?*$W4uHe-6mArQ*1d zwGMxofe;7)qCNrY6i*Ng3RZphUnRu*gyn_`k3Xum0LhSZ)vxPUK0L0&wJ`8-iSjBw zpqZ=3`B7>Xv9R}o&lBecULo2`!2*DS^hFBBZ)0=1&CiMj;8(qt5wPf&0UjBxfrN9% zz8{EtEk|^O51I~o-)w9+zD9beg1V|Ty9!-VRaKHzf@D-h+bLLE=>yJ(R6{`@V;n>? z%FOXoyB~1*k3JP&0fA{DUSbXa2D+Kev^p_Tcy^|l{viN!{T9$$(qtflHmY~N0cf}N zK)U3We`<$rg-M}r{!)XCQ?y`|MLb!)Pk@FF1SO6H0p|O zVB4^=U^CkehLdNj0E$T363F5S{nF0@g2r&2lL6qR66iaY)`uHoY@niK93^fvNBkI7 z(DI4782d%Aov;m*K8S$7=VWL9kcwiy%B+A&83)|Eg!UoHkVrl#+3i^P?Ft-(3sz9lDx9`O9?rrw_vb&%25o%qS`}%`Xkv(`7$=Utr)MynI{q{z1f4 z={UxJb>t}sw=yLLP~x?gq8{r1h2IKtxvOm-zsvPFQD&cXsXGyn04G00Vrc&Oqqwia zv%G)FEreZP@bV${SNg6hMP*iMlpwM{1S*nIwQ|jOo}PaDkFAnmkxqiHmSlX0+lewA zz%EO{xL|DaJYWlySGQh6sEprImOaf&BTc8&Ga^wJ7(BQmBC?hhP6f`1_H_3rwd6VG zuY>a_iw>DiVj8pRd^NoS{VLb|BLpF)=C=t@>c2j4x(;;d1~7hCCJ``}Odx^I0b_#^ z?MkC*7D1m*U?zcS*sJkJBj4-mKLajN1DL9bY7*($mRrbQ%g4R@!rK~o8DF(Y59$79 zUvt$j4g+I>Kvus9RMXs+7GMSu_QN?Ulq~mSD%vlm0qH1l|LYe58vn*eELyw`!_t&h zHQiSnfE0YT#CZt}`BpPXXb zs$rBK+Xz`P5wZe~pr+YS)+}C0`R1aMtWAZ%X)tPb$NUFuq>DZhiLS$zwob68ro27Xa_`6`gD zY7%axi3Xi|2LqYMUM8~D$35|295i1c|Lrwx@{wFzTYt0%(a_Wc?n{1-!vZU1^;pl2 zz88GvdzHxWY~O9F0KJaE-v8tRHo~mJumFD^0sLt|dcK8GPQ?JZzPM+`1i}^(7x=w+ z$nNhwlmb$belhp^cgK1dvC%w`N>8Q+gRhe8F-SB)rXfjE$NET6{UMPVVUr1o`gd^p z*M*GlSC}@xHZ7~^F$QWo3Yu%J8ccQdy(i5s^U#Qt8r@Zw6WUxImG07;^61kd+(bziI|ZPToQuTzs~<8YCwxOG&MB% zOKV607pUB-pScGP5q;;^dIN$Jl#y$_|F!vQA%xtT!TOLFpoXt)kffc^C*1r*Wv)x5 zsE3K@aNULzi9Fsa4e!hbdn@=7?ylnJ=f{c`#NTPZl#||R5d*sdQt()kGrA$EeicPo ze?PoQ>Mw5WTvY9+V->3)PAP$-mOts$)>~d0gS=BvX)6NF6k7Fo8N~j^B(WMQDnx)C zn2iFVG-2F`(6#kY|4QQmmSfH64&5z+(Zr_sV@8(gE!-^PVDF30j=SZ}ilCX4K!!nq zDp6g6&XenSxLm0DM^LrA<`R{rXUB=Y`HG*9?-i(WdrA9j12*;+0c6ubpdi|_=NkU$ zlC?Q#gV9uFbF@0!-*t_hAt?=y_y>^K92_AfU?A-;mX{7_D_QyXRkURNr#|YU_)%-) zk2-dNh&?dv59o32fX?Rkl9C(utVzv-4nM&@TJD-7pL}@>I&g)+wmoMFGl#6<6?0r~ z1sghA+{615Ncj~|&cvC}1<(SlrhtQ(Tgkt$Guv88F~8{Wk2Bb7*<0#P=mePfFvn5Pkwt253|6 z&H0c?cqmyZ?HYVG3g7r%V*9;krWpuIFP3u;`6@~H_p50^|4q3@SfBgbS6rvZ|3Ezv zq!C7@#>U3p6j-j_JoOLEQr>qm){Inpzw?H2;%fqUxmR3xyV0?e@pJ(pvn1Hx1PWZ> zU}yGa5}#|!zEF{4xrW#Y`>B(ZV$0Q}$`d<5?@Q{_E2;zfdi!%!-hKv>LVewU?w_j+ z;0i+wyL)o{;QyR6K~Hus=&fW})I`l7%sp-g(=UMnl2IPitpwBvS?^j#z}^Y^Q%%w` z`q9QtCBb#t#}k?SfXsJQJXvFv*V4ya)pH-Ct!`PBQNfvEL&}}cBvA6qnKVEE?pzsH zDW4o0A0JQo^vQ^+OR&xeu2QJ%S>P5$SI~&ahTk~UH;IHV#tGaP?VUWFFxOU63WvQb zJ(jMd-^~HdcY-l_8HRNA*6AejvP!&^&!+W%$$Cq6(>;9;wjF* zU$70*Lu1C17THJ>O+^(m#rzLBoi5U-rxu1^ShuEXc*Si06p0842~m4B)1|NZ#RF&- z`6lj?3j0|5^(`yHza6xM^MILMIU&t>g%FC4>2;uti)*XQje~S7^A|{G9Xe;jIp4&! zD67f(SB=t2xLq_d?B>^8L4jdqRzXfq4$Qun%dUB=o)vJ0CfK`LUy@_8-6$|K=*jst z0!87!q`vGAl-NJWybP?0YbjIzz$5f5i8_zvn6Hw8e;=xA{X5Jn^kiAo$`I6!8l0!9 z9noJ=akD^a{Ql21&Fv7uu8_hO%&V+?ub3RpR&)zN=G_@XD^U9OWmOtz;adbfwrIuF z<)izRO0Mn1Q0UYIJ)BWuYT&VU7O3&erAvC%`o>q$-hNyI0bpc?*7McK2Ode?khGCLTw3wf1Mm{v6d zbNT%0`I8TmZN$Qa#>+t;-^P#z;L-a`*`z#1Lk%{8T(0l4G1}V%=gICT5XQ$WM{^}} zRD7uQBb#&stv%g%mz`QtP~z?O%Zn9?>k;B>IxPL2{diT3y9P*zZc^v4ESLI&!2{x@ z)uBtgtQ7|Cj~|RL)<}H48N$?Q9dhH6)6TSTl+fd>b<=RA}B0e-IvR9 z5cJufK?V0egnkGs3mY!#(bTx=UIWD|Mt7F&@H9z3~UFTN-!M0JKIKzl{R-6|)BFBL7 zYz%-m1L5c3|IU2WFsDqP-8>3k}=d3kwwzQn#=2QB@fR_>aS+x9GiD&@dMK8cg{)a<7kA1g;FXZ567q=Mv zwA-_@v&%uLm+@dVKZhO}!>WfR6s0;I6;FtX;R1!6AlboY#6e4;?4Ur?ivxgd*^QIf zKWR<^p=9u|&!)GfD3x{9W&M-b{4yFOcTG{xP>`GG!H)qT?h9X4>C3!IONu8h{-hq! zr4~Q6&IRvykuUq1nIpgZ2jKq z0Z*EjCitoLqjmY;-@-xk-$LzCNpx7EQF~EHM8LSIGI^e&h-K$N3kTu|K?Po6I}Zz1 zu?tgxIf7@EqUW6JOIsv02EY=$8bOIkv*^;0KRPvSw35gnzQ*`ogq5 zy3e7x?>rpwFhr*WvmTi$0$JSKV#VEKZrj;$X^`$Um79y2mNYL7~kQ$Sy}@N|Pfp_FN2^Qp6N?LgYH5VK>f!#o@Jj z9yfc1t$`&zx6bXU!>8&Dx;{@_fBX1^*5cN#__o+r$P+4Q@V?nl{#W_T7~HqG-;4+{ zJaZiH%b;$PzXP#%KGwA_&CRcMNxsH+gM%i@Wp6ElY zJa}W1kLP|%yli?}P@%Rumd7KXdooyiDC!&P+rm#&e%S|Cw|3b#!w+6%& z)*PK1F`fo_%nRpA^*&&1&`}ulozwjx1!?~ogiF}>@?$-gd&V+2V?5XtIrnq+wYnp1 z6U`-5x9ASVc~cp!?{%z-hy#DWlt2O@sxF*$GALXI9zA~IA$P>t$wor6vKtH; z0QZnp$_dWJ11~@j>s#W!juNyU1X6=nd#v33!doh$r#inx#h>f$FnJEa&m-O1Djt#t zi69lO^%3~aieHz2|C;_f8N6rI(2suLV4}46$$4v8SB}L zsQ#@fGq@^#*?h=z>GDu4ok8|Ilf-M`9-QBGtn$R;Pi~8sSbSH)U}jwu+`Cg&g9(c& zv6|RycyFGy0v9nHb27fM-KS5244MsWkY(^C!}k0h#ou4~&c5r0w4YY|)6swUEFSl6 zdO_|3$NkF~O|Rc!!*IL9G;&YmDT?Fg6cg@*R&cI8Qq#R5kt4yKlN)Y@ZozL5`?v%z$OKv8s)rB8HW>3U5Q%WTa^=|7C5b~9io=R z)t4zsQ3XfPU0LxU*n=2$_azU{Er^U#?I5y&Xs-Y6hv?=q82L$}h+|kY%lX{n`qt!* z%;LoAA!-Ed_nW|{obP|4EZgH{6Z^e-8krHNi6bthk*4cdo`$5{x9%wi9WCcswq1Uhdaczz?6}|r++fMyM z`}Ol!H;u{gz4qz9qpLJ#6;hdsJ9wN6#G1eU31&WV;KQKgq}u~(Im&6Gbex;%0t0*Q zA@`W0FW@CUk37fdw{MRH^=|3^Hib88=UgBE5>ZKUz6$T#eI!>>(c?u|^zx~KdYZ0> z{2T0K2R|m5_ZKY0?{P%!VAk(2X|Eop@stLA!puW7`=W^TPxsre!0V&ouVV0qI__HnS}@{P3fY@-?yq9l2fTk2 z=M&tXZr2wdJs|qCp3+CQ9u??A7nT6;PJUxcojc_7W7dhfx?N0h(p*l%oxtmjbzr#r z_Tc$l)P-D!Yj6}hX|UwM5Y3T+S~k}G?@?4EjjGoGrZ>4>(Ojc)&N(}`B7ERlC8f`q z(RwJZl_L3|;@0NQ3?6m`uOXsRp5hhb={O;$K{a!OsPUYq)b;1x?GH7S4XPyk%#r^l12fGF*RT1J2O zzon;JJ4yKU<_XP2D-DUns7m}J`lvR-=A0K4l!ET&kGj=llw?xSxipWn-NH;BOZ3>) zeWW2yT=n1osu?I7uXS>_aZR{(2zzTXIr)-1HO#efL$Na8(_g+#rtKpL&YlYy<{Al4 zx9HZ)HV^H%0}OF5ub<-2V)qV~FFx|Z1n&szi8mw9jX07;VTm?xKlw>kf9Llk{?T1Q z#8H}~+Gw};M{*&|3gJ%CIEpynx?tvq!KGK5548z;T(VTbFCQ41r?cP`m{`_~%fj-9 zd|W$f;F@GR@>xBn>X>+AL112>Gs%=yN!}XGIx$ogDj0mWnx$ryEz+$8Ln1GD)*83& zA(!3<@7UezHNJ)sJ`*czNUDb_Fm8MwcgM z?MK&#gAxjHo6_--9A~OO2mg3A&mT;cM@zRI-MJaSAbxw2`)t_HxjD|FhT)R z(#qMl6u~6Jr1z=@+bJg7%(F8iarif^sbUJQzZkr@pLt$TH-yu=o$8htoIdzk7TNhg z9V2j7BXScfK-*0+S+2U@cD8(bUXc@{vr7y$wBwoY__y%brTamaWeB;+weRm0XV1}5wGWkL<2@I-dzU3WxHL7zuqr; zpa1Fu*M#Cu10|+OD0xcpr*G7$WzOZ1KkJ3l_rVzZ`GAFXROefGfVuji%(h9O*J-K4 zqb*j{19fxqdp(OhIMv!67%!R88M^W{OMOD$S{$9}>eXwQ&SH0J<`KlC6iNeWM z_`UDgE)?hO@!z<7XFvbS;pSwo-F)bbQ~0Iq`3YP^LH&qyTi3{|Bb~+Dly!i(o^S$v z&3F@w`1Y_wu_yVLBIojmmt5k6%|hU=+;(x;Du(;`cYZTijTbyNJnqqXY6wSo^F2E2 z)%JE&d95;3E&G!Fg7>E!;!DR;;7#9i0g4vOeS5B`-Va100{Dn{a(OrS$ikIYoeGNVN>7o zm*R^3s{yyO53bl3vJOR-3Vj>krU0+|gxqhpb;tipRa%bwAm@XiPgMoE<(HPvB@Cs& zy##x5;aT9P(Z&gWkN#5N?EtU4nedil{UiS0%CE&;5{gO*{s-}=xcg5@_Ze=qaC3sL zDAzB*Yw~?IKW^^_cul?&h%~3U%Xv$2m+^xZ`)}h00Oeckvr_0=;1-z)QlNs|r`}MO zj{6{mH|TNSJHTtwaFru0ClY*c_FjaubZ)bP7W!J?1l){t&FP4n^{3E5e%}Fp?(>@W zh<<{aHu!%lOY$JRbv5Fik8MuS30ZS6y#E9nnsfPWo?D~g5@{( zGE1LJkep$y)4i^sztl~|egeT}6&nsb1{^`~TS`eO^vB|!$8Ju0T*2*&8z{R2xt$;v z;RYDzBgk=|mVy@MaqsuJtRndXbFCeMt80IWV&l0MdT?{x?+R?y%xfxfBMyg=+x_b^ z@t@!K|A=oQmLS>lxm0S!?-#Nb{~`~;uB4>pa|v?tj$#Z(F2$+oGX4Z$&j7DK2sekp zkp%l$C8f|`9>YG)Bmc|mL5_n$21Ujx&f~rkcf2R%I7_N4aN}~G4|~J;+)J;<4L0!m zhBgE@dsG=W_g@5AemY*`<0bq-;1LQks z?j_vic|Yglo+tDZayd5F zVt)v3QXs!#Ukru*5#0Hx5J7%Fit)Iw3GkY53(SZ-;9D;b2D%fYl}R%F%QQR?270`xG(9UDD-*v>A+6{yzcwJ z=^Od~o$!B@t{(S+S&icMEA#-nM4qB>LFS{L5^!5VK9eB(v(_5_p3M^cM}p_0&_`*S zF5?F+_DACGPjBp!U=|Dg1Gv@j3b`l0p<4VG&*~0r8R9;p66BoaHy?hJAccPWfZNJ~ zCxLU=`0v#x1gn`PxZ0S@LSvg0#_Dws-~#y>%{OP3x*hk6(VRaMe1@*M-OLkn(V zg{c9y^$KfGG9v=K{&j*6N*Lp_t$f#`?;jxNDv~|eWDx%Ep_<&r8r&OTn~^*d8lngB ze-C~$G;WIDdh|gq-vw2LTeK+Xn%57o{?GbwpV!B=9{ptCEuZ!RZXv56@AHfha&IWe zc`NCQT%C*mdw48|?X|{T?eN=X8g6k0;kO=rkdAL_+;qu7j^iL}H~8|~NsvMh?wR=@ zKkUnK1K3vhyly$c=U#{C3Oxinv7A@H_M>(!{`qrVJO1@F zP=Xr}=Jy`X$#J`{2iRt$E40PPuh5STxSc_`Iv3>k^@Ku0`|TiTp$9j!kKgv{aDzPj zUYF~)@hA&@kU}|&kS?)=tTX;!syyJR9JjkU!M@ly2=^X+llKDb_kjrZ8wS5Gbsp~bI_RP-O@0$#yR-QJ?VVeU zomCabf1Q~!?VQq#v?(DPJEm%M0gMS=5@W;%qG*gEq5%^L zYD5w625Plps2Avifmp49ptMO_TOqWXX$b?Q%+So7rjrcHTKU&*szFxHbQqlGzcU6`xrcr9`sImjtg5T z9Wf^k|nT|hY&re zPW=hzz`M!6krx_@B8eV+=P55?*Xb`47vuA~zmZe_2OPRvXt7Zzc+TcuBW~N*BKjg* z8#hp%z4tfwtnvRk5Iu>WJg34Z@x2e%=!3DJM|Tq8NNBCK?<_3GtqpOUl2aZv%W6ls* zL?07~d7q&>lfwk|Rj?H1C3?^q@vOtf>%GD~$Wc-$CIKdC#>gP+WDZF}~>Bt8$eg{p{N zn9#Nd%kkB_Xdy2##rT%v-}!u+>Ua6_z#{seM(A$fcaV1yurRZf#SlICyvxr=+JN$i zehQUE^-hqX`KWoZ4Am&`PTr4`raD`OZwAqN{ToryC(FhRY2UqV4x5>rpu8>XRluVGW4kq>?W+Uq9zb(ySxAxIXA%9+BK2>< zmlE9~;F#D_6if8W=%0r5C(?9(D~sqsZ3jap%7^f^h0Kd}C?&=_p8pVk2HlBq6v*Qp z!!)YDUL>~RJCo1kiwSL8MBhwk%Zmx%kH)yV$siCtXgAAqJ+TvUC%)~p+d{Ddw^3}z z%S%yBb?=xSD_}g|4X{6r0=SFB7$U`833ORR|4;%Oo`Pg8v@>|6N1>%Cmgqr5)NPAv z2GvAQfij8SW3N+=q8!qcXM`gt7%r==?44+?NH~UeY(Oc7A@}8e(l-HFk0$ewaqJ@P zv$2T2mC#lQCJ3)}F(^bog1;1S-W0NKSb_g&D3d1$d}IS8H+b2 zCX;(9W-I?)EOtWP8L7`A`iCR+XSm<6lgTmE$G$HtB`Js=RJ5Wq=kd&wW*l*a%WH-jHjHMtI(Zk|WvZ+f%mQD1amHckQ zZZA0jdDKB2&{e*cH?PGib_!O!!Z=I)&qMG72qaU=VcuJ@2 zBR=z>8$3i`WigeIZTvwZdhnHFZ6QKQi4DlR$}=Av0KN>o1K$E<3N6}qpoMa%y;n%T zqtFdkk#8UqUz&L6-yB0;{zm!NXi0D7j=l_RU+9o#z8Ys&0+DJVwpdb4TXI87(i33N}^q8~v_y53&2D4CAX(F?btZPb1kqx_(} zc1$3T^1a6An#7;sz7GY5^}}Z5!Ng)YnzO9D`FTWkPbaXMVIw*QikWK`P~`dx=)Io- z&Oz_}B)YEiBJ@wAql;AYx`k_ymm5oJPUWFQ5MVYUBh>p`t+EV7rG=tW_Qy zrT>n@4$`(Nz7e%_x6nd;E4uDjSgVAT9zAF~=kmllp&E}q!x(xHc~$iU0(BFt z`uf}7>AdNLmPfxMr`~>e(6E`wi}Y{`tuLk~`l3)J(P!9>zl>oSvY)io!17=n@H?u% z7nwAH+bK5c83u>wL7vth=szjOqHLlExYJ>?MgN_^yckC*r+JQ(ijrRuSY#zJ-VFS? z%ebnNALP7_Du}*;uiSVGH7;7xq|Ex#km$p(h42X6)Utq5eyXZM*NIU?e4ZewBIO9_ z^5iXei2j_C#^OOgvL95UZ=h#x*?pPfslqG%TqqMXnBf!i*1YO z^K@VJ;BENN>z9<1x4tkW`f!Y*A696@X$sj(d=zg{*hp9rk5n>;0-eh2n&lz-JgtF? z#7f}zEFI^d5v{jVy#hn9wi6Skl%E^wwWxx;7)?&*0au8J?tJ0J4 z5PeSD%(q6pZp3@?H)j@BI+v^mqS%!wNKEUPeehk5w|#A%+-Y lhyleo#ReA*F~krB Date: Tue, 31 Aug 2021 10:25:11 +0200 Subject: [PATCH 86/88] fix attribute name from `host` to `hosts` --- .../default_modules/ftrack/plugins/publish/collect_username.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py index 7a303a1608..39b7433e11 100644 --- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py @@ -26,7 +26,7 @@ class CollectUsername(pyblish.api.ContextPlugin): """ order = pyblish.api.CollectorOrder - 0.488 label = "Collect ftrack username" - host = ["webpublisher"] + hosts = ["webpublisher"] _context = None From da0ea31ec9446509cf37e7036d81689adbb71a07 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 31 Aug 2021 10:31:36 +0200 Subject: [PATCH 87/88] Webpublisher - fixed documentation, host and ip are provided by command line --- website/docs/admin_webserver_for_webpublisher.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/website/docs/admin_webserver_for_webpublisher.md b/website/docs/admin_webserver_for_webpublisher.md index dced825bdc..6e72ccaf32 100644 --- a/website/docs/admin_webserver_for_webpublisher.md +++ b/website/docs/admin_webserver_for_webpublisher.md @@ -40,14 +40,13 @@ Deploy OP build distribution (Openpype Igniter) on an OS of your choice. ```sh #!/usr/bin/env bash export OPENPYPE_DEBUG=3 -export WEBSERVER_HOST_IP=localhost export FTRACK_BOT_API_USER=YOUR_API_USER export FTRACK_BOT_API_KEY=YOUR_API_KEY export PYTHONDONTWRITEBYTECODE=1 export OPENPYPE_MONGO=YOUR_MONGODB_CONNECTION pushd /opt/openpype -./openpype_console webpublisherwebserver --upload_dir YOUR_SHARED_FOLDER_ON_HOST --executable /opt/openpype/openpype_console > /tmp/openpype.log 2>&1 +./openpype_console webpublisherwebserver --upload_dir YOUR_SHARED_FOLDER_ON_HOST --executable /opt/openpype/openpype_console --host YOUR_HOST_IP --port YOUR_HOST_PORT > /tmp/openpype.log 2>&1 ``` 1. create service file `sudo vi /etc/systemd/system/openpye-webserver.service` From 3a8b1f403ae2155ffe705886dda8b222983ff80c Mon Sep 17 00:00:00 2001 From: OpenPype Date: Tue, 31 Aug 2021 09:35:27 +0000 Subject: [PATCH 88/88] [Automated] Bump version --- CHANGELOG.md | 9 +++++---- openpype/version.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4259a0f725..e1737458b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Changelog -## [3.4.0-nightly.3](https://github.com/pypeclub/OpenPype/tree/HEAD) +## [3.4.0-nightly.4](https://github.com/pypeclub/OpenPype/tree/HEAD) [Full Changelog](https://github.com/pypeclub/OpenPype/compare/3.3.1...HEAD) **Merged pull requests:** +- Ftrack: Fix hosts attribute in collect ftrack username [\#1972](https://github.com/pypeclub/OpenPype/pull/1972) - Removed deprecated submodules [\#1967](https://github.com/pypeclub/OpenPype/pull/1967) - Launcher: Fix crashes on action click [\#1964](https://github.com/pypeclub/OpenPype/pull/1964) - Settings: Minor fixes in UI and missing default values [\#1963](https://github.com/pypeclub/OpenPype/pull/1963) @@ -18,7 +19,9 @@ - Add face sets to exported alembics [\#1942](https://github.com/pypeclub/OpenPype/pull/1942) - Bump path-parse from 1.0.6 to 1.0.7 in /website [\#1933](https://github.com/pypeclub/OpenPype/pull/1933) - \#1894 - adds host to template\_name\_profiles for filtering [\#1915](https://github.com/pypeclub/OpenPype/pull/1915) +- Environments: Tool environments in alphabetical order [\#1910](https://github.com/pypeclub/OpenPype/pull/1910) - Disregard publishing time. [\#1888](https://github.com/pypeclub/OpenPype/pull/1888) +- Feature/webpublisher backend [\#1876](https://github.com/pypeclub/OpenPype/pull/1876) - Dynamic modules [\#1872](https://github.com/pypeclub/OpenPype/pull/1872) - Houdini: add Camera, Point Cache, Composite, Redshift ROP and VDB Cache support [\#1821](https://github.com/pypeclub/OpenPype/pull/1821) @@ -56,7 +59,6 @@ - Nuke: update video file crassing [\#1916](https://github.com/pypeclub/OpenPype/pull/1916) - Fix - texture validators for workfiles triggers only for textures workfiles [\#1914](https://github.com/pypeclub/OpenPype/pull/1914) - submodules: avalon-core update [\#1911](https://github.com/pypeclub/OpenPype/pull/1911) -- Environments: Tool environments in alphabetical order [\#1910](https://github.com/pypeclub/OpenPype/pull/1910) - Settings UI: List order works as expected [\#1906](https://github.com/pypeclub/OpenPype/pull/1906) - Add support for multiple Deadline β˜ οΈβž– servers [\#1905](https://github.com/pypeclub/OpenPype/pull/1905) - Hiero: loaded clip was not set colorspace from version data [\#1904](https://github.com/pypeclub/OpenPype/pull/1904) @@ -75,7 +77,6 @@ - TVPaint: Increment workfile [\#1885](https://github.com/pypeclub/OpenPype/pull/1885) - Allow Multiple Notes to run on tasks. [\#1882](https://github.com/pypeclub/OpenPype/pull/1882) - Normalize path returned from Workfiles. [\#1880](https://github.com/pypeclub/OpenPype/pull/1880) -- Feature/webpublisher backend [\#1876](https://github.com/pypeclub/OpenPype/pull/1876) - Prepare for pyside2 [\#1869](https://github.com/pypeclub/OpenPype/pull/1869) - Filter hosts in settings host-enum [\#1868](https://github.com/pypeclub/OpenPype/pull/1868) - Local actions with process identifier [\#1867](https://github.com/pypeclub/OpenPype/pull/1867) @@ -83,8 +84,8 @@ - Maya: add support for `RedshiftNormalMap` node, fix `tx` linear space πŸš€ [\#1863](https://github.com/pypeclub/OpenPype/pull/1863) - Workfiles tool event arguments fix [\#1862](https://github.com/pypeclub/OpenPype/pull/1862) - Maya: support for configurable `dirmap` πŸ—ΊοΈ [\#1859](https://github.com/pypeclub/OpenPype/pull/1859) +- Maya: don't add reference members as connections to the container set πŸ“¦ [\#1855](https://github.com/pypeclub/OpenPype/pull/1855) - Settings list can use template or schema as object type [\#1815](https://github.com/pypeclub/OpenPype/pull/1815) -- Maya: expected files -\> render products βš™οΈ overhaul [\#1812](https://github.com/pypeclub/OpenPype/pull/1812) ## [3.2.0](https://github.com/pypeclub/OpenPype/tree/3.2.0) (2021-07-13) diff --git a/openpype/version.py b/openpype/version.py index 2e769a1b62..17bd0ff892 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.4.0-nightly.3" +__version__ = "3.4.0-nightly.4"