From f92ee311b8fdb05ffae23194a64fa9e93edd4bfc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:23:10 +0200 Subject: [PATCH 01/24] PYPE-1901 - Fix ConsoleTrayApp sending lines --- openpype/tools/tray_app/app.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/openpype/tools/tray_app/app.py b/openpype/tools/tray_app/app.py index 03f8321464..476f061e26 100644 --- a/openpype/tools/tray_app/app.py +++ b/openpype/tools/tray_app/app.py @@ -142,18 +142,23 @@ class ConsoleTrayApp: self.tray_reconnect = False ConsoleTrayApp.webserver_client.close() - def _send_text(self, new_text): + def _send_text_queue(self): + """Sends lines and purges queue""" + lines = tuple(self.new_text) + self.new_text.clear() + + if lines: + self._send_lines(lines) + + def _send_lines(self, lines): """ Send console content. """ if not ConsoleTrayApp.webserver_client: return - if isinstance(new_text, str): - new_text = collections.deque(new_text.split("\n")) - payload = { "host": self.host_id, "action": host_console_listener.MsgAction.ADD, - "text": "\n".join(new_text) + "text": "\n".join(lines) } self._send(payload) @@ -174,14 +179,7 @@ class ConsoleTrayApp: if self.tray_reconnect: self._connect() # reconnect - if ConsoleTrayApp.webserver_client and self.new_text: - self._send_text(self.new_text) - self.new_text = collections.deque() - - if self.new_text: # no webserver_client, text keeps stashing - start = max(len(self.new_text) - self.MAX_LINES, 0) - self.new_text = itertools.islice(self.new_text, - start, self.MAX_LINES) + self._send_text_queue() if not self.initialized: if self.initializing: @@ -191,7 +189,7 @@ class ConsoleTrayApp: elif not host_connected: text = "{} process is not alive. Exiting".format(self.host) print(text) - self._send_text([text]) + self._send_lines([text]) ConsoleTrayApp.websocket_server.stop() sys.exit(1) elif host_connected: From b90bcd9877e56cfd91040d110a1cc13fc7aea2c5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:25:42 +0200 Subject: [PATCH 02/24] PYPE-1901 - New pluging for remote publishing (webpublish) Extracted _json_load method --- .../publish/collect_remote_instances.py | 88 +++++++++++++++++++ .../publish/collect_published_files.py | 23 +---- openpype/lib/plugin_tools.py | 17 ++++ 3 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py new file mode 100644 index 0000000000..a9d6d6fec6 --- /dev/null +++ b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py @@ -0,0 +1,88 @@ +import pyblish.api +import os + +from avalon import photoshop +from openpype.lib import prepare_template_data + + +class CollectRemoteInstances(pyblish.api.ContextPlugin): + """Gather instances configured color code of a layer. + + Used in remote publishing when artists marks publishable layers by color- + coding. + + Identifier: + id (str): "pyblish.avalon.instance" + """ + order = pyblish.api.CollectorOrder + 0.100 + + label = "Instances" + order = pyblish.api.CollectorOrder + hosts = ["photoshop"] + + # configurable by Settings + families = ["background"] + color_code = ["red"] + subset_template_name = "" + + def process(self, context): + self.log.info("CollectRemoteInstances") + if not os.environ.get("IS_HEADLESS"): + self.log.debug("Not headless publishing, skipping.") + return + + # parse variant if used in webpublishing, comes from webpublisher batch + batch_dir = os.environ.get("OPENPYPE_PUBLISH_DATA") + variant = "Main" + if batch_dir and os.path.exists(batch_dir): + # TODO check if batch manifest is same as tasks manifests + task_data = self.parse_json(os.path.join(batch_dir, + "manifest.json")) + variant = task_data["variant"] + + stub = photoshop.stub() + layers = stub.get_layers() + + instance_names = [] + for layer in layers: + self.log.info("!!!Layer:: {}".format(layer)) + if layer.color_code not in self.color_code: + self.log.debug("Not marked, skip") + continue + + if layer.parents: + self.log.debug("Not a top layer, skip") + continue + + instance = context.create_instance(layer.name) + instance.append(layer) + instance.data["family"] = self.families[0] + instance.data["publish"] = layer.visible + + # populate data from context, coming from outside?? TODO + # TEMP + self.log.info("asset {}".format(context.data["assetEntity"])) + self.log.info("taskType {}".format(context.data["taskType"])) + instance.data["asset"] = context.data["assetEntity"]["name"] + instance.data["task"] = context.data["taskType"] + + fill_pairs = { + "variant": variant, + "family": instance.data["family"], + "task": instance.data["task"], + "layer": layer.name + } + subset = self.subset_template.format( + **prepare_template_data(fill_pairs)) + instance.data["subset"] = subset + + instance_names.append(layer.name) + + # Produce diagnostic message for any graphical + # user interface interested in visualising it. + self.log.info("Found: \"%s\" " % instance.data["name"]) + self.log.info("instance: {} ".format(instance.data)) + + if len(instance_names) != len(set(instance_names)): + self.log.warning("Duplicate instances found. " + + "Remove unwanted via SubsetManager") diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 7e9b98956a..2b4a1273b8 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -15,6 +15,7 @@ import tempfile import pyblish.api from avalon import io from openpype.lib import prepare_template_data +from openpype.lib.plugin_tools import parse_json class CollectPublishedFiles(pyblish.api.ContextPlugin): @@ -33,22 +34,6 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): # from Settings task_type_to_family = {} - def _load_json(self, path): - path = path.strip('\"') - assert os.path.isfile(path), ( - "Path to json file doesn't exist. \"{}\"".format(path) - ) - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - self.log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data - def _process_batch(self, dir_url): task_subfolders = [ os.path.join(dir_url, o) @@ -56,8 +41,8 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): if os.path.isdir(os.path.join(dir_url, o))] self.log.info("task_sub:: {}".format(task_subfolders)) for task_dir in task_subfolders: - task_data = self._load_json(os.path.join(task_dir, - "manifest.json")) + task_data = parse_json(os.path.join(task_dir, + "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] task_type = "default_task_type" @@ -261,7 +246,7 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): assert batch_dir, ( "Missing `OPENPYPE_PUBLISH_DATA`") - assert batch_dir, \ + assert os.path.exists(batch_dir), \ "Folder {} doesn't exist".format(batch_dir) project_name = os.environ.get("AVALON_PROJECT") diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 9dccadc44e..2158a3e28d 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -483,3 +483,20 @@ def should_decompress(file_url): "compression: \"dwab\"" in output return False + + +def parse_json(path): + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + log.error( + "Error loading json: " + "{} - Exception: {}".format(path, exc) + ) + return data From 93766c1d0b9515764b1e57f80406948d843f62eb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:27:15 +0200 Subject: [PATCH 03/24] PYPE-1901 - Added plugin to close PS after remote publishing --- .../photoshop/plugins/publish/closePS.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 openpype/hosts/photoshop/plugins/publish/closePS.py diff --git a/openpype/hosts/photoshop/plugins/publish/closePS.py b/openpype/hosts/photoshop/plugins/publish/closePS.py new file mode 100644 index 0000000000..ce229c86bb --- /dev/null +++ b/openpype/hosts/photoshop/plugins/publish/closePS.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +"""Close PS after publish. For Webpublishing only.""" +import os + +import pyblish.api + +from avalon import photoshop + +class ClosePS(pyblish.api.InstancePlugin): + """Close PS after publish. For Webpublishing only. + """ + + order = pyblish.api.IntegratorOrder + 14 + label = "Close PS" + optional = True + active = True + + hosts = ["photoshop"] + + def process(self, instance): + self.log.info("ClosePS") + if not os.environ.get("IS_HEADLESS"): + return + + stub = photoshop.stub() + self.log.info("Shutting down PS") + stub.save() + stub.close() From 185d3ef399ebe25211b038dd0a8cfca7055ac81e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:35:11 +0200 Subject: [PATCH 04/24] PYPE-1901 - Fix wrong variable used --- .../hosts/photoshop/plugins/publish/collect_remote_instances.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py index a9d6d6fec6..fa4364b700 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py @@ -72,7 +72,7 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): "task": instance.data["task"], "layer": layer.name } - subset = self.subset_template.format( + subset = self.subset_template_name.format( **prepare_template_data(fill_pairs)) instance.data["subset"] = subset From d2c4678fd45b8fb5d38d3a3e0382ff99ab9a0d06 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:35:38 +0200 Subject: [PATCH 05/24] PYPE-1901 - Added background family to ExtractImage --- openpype/hosts/photoshop/plugins/publish/extract_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/photoshop/plugins/publish/extract_image.py b/openpype/hosts/photoshop/plugins/publish/extract_image.py index 87574d1269..ae9892e290 100644 --- a/openpype/hosts/photoshop/plugins/publish/extract_image.py +++ b/openpype/hosts/photoshop/plugins/publish/extract_image.py @@ -12,7 +12,7 @@ class ExtractImage(openpype.api.Extractor): label = "Extract Image" hosts = ["photoshop"] - families = ["image"] + families = ["image", "background"] formats = ["png", "jpg"] def process(self, instance): From 57de11638c91c6518dd599db39c26347f00ad459 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 17:36:13 +0200 Subject: [PATCH 06/24] PYPE-1901 - Added settings for CollectRemoteInstances --- .../defaults/project_settings/photoshop.json | 5 ++++ .../schema_project_photoshop.json | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index 4c36e4bd49..14c294c0c5 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -12,6 +12,11 @@ "optional": true, "active": true }, + "CollectRemoteInstances": { + "color_code": [], + "families": [], + "subset_template_name": "" + }, "ExtractImage": { "formats": [ "png", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index 3b65f08ac4..008f1a265d 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -43,6 +43,35 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "CollectRemoteInstances", + "label": "Collect Instances for Webpublish", + "children": [ + { + "type": "label", + "label": "Set color for publishable layers, set publishable families." + }, + { + "type": "list", + "key": "color_code", + "label": "Color codes for layers", + "object_type": "text" + }, + { + "key": "families", + "label": "Families", + "type": "list", + "object_type": "text" + }, + { + "type": "text", + "key": "subset_template_name", + "label": "Subset template name" + } + ] + }, { "type": "dict", "collapsible": true, From d6b85f1c1a91901581a410aba9688f2e92410aea Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 18:46:06 +0200 Subject: [PATCH 07/24] PYPE-1901 - Added testing class for PS publishing --- .../photoshop/test_publish_in_photoshop.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 tests/integration/hosts/photoshop/test_publish_in_photoshop.py diff --git a/tests/integration/hosts/photoshop/test_publish_in_photoshop.py b/tests/integration/hosts/photoshop/test_publish_in_photoshop.py new file mode 100644 index 0000000000..396468a966 --- /dev/null +++ b/tests/integration/hosts/photoshop/test_publish_in_photoshop.py @@ -0,0 +1,94 @@ +import pytest +import os +import shutil + +from tests.lib.testing_classes import PublishTest + + +class TestPublishInPhotoshop(PublishTest): + """Basic test case for publishing in Photoshop + + Uses generic TestCase to prepare fixtures for test data, testing DBs, + env vars. + + Opens Maya, run publish on prepared workile. + + Then checks content of DB (if subset, version, representations were + created. + Checks tmp folder if all expected files were published. + + """ + PERSIST = True + + TEST_FILES = [ + ("1Bciy2pCwMKl1UIpxuPnlX_LHMo_Xkq0K", "test_photoshop_publish.zip", "") + ] + + APP = "photoshop" + APP_VARIANT = "2020" + + APP_NAME = "{}/{}".format(APP, APP_VARIANT) + + TIMEOUT = 120 # publish timeout + + @pytest.fixture(scope="module") + def last_workfile_path(self, download_test_data): + """Get last_workfile_path from source data. + + Maya expects workfile in proper folder, so copy is done first. + """ + src_path = os.path.join(download_test_data, + "input", + "workfile", + "test_project_test_asset_TestTask_v001.psd") + dest_folder = os.path.join(download_test_data, + self.PROJECT, + self.ASSET, + "work", + self.TASK) + os.makedirs(dest_folder) + dest_path = os.path.join(dest_folder, + "test_project_test_asset_TestTask_v001.psd") + shutil.copy(src_path, dest_path) + + yield dest_path + + @pytest.fixture(scope="module") + def startup_scripts(self, monkeypatch_session, download_test_data): + """Points Maya to userSetup file from input data""" + os.environ["IS_HEADLESS"] = "true" + + def test_db_asserts(self, dbcon, publish_finished): + """Host and input data dependent expected results in DB.""" + print("test_db_asserts") + assert 5 == dbcon.count_documents({"type": "version"}), \ + "Not expected no of versions" + + assert 0 == dbcon.count_documents({"type": "version", + "name": {"$ne": 1}}), \ + "Only versions with 1 expected" + + assert 1 == dbcon.count_documents({"type": "subset", + "name": "modelMain"}), \ + "modelMain subset must be present" + + assert 1 == dbcon.count_documents({"type": "subset", + "name": "workfileTest_task"}), \ + "workfileTest_task subset must be present" + + assert 11 == dbcon.count_documents({"type": "representation"}), \ + "Not expected no of representations" + + assert 2 == dbcon.count_documents({"type": "representation", + "context.subset": "modelMain", + "context.ext": "abc"}), \ + "Not expected no of representations with ext 'abc'" + + assert 2 == dbcon.count_documents({"type": "representation", + "context.subset": "modelMain", + "context.ext": "ma"}), \ + "Not expected no of representations with ext 'abc'" + + +if __name__ == "__main__": + test_case = TestPublishInPhotoshop() From cff9c793464abb3211ef49780f7a0c94656c3831 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Oct 2021 18:46:29 +0200 Subject: [PATCH 08/24] PYPE-1901 - Added WIP for PS webpublishing --- openpype/cli.py | 19 +++++++++++++++++++ openpype/pype_commands.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index c69407e295..8438703bd3 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -158,6 +158,25 @@ def publish(debug, paths, targets): PypeCommands.publish(list(paths), targets) +@main.command() +@click.argument("path") +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("-h", "--host", help="Host") +@click.option("-u", "--user", help="User email address") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets", default=None, + multiple=True) +def remotepublishfromapp(debug, project, path, host, targets=None, user=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + if debug: + os.environ['OPENPYPE_DEBUG'] = '3' + PypeCommands.remotepublishfromapp(project, path, host, user, + targets=targets) + @main.command() @click.argument("path") @click.option("-d", "--debug", is_flag=True, help="Print debug messages") diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5288749e8b..318f2476ed 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -4,6 +4,7 @@ import os import sys import json from datetime import datetime +import time from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context @@ -110,6 +111,42 @@ class PypeCommands: log.info("Publish finished.") uninstall() + @staticmethod + def remotepublishfromapp(project, batch_path, host, user, targets=None): + from openpype import install, uninstall + from openpype.api import Logger + + log = Logger.get_logger() + + log.info("remotepublishphotoshop command") + + install() + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + os.environ["AVALON_PROJECT"] = project + os.environ["AVALON_APP"] = host + + os.environ["OPENPYPE_EXECUTABLE"] = sys.executable + os.environ["IS_HEADLESS"] = "true" + from openpype.lib import ApplicationManager + + application_manager = ApplicationManager() + data = { + "last_workfile_path": "c:/projects/test_project_test_asset_TestTask_v001.psd", + "start_last_workfile": True, + "project_name": project, + "asset_name": "test_asset", + "task_name": "test_task" + } + + launched_app = application_manager.launch( + os.environ["AVALON_APP"] + "/2020", **data) + + while launched_app.poll() is None: + time.sleep(0.5) + + print(launched_app) + @staticmethod def remotepublish(project, batch_path, host, user, targets=None): """Start headless publishing. From 280c7d96ea19ae3cc7076b0bde694eb6eff39509 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 12 Oct 2021 11:53:46 +0200 Subject: [PATCH 09/24] PYPE-1901 - wip of remotepublishfromapp --- openpype/pype_commands.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 318f2476ed..4f3e173f3e 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -125,6 +125,17 @@ class PypeCommands: os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project os.environ["AVALON_APP"] = host + os.environ["AVALON_APP_NAME"] = os.environ["AVALON_APP"] + "/2020" + os.environ["AVALON_ASSET"] = "test_asset" + os.environ["AVALON_TASK"] = "test_task" + + env = get_app_environments_for_context( + os.environ["AVALON_PROJECT"], + os.environ["AVALON_ASSET"], + os.environ["AVALON_TASK"], + os.environ["AVALON_APP_NAME"] + ) + os.environ.update(env) os.environ["OPENPYPE_EXECUTABLE"] = sys.executable os.environ["IS_HEADLESS"] = "true" From 9607026daec25e2e0803dd19093b528b03cea9fd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 12 Oct 2021 11:54:05 +0200 Subject: [PATCH 10/24] PYPE-1901 - terminate process after timeout --- tests/lib/testing_classes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index 1832efb7ed..59d4abb3aa 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -228,6 +228,7 @@ class PublishTest(ModuleUnitTest): while launched_app.poll() is None: time.sleep(0.5) if time.time() - time_start > self.TIMEOUT: + launched_app.terminate() raise ValueError("Timeout reached") # some clean exit test possible? From eb4cd6d7c622175e1204a0a0c9da027b05e20b07 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 12 Oct 2021 18:27:32 +0200 Subject: [PATCH 11/24] PYPE-1901 - extracted method for task parsing --- .../publish/collect_remote_instances.py | 8 ++++-- .../publish/collect_published_files.py | 17 ++++------- openpype/lib/plugin_tools.py | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py index fa4364b700..62d94483e5 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py @@ -3,6 +3,7 @@ import os from avalon import photoshop from openpype.lib import prepare_template_data +from openpype.lib.plugin_tools import parse_json class CollectRemoteInstances(pyblish.api.ContextPlugin): @@ -36,8 +37,11 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): variant = "Main" if batch_dir and os.path.exists(batch_dir): # TODO check if batch manifest is same as tasks manifests - task_data = self.parse_json(os.path.join(batch_dir, - "manifest.json")) + task_data = parse_json(os.path.join(batch_dir, + "manifest.json")) + if not task_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(batch_dir)) variant = task_data["variant"] stub = photoshop.stub() diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 2b4a1273b8..ecd65ebae4 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -15,7 +15,7 @@ import tempfile import pyblish.api from avalon import io from openpype.lib import prepare_template_data -from openpype.lib.plugin_tools import parse_json +from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info class CollectPublishedFiles(pyblish.api.ContextPlugin): @@ -45,18 +45,11 @@ class CollectPublishedFiles(pyblish.api.ContextPlugin): "manifest.json")) self.log.info("task_data:: {}".format(task_data)) ctx = task_data["context"] - task_type = "default_task_type" - task_name = None - if ctx["type"] == "task": - items = ctx["path"].split('/') - asset = items[-2] - os.environ["AVALON_TASK"] = ctx["name"] - task_name = ctx["name"] - task_type = ctx["attributes"]["type"] - else: - asset = ctx["name"] - os.environ["AVALON_TASK"] = "" + asset, task_name, task_type = get_batch_asset_task_info(ctx) + + if task_name: + os.environ["AVALON_TASK"] = task_name is_sequence = len(task_data["files"]) > 1 diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 2158a3e28d..62a9d7c51e 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -486,6 +486,13 @@ def should_decompress(file_url): def parse_json(path): + """Parses json file at 'path' location + + Returns: + (dict) or None if unparsable + Raises: + AsssertionError if 'path' doesn't exist + """ path = path.strip('\"') assert os.path.isfile(path), ( "Path to json file doesn't exist. \"{}\"".format(path) @@ -500,3 +507,24 @@ def parse_json(path): "{} - Exception: {}".format(path, exc) ) return data + + +def get_batch_asset_task_info(ctx): + """Parses context data from webpublisher's batch metadata + + Returns: + (tuple): asset, task_name (Optional), task_type + """ + task_type = "default_task_type" + task_name = None + asset = None + + if ctx["type"] == "task": + items = ctx["path"].split('/') + asset = items[-2] + task_name = ctx["name"] + task_type = ctx["attributes"]["type"] + else: + asset = ctx["name"] + + return asset, task_name, task_type From 5d1a83a28ab886b624866920aa17158b3a56dc95 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 12 Oct 2021 18:42:57 +0200 Subject: [PATCH 12/24] PYPE-1901 - add resolving of ftrack username to remote publish of Photoshop too --- .../ftrack/plugins/publish/collect_username.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py index 39b7433e11..438ef2f31b 100644 --- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py @@ -26,14 +26,20 @@ class CollectUsername(pyblish.api.ContextPlugin): """ order = pyblish.api.CollectorOrder - 0.488 label = "Collect ftrack username" - hosts = ["webpublisher"] + hosts = ["webpublisher", "photoshop"] _context = None def process(self, context): + self.log.info("CollectUsername") + # photoshop could be triggered remotely in webpublisher fashion + if os.environ["AVALON_APP"] == "photoshop": + if not os.environ.get("IS_HEADLESS"): + self.log.debug("Regular process, skipping") + os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] - self.log.info("CollectUsername") + for instance in context: email = instance.data["user_email"] self.log.info("email:: {}".format(email)) From 7d3c1863c278bbbae27aac9c5a34f0eb0736bfca Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 12 Oct 2021 19:25:55 +0200 Subject: [PATCH 13/24] PYPE-1901 - working remotepublishfromapp command --- openpype/pype_commands.py | 83 ++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 24 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 4f3e173f3e..e06ab5b493 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -8,6 +8,7 @@ import time from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context +from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info class PypeCommands: @@ -112,7 +113,17 @@ class PypeCommands: uninstall() @staticmethod - def remotepublishfromapp(project, batch_path, host, user, targets=None): + def remotepublishfromapp(project, batch_dir, host, user, targets=None): + """Opens installed variant of 'host' and run remote publish there. + + Currently implemented and tested for Photoshop where customer + wants to process uploaded .psd file and publish collected layers + from there. + + Requires installed host application on the machine. + + Runs publish process as user would, in automatic fashion. + """ from openpype import install, uninstall from openpype.api import Logger @@ -122,36 +133,60 @@ class PypeCommands: install() - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host - os.environ["AVALON_APP_NAME"] = os.environ["AVALON_APP"] + "/2020" - os.environ["AVALON_ASSET"] = "test_asset" - os.environ["AVALON_TASK"] = "test_task" + from openpype.lib import ApplicationManager + application_manager = ApplicationManager() + app_group = application_manager.app_groups.get(host) + if not app_group or not app_group.enabled: + raise ValueError("No application {} configured".format(host)) + + found_variant_key = None + # finds most up-to-date variant if any installed + for variant_key, variant in app_group.variants.items(): + for executable in variant.executables: + if executable.exists(): + found_variant_key = variant_key + + if not found_variant_key: + raise ValueError("No executable for {} found".format(host)) + + app_name = "{}/{}".format(host, found_variant_key) + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_dir + + batch_data = None + if batch_dir and os.path.exists(batch_dir): + # TODO check if batch manifest is same as tasks manifests + batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) + + if not batch_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(batch_dir)) + + asset, task_name, _task_type = get_batch_asset_task_info( + batch_data["context"]) + + workfile_path = os.path.join(batch_dir, + batch_data["task"], + batch_data["files"][0]) + print("workfile_path {}".format(workfile_path)) + data = { + "last_workfile_path": workfile_path, + "start_last_workfile": True + } + + # must have for proper launch of app env = get_app_environments_for_context( - os.environ["AVALON_PROJECT"], - os.environ["AVALON_ASSET"], - os.environ["AVALON_TASK"], - os.environ["AVALON_APP_NAME"] + project, + asset, + task_name, + app_name ) os.environ.update(env) - os.environ["OPENPYPE_EXECUTABLE"] = sys.executable os.environ["IS_HEADLESS"] = "true" - from openpype.lib import ApplicationManager - application_manager = ApplicationManager() - data = { - "last_workfile_path": "c:/projects/test_project_test_asset_TestTask_v001.psd", - "start_last_workfile": True, - "project_name": project, - "asset_name": "test_asset", - "task_name": "test_task" - } - - launched_app = application_manager.launch( - os.environ["AVALON_APP"] + "/2020", **data) + launched_app = application_manager.launch(app_name, **data) while launched_app.poll() is None: time.sleep(0.5) From d085e4ff7df89cfe4e770fd9ae0894e8506265c0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 13 Oct 2021 13:03:47 +0200 Subject: [PATCH 14/24] PYPE-1901 - switch to context plugin to limit double closing --- openpype/hosts/photoshop/plugins/publish/closePS.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/closePS.py b/openpype/hosts/photoshop/plugins/publish/closePS.py index ce229c86bb..fa9d27688b 100644 --- a/openpype/hosts/photoshop/plugins/publish/closePS.py +++ b/openpype/hosts/photoshop/plugins/publish/closePS.py @@ -6,7 +6,7 @@ import pyblish.api from avalon import photoshop -class ClosePS(pyblish.api.InstancePlugin): +class ClosePS(pyblish.api.ContextPlugin): """Close PS after publish. For Webpublishing only. """ @@ -17,7 +17,7 @@ class ClosePS(pyblish.api.InstancePlugin): hosts = ["photoshop"] - def process(self, instance): + def process(self, context): self.log.info("ClosePS") if not os.environ.get("IS_HEADLESS"): return @@ -26,3 +26,4 @@ class ClosePS(pyblish.api.InstancePlugin): self.log.info("Shutting down PS") stub.save() stub.close() + self.log.info("PS closed") From 15b67e239ba7a6619b13083ff461bf376aa34485 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 13 Oct 2021 14:44:51 +0200 Subject: [PATCH 15/24] PYPE-1901 - fix missing return for standard publishing --- .../default_modules/ftrack/plugins/publish/collect_username.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py index 438ef2f31b..844a397066 100644 --- a/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py +++ b/openpype/modules/default_modules/ftrack/plugins/publish/collect_username.py @@ -36,6 +36,7 @@ class CollectUsername(pyblish.api.ContextPlugin): if os.environ["AVALON_APP"] == "photoshop": if not os.environ.get("IS_HEADLESS"): self.log.debug("Regular process, skipping") + return os.environ["FTRACK_API_USER"] = os.environ["FTRACK_BOT_API_USER"] os.environ["FTRACK_API_KEY"] = os.environ["FTRACK_BOT_API_KEY"] From a08b3da447f24be48c3448070ef5f493e8e51928 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 13 Oct 2021 16:52:33 +0200 Subject: [PATCH 16/24] PYPE-1901 - fix order of closing and processing from queue --- openpype/tools/tray_app/app.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/openpype/tools/tray_app/app.py b/openpype/tools/tray_app/app.py index 476f061e26..f1363d0cab 100644 --- a/openpype/tools/tray_app/app.py +++ b/openpype/tools/tray_app/app.py @@ -203,14 +203,15 @@ class ConsoleTrayApp: self.initializing = True self.launch_method(*self.subprocess_args) - elif ConsoleTrayApp.process.poll() is not None: - self.exit() - elif ConsoleTrayApp.callback_queue: + elif ConsoleTrayApp.callback_queue and \ + not ConsoleTrayApp.callback_queue.empty(): try: callback = ConsoleTrayApp.callback_queue.get(block=False) callback() except queue.Empty: pass + elif ConsoleTrayApp.process.poll() is not None: + self.exit() @classmethod def execute_in_main_thread(cls, func_to_call_from_main_thread): @@ -230,8 +231,9 @@ class ConsoleTrayApp: self._close() if ConsoleTrayApp.websocket_server: ConsoleTrayApp.websocket_server.stop() - ConsoleTrayApp.process.kill() - ConsoleTrayApp.process.wait() + if ConsoleTrayApp.process: + ConsoleTrayApp.process.kill() + ConsoleTrayApp.process.wait() if self.timer: self.timer.stop() QtCore.QCoreApplication.exit() From 2150ac836edccf74420ee12d71c39bf496c9a26a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 13 Oct 2021 16:55:42 +0200 Subject: [PATCH 17/24] PYPE-1901 - extracted methods into remote_publish library Methods are used in both remote* approaches for logging and reusability. --- openpype/lib/remote_publish.py | 92 ++++++++++++++++++++++++++++++++++ openpype/pype_commands.py | 86 +++++++++---------------------- 2 files changed, 116 insertions(+), 62 deletions(-) create mode 100644 openpype/lib/remote_publish.py diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py new file mode 100644 index 0000000000..aa8d8821a8 --- /dev/null +++ b/openpype/lib/remote_publish.py @@ -0,0 +1,92 @@ +import os +from datetime import datetime +import sys +from bson.objectid import ObjectId + +import pyblish.util + +from openpype import uninstall +from openpype.lib.mongo import OpenPypeMongoConnection + + +def get_webpublish_conn(): + """Get connection to OP 'webpublishes' collection.""" + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + return mongo_client[database_name]["webpublishes"] + + +def start_webpublish_log(dbcon, batch_id, user): + """Start new log record for 'batch_id' + + Args: + dbcon (OpenPypeMongoConnection) + batch_id (str) + user (str) + Returns + (ObjectId) from DB + """ + return dbcon.insert_one({ + "batch_id": batch_id, + "start_date": datetime.now(), + "user": user, + "status": "in_progress" + }).inserted_id + + +def publish_and_log(dbcon, _id, log): + """Loops through all plugins, logs ok and fails into OP DB. + + Args: + dbcon (OpenPypeMongoConnection) + _id (str) + log (OpenPypeLogger) + """ + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + if isinstance(_id, str): + _id = ObjectId(_id) + + log_lines = [] + for result in pyblish.util.publish_iter(): + for record in result["records"]: + log_lines.append("{}: {}".format( + result["plugin"].label, record.msg)) + + if result["error"]: + log.error(error_format.format(**result)) + uninstall() + log_lines.append(error_format.format(**result)) + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": "error", + "log": os.linesep.join(log_lines) + + }} + ) + sys.exit(1) + else: + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "progress": max(result["progress"], 0.95), + "log": os.linesep.join(log_lines) + }} + ) + + # final update + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": "finished_ok", + "progress": 1, + "log": os.linesep.join(log_lines) + }} + ) \ No newline at end of file diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index e06ab5b493..e2869a956d 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -9,6 +9,11 @@ import time from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context from openpype.lib.plugin_tools import parse_json, get_batch_asset_task_info +from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + publish_and_log +) class PypeCommands: @@ -152,8 +157,6 @@ class PypeCommands: app_name = "{}/{}".format(host, found_variant_key) - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_dir - batch_data = None if batch_dir and os.path.exists(batch_dir): # TODO check if batch manifest is same as tasks manifests @@ -170,10 +173,6 @@ class PypeCommands: batch_data["task"], batch_data["files"][0]) print("workfile_path {}".format(workfile_path)) - data = { - "last_workfile_path": workfile_path, - "start_last_workfile": True - } # must have for proper launch of app env = get_app_environments_for_context( @@ -184,19 +183,34 @@ class PypeCommands: ) os.environ.update(env) + _, batch_id = os.path.split(batch_dir) + dbcon = get_webpublish_conn() + # safer to start logging here, launch might be broken altogether + _id = start_webpublish_log(dbcon, batch_id, user) + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_dir os.environ["IS_HEADLESS"] = "true" + # must pass identifier to update log lines for a batch + os.environ["BATCH_LOG_ID"] = str(_id) + + data = { + "last_workfile_path": workfile_path, + "start_last_workfile": True + } launched_app = application_manager.launch(app_name, **data) while launched_app.poll() is None: time.sleep(0.5) - print(launched_app) + uninstall() @staticmethod def remotepublish(project, batch_path, host, user, targets=None): """Start headless publishing. + Used to publish rendered assets, workfiles etc. + Publish use json from passed paths argument. Args: @@ -217,7 +231,6 @@ class PypeCommands: from openpype import install, uninstall from openpype.api import Logger - from openpype.lib import OpenPypeMongoConnection # Register target and host import pyblish.api @@ -249,62 +262,11 @@ class PypeCommands: log.info("Running publish ...") - # Error exit as soon as any error occurs. - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" - - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - dbcon = mongo_client[database_name]["webpublishes"] - _, batch_id = os.path.split(batch_path) - _id = dbcon.insert_one({ - "batch_id": batch_id, - "start_date": datetime.now(), - "user": user, - "status": "in_progress" - }).inserted_id + dbcon = get_webpublish_conn() + _id = start_webpublish_log(dbcon, batch_id, user) - log_lines = [] - for result in pyblish.util.publish_iter(): - for record in result["records"]: - log_lines.append("{}: {}".format( - result["plugin"].label, record.msg)) - - if result["error"]: - log.error(error_format.format(**result)) - uninstall() - log_lines.append(error_format.format(**result)) - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": "error", - "log": os.linesep.join(log_lines) - - }} - ) - sys.exit(1) - else: - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "progress": max(result["progress"], 0.95), - "log": os.linesep.join(log_lines) - }} - ) - - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": "finished_ok", - "progress": 1, - "log": os.linesep.join(log_lines) - }} - ) + publish_and_log(dbcon, _id, log) log.info("Publish finished.") uninstall() From b0705417ab1236bce923142c92cb3d2b0a0c463a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 13 Oct 2021 17:38:37 +0200 Subject: [PATCH 18/24] PYPE-1901 - close host if any plugin fails --- .../photoshop/plugins/publish/closePS.py | 1 + openpype/lib/remote_publish.py | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/closePS.py b/openpype/hosts/photoshop/plugins/publish/closePS.py index fa9d27688b..19994a0db8 100644 --- a/openpype/hosts/photoshop/plugins/publish/closePS.py +++ b/openpype/hosts/photoshop/plugins/publish/closePS.py @@ -6,6 +6,7 @@ import pyblish.api from avalon import photoshop + class ClosePS(pyblish.api.ContextPlugin): """Close PS after publish. For Webpublishing only. """ diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index aa8d8821a8..6cca9e4217 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -4,6 +4,7 @@ import sys from bson.objectid import ObjectId import pyblish.util +import pyblish.api from openpype import uninstall from openpype.lib.mongo import OpenPypeMongoConnection @@ -34,17 +35,21 @@ def start_webpublish_log(dbcon, batch_id, user): }).inserted_id -def publish_and_log(dbcon, _id, log): +def publish_and_log(dbcon, _id, log, close_plugin_name=None): """Loops through all plugins, logs ok and fails into OP DB. Args: dbcon (OpenPypeMongoConnection) _id (str) log (OpenPypeLogger) + close_plugin_name (str): name of plugin with responsibility to + close host app """ # Error exit as soon as any error occurs. error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + close_plugin = _get_close_plugin(close_plugin_name, log) + if isinstance(_id, str): _id = ObjectId(_id) @@ -68,6 +73,9 @@ def publish_and_log(dbcon, _id, log): }} ) + if close_plugin: # close host app explicitly after error + context = pyblish.api.Context() + close_plugin(context).process() sys.exit(1) else: dbcon.update_one( @@ -89,4 +97,14 @@ def publish_and_log(dbcon, _id, log): "progress": 1, "log": os.linesep.join(log_lines) }} - ) \ No newline at end of file + ) + + +def _get_close_plugin(close_plugin_name, log): + if close_plugin_name: + plugins = pyblish.api.discover() + for plugin in plugins: + if plugin.__name__ == close_plugin_name: + return plugin + + log.warning("Close plugin not found, app might not close.") From 6fe7517303a359fdf91ab54353769e69cca1677d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Oct 2021 13:21:34 +0200 Subject: [PATCH 19/24] PYPE-1901 - fix close plugin --- openpype/lib/remote_publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index 6cca9e4217..4946e1bd53 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -75,7 +75,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None): ) if close_plugin: # close host app explicitly after error context = pyblish.api.Context() - close_plugin(context).process() + close_plugin().process(context) sys.exit(1) else: dbcon.update_one( From b0628953d8039a65ffcc366a28f5cbd5c564e10a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Oct 2021 19:18:59 +0200 Subject: [PATCH 20/24] PYPE-1901 - reworked Settings Layer could be resolved to final family according to color_code OR name (or both). --- openpype/hooks/pre_foundry_apps.py | 2 +- .../publish/collect_remote_instances.py | 72 +++++++++++++++---- .../defaults/project_settings/photoshop.json | 11 ++- .../schema_project_photoshop.json | 49 +++++++++---- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/openpype/hooks/pre_foundry_apps.py b/openpype/hooks/pre_foundry_apps.py index 85f68c6b60..7df1a6a833 100644 --- a/openpype/hooks/pre_foundry_apps.py +++ b/openpype/hooks/pre_foundry_apps.py @@ -13,7 +13,7 @@ class LaunchFoundryAppsWindows(PreLaunchHook): # Should be as last hook because must change launch arguments to string order = 1000 - app_groups = ["nuke", "nukex", "hiero", "nukestudio"] + app_groups = ["nuke", "nukex", "hiero", "nukestudio", "photoshop"] platforms = ["windows"] def execute(self): diff --git a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py index 62d94483e5..9bb8e90350 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_remote_instances.py @@ -1,5 +1,6 @@ import pyblish.api import os +import re from avalon import photoshop from openpype.lib import prepare_template_data @@ -22,12 +23,11 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): hosts = ["photoshop"] # configurable by Settings - families = ["background"] - color_code = ["red"] - subset_template_name = "" + color_code_mapping = [] def process(self, context): self.log.info("CollectRemoteInstances") + self.log.info("mapping:: {}".format(self.color_code_mapping)) if not os.environ.get("IS_HEADLESS"): self.log.debug("Not headless publishing, skipping.") return @@ -49,24 +49,26 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): instance_names = [] for layer in layers: - self.log.info("!!!Layer:: {}".format(layer)) - if layer.color_code not in self.color_code: - self.log.debug("Not marked, skip") + self.log.info("Layer:: {}".format(layer)) + resolved_family, resolved_subset_template = self._resolve_mapping( + layer + ) + self.log.info("resolved_family {}".format(resolved_family)) + self.log.info("resolved_subset_template {}".format( + resolved_subset_template)) + + if not resolved_subset_template or not resolved_family: + self.log.debug("!!! Not marked, skip") continue if layer.parents: - self.log.debug("Not a top layer, skip") + self.log.debug("!!! Not a top layer, skip") continue instance = context.create_instance(layer.name) instance.append(layer) - instance.data["family"] = self.families[0] + instance.data["family"] = resolved_family instance.data["publish"] = layer.visible - - # populate data from context, coming from outside?? TODO - # TEMP - self.log.info("asset {}".format(context.data["assetEntity"])) - self.log.info("taskType {}".format(context.data["taskType"])) instance.data["asset"] = context.data["assetEntity"]["name"] instance.data["task"] = context.data["taskType"] @@ -76,7 +78,7 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): "task": instance.data["task"], "layer": layer.name } - subset = self.subset_template_name.format( + subset = resolved_subset_template.format( **prepare_template_data(fill_pairs)) instance.data["subset"] = subset @@ -90,3 +92,45 @@ class CollectRemoteInstances(pyblish.api.ContextPlugin): if len(instance_names) != len(set(instance_names)): self.log.warning("Duplicate instances found. " + "Remove unwanted via SubsetManager") + + def _resolve_mapping(self, layer): + """Matches 'layer' color code and name to mapping. + + If both color code AND name regex is configured, BOTH must be valid + If layer matches to multiple mappings, only first is used! + """ + family_list = [] + family = None + subset_name_list = [] + resolved_subset_template = None + for mapping in self.color_code_mapping: + if mapping["color_code"] and \ + layer.color_code not in mapping["color_code"]: + break + + if mapping["layer_name_regex"] and \ + not any(re.search(pattern, layer.name) + for pattern in mapping["layer_name_regex"]): + break + + family_list.append(mapping["family"]) + subset_name_list.append(mapping["subset_template_name"]) + + if len(subset_name_list) > 1: + self.log.warning("Multiple mappings found for '{}'". + format(layer.name)) + self.log.warning("Only first subset name template used!") + subset_name_list[:] = subset_name_list[0] + + if len(family_list) > 1: + self.log.warning("Multiple mappings found for '{}'". + format(layer.name)) + self.log.warning("Only first family used!") + family_list[:] = family_list[0] + + if subset_name_list: + resolved_subset_template = subset_name_list.pop() + if family_list: + family = family_list.pop() + + return family, resolved_subset_template diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index 14c294c0c5..03fcbc162c 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -13,9 +13,14 @@ "active": true }, "CollectRemoteInstances": { - "color_code": [], - "families": [], - "subset_template_name": "" + "color_code_mapping": [ + { + "color_code": [], + "layer_name_regex": [], + "family": "", + "subset_template_name": "" + } + ] }, "ExtractImage": { "formats": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index 008f1a265d..cd457ee21d 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -46,6 +46,7 @@ { "type": "dict", "collapsible": true, + "is_group": true, "key": "CollectRemoteInstances", "label": "Collect Instances for Webpublish", "children": [ @@ -55,20 +56,40 @@ }, { "type": "list", - "key": "color_code", - "label": "Color codes for layers", - "object_type": "text" - }, - { - "key": "families", - "label": "Families", - "type": "list", - "object_type": "text" - }, - { - "type": "text", - "key": "subset_template_name", - "label": "Subset template name" + "key": "color_code_mapping", + "label": "Color code mappings", + "use_label_wrap": false, + "collapsible": false, + "object_type": { + "type": "dict", + "children": [ + { + "type": "list", + "key": "color_code", + "label": "Color codes for layers", + "object_type": "text" + }, + { + "type": "list", + "key": "layer_name_regex", + "label": "Layer name regex", + "object_type": "text" + }, + { + "type": "splitter" + }, + { + "key": "family", + "label": "Resulting family", + "type": "text" + }, + { + "type": "text", + "key": "subset_template_name", + "label": "Subset template name" + } + ] + } } ] }, From 3fd25b14a770107d3282bf6116a0d72f37fbd163 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 18 Oct 2021 11:26:25 +0200 Subject: [PATCH 21/24] OP-1206 - added reference loader wip --- .../photoshop/plugins/load/load_image.py | 5 ++++- .../photoshop/plugins/load/load_reference.py | 22 +++++++++++++++++++ .../collect_default_deadline_server.py | 2 +- .../defaults/project_anatomy/imageio.json | 4 +--- 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 openpype/hosts/photoshop/plugins/load/load_reference.py diff --git a/openpype/hosts/photoshop/plugins/load/load_image.py b/openpype/hosts/photoshop/plugins/load/load_image.py index d043323768..d97894b269 100644 --- a/openpype/hosts/photoshop/plugins/load/load_image.py +++ b/openpype/hosts/photoshop/plugins/load/load_image.py @@ -21,7 +21,7 @@ class ImageLoader(api.Loader): context["asset"]["name"], name) with photoshop.maintained_selection(): - layer = stub.import_smart_object(self.fname, layer_name) + layer = self.import_layer(self.fname, layer_name) self[:] = [layer] namespace = namespace or layer_name @@ -72,3 +72,6 @@ class ImageLoader(api.Loader): def switch(self, container, representation): self.update(container, representation) + + def import_layer(self, file_name, layer_name): + return stub.import_smart_object(file_name, layer_name) diff --git a/openpype/hosts/photoshop/plugins/load/load_reference.py b/openpype/hosts/photoshop/plugins/load/load_reference.py new file mode 100644 index 0000000000..306647c032 --- /dev/null +++ b/openpype/hosts/photoshop/plugins/load/load_reference.py @@ -0,0 +1,22 @@ +import re + +from avalon import api, photoshop + +from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name +from openpype.hosts.photoshop.plugins.load.load_image import ImageLoader + +stub = photoshop.stub() + + +class ReferenceLoader(ImageLoader): + """Load reference images + + Stores the imported asset in a container named after the asset. + """ + + families = ["image", "render"] + representations = ["*"] + + def import_layer(self, file_name, layer_name): + return stub.import_smart_object(file_name, layer_name, + as_reference=True) diff --git a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py b/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py index afb8583069..53231bd7e4 100644 --- a/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py +++ b/openpype/modules/default_modules/deadline/plugins/publish/collect_default_deadline_server.py @@ -6,7 +6,7 @@ import pyblish.api class CollectDefaultDeadlineServer(pyblish.api.ContextPlugin): """Collect default Deadline Webservice URL.""" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder + 0.01 label = "Default Deadline Webservice" def process(self, context): diff --git a/openpype/settings/defaults/project_anatomy/imageio.json b/openpype/settings/defaults/project_anatomy/imageio.json index 38313a3d84..25608f67c6 100644 --- a/openpype/settings/defaults/project_anatomy/imageio.json +++ b/openpype/settings/defaults/project_anatomy/imageio.json @@ -162,9 +162,7 @@ ] } ], - "customNodes": [ - - ] + "customNodes": [] }, "regexInputs": { "inputs": [ From 62977a9b4f184fc3316a0073e0e317f3967b1632 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Oct 2021 12:05:40 +0200 Subject: [PATCH 22/24] OP-1206 - added reference loader wip --- .../photoshop/plugins/load/load_image.py | 6 +- .../photoshop/plugins/load/load_reference.py | 65 ++++++++++++++++++- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/load/load_image.py b/openpype/hosts/photoshop/plugins/load/load_image.py index d97894b269..981a1ed204 100644 --- a/openpype/hosts/photoshop/plugins/load/load_image.py +++ b/openpype/hosts/photoshop/plugins/load/load_image.py @@ -6,7 +6,6 @@ from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name stub = photoshop.stub() - class ImageLoader(api.Loader): """Load images @@ -45,8 +44,9 @@ class ImageLoader(api.Loader): layer_name = "{}_{}".format(context["asset"], context["subset"]) # switching assets if namespace_from_container != layer_name: - layer_name = self._get_unique_layer_name(context["asset"], - context["subset"]) + layer_name = get_unique_layer_name(stub.get_layers(), + context["asset"], + context["subset"]) else: # switching version - keep same name layer_name = container["namespace"] diff --git a/openpype/hosts/photoshop/plugins/load/load_reference.py b/openpype/hosts/photoshop/plugins/load/load_reference.py index 306647c032..1b54bd97f1 100644 --- a/openpype/hosts/photoshop/plugins/load/load_reference.py +++ b/openpype/hosts/photoshop/plugins/load/load_reference.py @@ -8,15 +8,76 @@ from openpype.hosts.photoshop.plugins.load.load_image import ImageLoader stub = photoshop.stub() -class ReferenceLoader(ImageLoader): +class ReferenceLoader(api.Loader): """Load reference images - Stores the imported asset in a container named after the asset. + Stores the imported asset in a container named after the asset. + + Inheriting from 'load_image' didn't work because of + "Cannot write to closing transport", possible refactor. """ families = ["image", "render"] representations = ["*"] + def load(self, context, name=None, namespace=None, data=None): + layer_name = get_unique_layer_name(stub.get_layers(), + context["asset"]["name"], + name) + with photoshop.maintained_selection(): + layer = self.import_layer(self.fname, layer_name) + + self[:] = [layer] + namespace = namespace or layer_name + + return photoshop.containerise( + name, + namespace, + layer, + context, + self.__class__.__name__ + ) + + def update(self, container, representation): + """ Switch asset or change version """ + layer = container.pop("layer") + + context = representation.get("context", {}) + + namespace_from_container = re.sub(r'_\d{3}$', '', + container["namespace"]) + layer_name = "{}_{}".format(context["asset"], context["subset"]) + # switching assets + if namespace_from_container != layer_name: + layer_name = get_unique_layer_name(stub.get_layers(), + context["asset"], + context["subset"]) + else: # switching version - keep same name + layer_name = container["namespace"] + + path = api.get_representation_path(representation) + with photoshop.maintained_selection(): + stub.replace_smart_object( + layer, path, layer_name + ) + + stub.imprint( + layer, {"representation": str(representation["_id"])} + ) + + def remove(self, container): + """ + Removes element from scene: deletes layer + removes from Headline + Args: + container (dict): container to be removed - used to get layer_id + """ + layer = container.pop("layer") + stub.imprint(layer, {}) + stub.delete_layer(layer.id) + + def switch(self, container, representation): + self.update(container, representation) + def import_layer(self, file_name, layer_name): return stub.import_smart_object(file_name, layer_name, as_reference=True) From 8b1952ca4be2d4a8a477fc0361913f259c88f83a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Oct 2021 15:22:38 +0200 Subject: [PATCH 23/24] Hound --- openpype/hosts/photoshop/plugins/load/load_reference.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/photoshop/plugins/load/load_reference.py b/openpype/hosts/photoshop/plugins/load/load_reference.py index 1b54bd97f1..0cb4e4a69f 100644 --- a/openpype/hosts/photoshop/plugins/load/load_reference.py +++ b/openpype/hosts/photoshop/plugins/load/load_reference.py @@ -3,7 +3,6 @@ import re from avalon import api, photoshop from openpype.hosts.photoshop.plugins.lib import get_unique_layer_name -from openpype.hosts.photoshop.plugins.load.load_image import ImageLoader stub = photoshop.stub() From 9b529ce589c4b5e4cc80ce949677372146dcf1ee Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 17:56:40 +0200 Subject: [PATCH 24/24] OP-1206 - fix no repres_widget issue without sync server --- openpype/tools/loader/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/tools/loader/app.py b/openpype/tools/loader/app.py index dac5e11d4c..04da08326f 100644 --- a/openpype/tools/loader/app.py +++ b/openpype/tools/loader/app.py @@ -164,8 +164,9 @@ class LoaderWindow(QtWidgets.QDialog): subsets_widget.load_started.connect(self._on_load_start) subsets_widget.load_ended.connect(self._on_load_end) - repres_widget.load_started.connect(self._on_load_start) - repres_widget.load_ended.connect(self._on_load_end) + if repres_widget: + repres_widget.load_started.connect(self._on_load_start) + repres_widget.load_ended.connect(self._on_load_end) self._sync_server_enabled = sync_server_enabled