From 397ecb529e2a0524435055cfaedb5f465fa1a103 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 17:46:35 +0200 Subject: [PATCH 01/80] nuke validate backdrop with help --- .../publish/help/validate_backdrop.xml | 36 +++++++++++++++++++ .../nuke/plugins/publish/validate_backdrop.py | 19 ++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_backdrop.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_backdrop.xml b/openpype/hosts/nuke/plugins/publish/help/validate_backdrop.xml new file mode 100644 index 0000000000..ab1b650773 --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_backdrop.xml @@ -0,0 +1,36 @@ + + + + Found multiple outputs + +## Invalid output amount + +Backdrop is having more than one outgoing connections. + +### How to repair? + +1. Use button `Center node in node graph` and navigate to the backdrop. +2. Reorganize nodes the way only one outgoing connection is present. +3. Hit reload button on the publisher. + + +### How could this happen? + +More than one node, which are found above the backdrop, are linked downstream or more output connections from a node also linked downstream. + + + + Empty backdrop + +## Invalid empty backdrop + +Backdrop is empty and no nodes are found above it. + +### How to repair? + +1. Use button `Center node in node graph` and navigate to the backdrop. +2. Add any node above it or delete it. +3. Hit reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_backdrop.py b/openpype/hosts/nuke/plugins/publish/validate_backdrop.py index e2843d146e..2a0d3309a0 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_backdrop.py +++ b/openpype/hosts/nuke/plugins/publish/validate_backdrop.py @@ -1,6 +1,7 @@ import nuke import pyblish from openpype.hosts.nuke.api.lib import maintained_selection +from openpype.pipeline import PublishXmlValidationError class SelectCenterInNodeGraph(pyblish.api.Action): @@ -63,8 +64,20 @@ class ValidateBackdrop(pyblish.api.InstancePlugin): msg_multiple_outputs = ( "Only one outcoming connection from " "\"{}\" is allowed").format(instance.data["name"]) - assert len(connections_out.keys()) <= 1, msg_multiple_outputs - msg_no_content = "No content on backdrop node: \"{}\"".format( + if len(connections_out.keys()) > 1: + raise PublishXmlValidationError( + self, + msg_multiple_outputs, + "multiple_outputs" + ) + + msg_no_nodes = "No content on backdrop node: \"{}\"".format( instance.data["name"]) - assert len(instance) > 1, msg_no_content + + if len(instance) == 0: + raise PublishXmlValidationError( + self, + msg_no_nodes, + "no_nodes" + ) From d097503f4de4828d4a30071769a26328a71f90f9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 17:54:22 +0200 Subject: [PATCH 02/80] nuke validate write nodes with help --- .../publish/help/validate_write_nodes.xml | 17 ++++++++++++++ .../plugins/publish/validate_write_nodes.py | 22 +++++++++++-------- 2 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml new file mode 100644 index 0000000000..d209329434 --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml @@ -0,0 +1,17 @@ + + + + Knobs values + +## Invalid node's knobs values + +Following write node knobs needs to be repaired: +{xml_msg} + +### How to repair? + +1. Use Repair button. +3. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index c0d5c8f402..94dabecc97 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -1,10 +1,10 @@ -import os import pyblish.api -import openpype.utils +from openpype.api import get_errored_instances_from_context from openpype.hosts.nuke.api.lib import ( get_write_node_template_attr, get_node_path ) +from openpype.pipeline import PublishXmlValidationError @pyblish.api.log @@ -14,7 +14,7 @@ class RepairNukeWriteNodeAction(pyblish.api.Action): icon = "wrench" def process(self, context, plugin): - instances = openpype.utils.filter_instances(context, plugin) + instances = get_errored_instances_from_context(context) for instance in instances: node = instance[1] @@ -60,13 +60,17 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): self.log.info(check) - msg = "Node's attribute `{0}` is not correct!\n" \ - "\nCorrect: `{1}` \n\nWrong: `{2}` \n\n" + msg = "Write node's knobs values are not correct!\n" + msg_add = "Knob `{0}` Correct: `{1}` Wrong: `{2}` \n" + xml_msg = "" if check: - print_msg = "" + dbg_msg = msg for item in check: - print_msg += msg.format(item[0], item[1], item[2]) - print_msg += "`RMB` click to the validator and `A` to fix!" + _msg_add = msg_add.format(item[0], item[1], item[2]) + dbg_msg += _msg_add + xml_msg += _msg_add - assert not check, print_msg + raise PublishXmlValidationError( + self, dbg_msg, {"xml_msg": xml_msg} + ) From bdf8c158e5877b1f2bfa08eb8e766e339b52867a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 21 Dec 2021 14:42:25 +0100 Subject: [PATCH 03/80] nuke adding better docstring --- openpype/hosts/nuke/plugins/publish/validate_write_nodes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 94dabecc97..069c6f4d8c 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -25,7 +25,11 @@ class RepairNukeWriteNodeAction(pyblish.api.Action): class ValidateNukeWriteNode(pyblish.api.InstancePlugin): - """ Validates file output. """ + """ Validate Write node's knobs. + + Compare knobs on write node inside the render group + with settings. At the moment supporting only `file` knob. + """ order = pyblish.api.ValidatorOrder optional = True From 1a86a2e2655e81e2633841bf26935d31f42a888a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 17:56:07 +0200 Subject: [PATCH 04/80] nuke validate gizmo with help --- .../plugins/publish/help/validate_gizmo.xml | 36 +++++++++++++++++++ .../nuke/plugins/publish/validate_gizmo.py | 31 +++++++++++----- 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_gizmo.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_gizmo.xml b/openpype/hosts/nuke/plugins/publish/help/validate_gizmo.xml new file mode 100644 index 0000000000..f39a41a4f9 --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_gizmo.xml @@ -0,0 +1,36 @@ + + + + Found multiple outputs + +## Invalid amount of Output nodes + +Group node `{node_name}` is having more than one Output node. + +### How to repair? + +1. Use button `Open Group`. +2. Remove redundant Output node. +3. Hit reload button on the publisher. + + +### How could this happen? + +Perhaps you had created exciently more than one Output node. + + + + Missing Input nodes + +## Missing Input nodes + +Make sure there is at least one connected Input node inside the group node with name `{node_name}` + +### How to repair? + +1. Use button `Open Group`. +2. Add at least one Input node and connect to other nodes. +3. Hit reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_gizmo.py b/openpype/hosts/nuke/plugins/publish/validate_gizmo.py index d0d930f50c..2321bd1fd4 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_gizmo.py +++ b/openpype/hosts/nuke/plugins/publish/validate_gizmo.py @@ -1,6 +1,7 @@ -import nuke import pyblish -from openpype.hosts.nuke.api.lib import maintained_selection +from openpype.pipeline import PublishXmlValidationError +from openpype.hosts.nuke.api import maintained_selection +import nuke class OpenFailedGroupNode(pyblish.api.Action): @@ -8,7 +9,7 @@ class OpenFailedGroupNode(pyblish.api.Action): Centering failed instance node in node grap """ - label = "Open Gizmo in Node Graph" + label = "Open Group" icon = "wrench" on = "failed" @@ -48,11 +49,23 @@ class ValidateGizmo(pyblish.api.InstancePlugin): with grpn: connections_out = nuke.allNodes('Output') - msg_multiple_outputs = "Only one outcoming connection from " - "\"{}\" is allowed".format(instance.data["name"]) - assert len(connections_out) <= 1, msg_multiple_outputs + msg_multiple_outputs = ( + "Only one outcoming connection from " + "\"{}\" is allowed").format(instance.data["name"]) + + if len(connections_out) > 1: + raise PublishXmlValidationError( + self, msg_multiple_outputs, "multiple_outputs", + {"node_name": grpn["name"].value()} + ) connections_in = nuke.allNodes('Input') - msg_missing_inputs = "At least one Input node has to be used in: " - "\"{}\"".format(instance.data["name"]) - assert len(connections_in) >= 1, msg_missing_inputs + msg_missing_inputs = ( + "At least one Input node has to be inside Group: " + "\"{}\"").format(instance.data["name"]) + + if len(connections_in) == 0: + raise PublishXmlValidationError( + self, msg_missing_inputs, "no_inputs", + {"node_name": grpn["name"].value()} + ) From 0666bce2be03be03e9a4ff73ff48ee0d36343735 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 17:58:33 +0200 Subject: [PATCH 05/80] nuke validate asset name with help rename plugin, fix order on how to fix in validate write nodes xml --- .../publish/help/validate_asset_name.xml | 18 ++++++++ .../publish/help/validate_write_nodes.xml | 2 +- ...e_in_context.py => validate_asset_name.py} | 43 ++++++++++++------- 3 files changed, 46 insertions(+), 17 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml rename openpype/hosts/nuke/plugins/publish/{validate_instance_in_context.py => validate_asset_name.py} (75%) diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml b/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml new file mode 100644 index 0000000000..7ea552597a --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml @@ -0,0 +1,18 @@ + + + + Shot/Asset mame + +## Invalid Shot/Asset name in subset + +Following Node with name `{node_name}`: +Is in context of `{context_asset}` but Node _asset_ knob is set as `{asset}`. + +### How to repair? + +1. Either use Repair or Select button. +2. If you chose Select then rename asset knob to correct name. +3. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml index d209329434..c1f59a94f8 100644 --- a/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml +++ b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml @@ -11,7 +11,7 @@ Following write node knobs needs to be repaired: ### How to repair? 1. Use Repair button. -3. Hit Reload button on the publisher. +2. Hit Reload button on the publisher. \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_instance_in_context.py b/openpype/hosts/nuke/plugins/publish/validate_asset_name.py similarity index 75% rename from openpype/hosts/nuke/plugins/publish/validate_instance_in_context.py rename to openpype/hosts/nuke/plugins/publish/validate_asset_name.py index 842f74b6f6..f9adac81f8 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_instance_in_context.py +++ b/openpype/hosts/nuke/plugins/publish/validate_asset_name.py @@ -3,20 +3,17 @@ from __future__ import absolute_import import nuke - import pyblish.api import openpype.api -from openpype.hosts.nuke.api.lib import ( - recreate_instance, - reset_selection, - select_nodes -) +import openpype.hosts.nuke.api.lib as nlib +import openpype.hosts.nuke.api as nuke_api +from openpype.pipeline import PublishXmlValidationError class SelectInvalidInstances(pyblish.api.Action): """Select invalid instances in Outliner.""" - label = "Select Instances" + label = "Select" icon = "briefcase" on = "failed" @@ -50,12 +47,12 @@ class SelectInvalidInstances(pyblish.api.Action): self.deselect() def select(self, instances): - select_nodes( + nlib.select_nodes( [nuke.toNode(str(x)) for x in instances] ) def deselect(self): - reset_selection() + nlib.reset_selection() class RepairSelectInvalidInstances(pyblish.api.Action): @@ -85,12 +82,12 @@ class RepairSelectInvalidInstances(pyblish.api.Action): context_asset = context.data["assetEntity"]["name"] for instance in instances: origin_node = instance[0] - recreate_instance( + nuke_api.lib.recreate_instance( origin_node, avalon_data={"asset": context_asset} ) -class ValidateInstanceInContext(pyblish.api.InstancePlugin): +class ValidateCorrectAssetName(pyblish.api.InstancePlugin): """Validator to check if instance asset match context asset. When working in per-shot style you always publish data in context of @@ -99,15 +96,29 @@ class ValidateInstanceInContext(pyblish.api.InstancePlugin): Action on this validator will select invalid instances in Outliner. """ - order = openpype.api.ValidateContentsOrder - label = "Instance in same Context" + label = "Validate correct asset name" hosts = ["nuke"] - actions = [SelectInvalidInstances, RepairSelectInvalidInstances] + actions = [ + SelectInvalidInstances, + RepairSelectInvalidInstances + ] optional = True def process(self, instance): asset = instance.data.get("asset") context_asset = instance.context.data["assetEntity"]["name"] - msg = "{} has asset {}".format(instance.name, asset) - assert asset == context_asset, msg + + msg = ( + "Instance `{}` has wrong shot/asset name:\n" + "Correct: `{}` | Wrong: `{}`").format( + instance.name, asset, context_asset) + + if asset != context_asset: + PublishXmlValidationError( + self, msg, formatting_data={ + "node_name": instance[0]["name"].value(), + "wrong_name": asset, + "correct_name": context_asset + } + ) From af3623c4dc3626ee474dd7606ac01a757323787a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 18:00:36 +0200 Subject: [PATCH 06/80] nuke fixes on validation and precollect --- .../plugins/publish/help/validate_asset_name.xml | 2 +- .../nuke/plugins/publish/validate_asset_name.py | 5 ++++- .../hosts/nuke/plugins/publish/validate_backdrop.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml b/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml index 7ea552597a..1097909a5f 100644 --- a/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml +++ b/openpype/hosts/nuke/plugins/publish/help/validate_asset_name.xml @@ -6,7 +6,7 @@ ## Invalid Shot/Asset name in subset Following Node with name `{node_name}`: -Is in context of `{context_asset}` but Node _asset_ knob is set as `{asset}`. +Is in context of `{correct_name}` but Node _asset_ knob is set as `{wrong_name}`. ### How to repair? diff --git a/openpype/hosts/nuke/plugins/publish/validate_asset_name.py b/openpype/hosts/nuke/plugins/publish/validate_asset_name.py index f9adac81f8..7647471f8a 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_asset_name.py +++ b/openpype/hosts/nuke/plugins/publish/validate_asset_name.py @@ -36,6 +36,7 @@ class SelectInvalidInstances(pyblish.api.Action): instances = pyblish.api.instances_by_plugin(failed, plugin) if instances: + self.deselect() self.log.info( "Selecting invalid nodes: %s" % ", ".join( [str(x) for x in instances] @@ -114,8 +115,10 @@ class ValidateCorrectAssetName(pyblish.api.InstancePlugin): "Correct: `{}` | Wrong: `{}`").format( instance.name, asset, context_asset) + self.log.debug(msg) + if asset != context_asset: - PublishXmlValidationError( + raise PublishXmlValidationError( self, msg, formatting_data={ "node_name": instance[0]["name"].value(), "wrong_name": asset, diff --git a/openpype/hosts/nuke/plugins/publish/validate_backdrop.py b/openpype/hosts/nuke/plugins/publish/validate_backdrop.py index 2a0d3309a0..17dc79dc56 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_backdrop.py +++ b/openpype/hosts/nuke/plugins/publish/validate_backdrop.py @@ -48,8 +48,9 @@ class SelectCenterInNodeGraph(pyblish.api.Action): @pyblish.api.log class ValidateBackdrop(pyblish.api.InstancePlugin): - """Validate amount of nodes on backdrop node in case user - forgotten to add nodes above the publishing backdrop node""" + """ Validate amount of nodes on backdrop node in case user + forgoten to add nodes above the publishing backdrop node. + """ order = pyblish.api.ValidatorOrder optional = True @@ -75,7 +76,12 @@ class ValidateBackdrop(pyblish.api.InstancePlugin): msg_no_nodes = "No content on backdrop node: \"{}\"".format( instance.data["name"]) - if len(instance) == 0: + self.log.debug( + "Amount of nodes on instance: {}".format( + len(instance)) + ) + + if len(instance) == 1: raise PublishXmlValidationError( self, msg_no_nodes, From 44bd543d131e3b60be44fc79004e8487882ca61c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 21 Dec 2021 18:28:30 +0100 Subject: [PATCH 07/80] nuke: fixing name for validate asset name --- openpype/settings/defaults/project_settings/nuke.json | 2 +- .../schemas/projects_schema/schemas/schema_nuke_publish.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/settings/defaults/project_settings/nuke.json b/openpype/settings/defaults/project_settings/nuke.json index 33ddc2f251..fb1e475e9f 100644 --- a/openpype/settings/defaults/project_settings/nuke.json +++ b/openpype/settings/defaults/project_settings/nuke.json @@ -91,7 +91,7 @@ "write" ] }, - "ValidateInstanceInContext": { + "ValidateCorrectAssetName": { "enabled": true, "optional": true, "active": true diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json index 04df957d67..5635f871d5 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_nuke_publish.json @@ -61,8 +61,8 @@ "name": "template_publish_plugin", "template_data": [ { - "key": "ValidateInstanceInContext", - "label": "Validate Instance In Context" + "key": "ValidateCorrectAssetName", + "label": "Validate Correct Asset name" } ] }, From e2b2bdc15d5f919793848e15b3b2fb2f00c2aec5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 22 Dec 2021 15:29:49 +0100 Subject: [PATCH 08/80] nuke validate knobs with help --- .../nuke/plugins/publish/help/validate_knobs.xml | 16 ++++++++++++++++ .../hosts/nuke/plugins/publish/validate_knobs.py | 8 +++++--- .../nuke/plugins/publish/validate_write_nodes.py | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml b/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml new file mode 100644 index 0000000000..cb5494729b --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml @@ -0,0 +1,16 @@ + + + + Knobs value + +## Invalid node's knobs values + +Following node knobs needs to be repaired: + +### How to repair? + +1. Use Repair button. +2. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_knobs.py b/openpype/hosts/nuke/plugins/publish/validate_knobs.py index d290ff4541..d20051a9d5 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_knobs.py +++ b/openpype/hosts/nuke/plugins/publish/validate_knobs.py @@ -2,6 +2,7 @@ import nuke import pyblish.api import openpype.api +from openpype.pipeline import PublishXmlValidationError class ValidateKnobs(pyblish.api.ContextPlugin): @@ -27,11 +28,12 @@ class ValidateKnobs(pyblish.api.ContextPlugin): optional = True def process(self, context): - invalid = self.get_invalid(context, compute=True) if invalid: - raise RuntimeError( - "Found knobs with invalid values:\n{}".format(invalid) + raise PublishXmlValidationError( + self, + "Found knobs with invalid values:\n{}".format(invalid), + formatting_data={} ) @classmethod diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 069c6f4d8c..f8d8393730 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -76,5 +76,5 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): xml_msg += _msg_add raise PublishXmlValidationError( - self, dbg_msg, {"xml_msg": xml_msg} + self, dbg_msg, formatting_data={"xml_msg": xml_msg} ) From 447742506dcf05301c3818d3ac154c0dd4e25d8d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 21:04:45 +0200 Subject: [PATCH 09/80] Nuke: improving knob validator with p3 compatibility --- openpype/hosts/nuke/plugins/publish/validate_knobs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_knobs.py b/openpype/hosts/nuke/plugins/publish/validate_knobs.py index d20051a9d5..24723edc7a 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_knobs.py +++ b/openpype/hosts/nuke/plugins/publish/validate_knobs.py @@ -1,5 +1,5 @@ import nuke - +import six import pyblish.api import openpype.api from openpype.pipeline import PublishXmlValidationError @@ -64,7 +64,7 @@ class ValidateKnobs(pyblish.api.ContextPlugin): knobs = {} for family in families: for preset in cls.knobs[family]: - knobs.update({preset: cls.knobs[family][preset]}) + knobs[preset] = cls.knobs[family][preset] # Get invalid knobs. nodes = [] @@ -73,8 +73,7 @@ class ValidateKnobs(pyblish.api.ContextPlugin): nodes.append(node) if node.Class() == "Group": node.begin() - for i in nuke.allNodes(): - nodes.append(i) + nodes.extend(iter(nuke.allNodes())) node.end() for node in nodes: @@ -101,7 +100,9 @@ class ValidateKnobs(pyblish.api.ContextPlugin): def repair(cls, instance): invalid = cls.get_invalid(instance) for data in invalid: - if isinstance(data["expected"], unicode): + # TODO: will need to improve type definitions + # with the new settings for knob types + if isinstance(data["expected"], six.text_type): data["knob"].setValue(str(data["expected"])) continue From 568ce1858163a79ad8b63db78c72a5f28bd77a63 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 22 Dec 2021 20:36:49 +0100 Subject: [PATCH 10/80] fixing validate knobs plugin --- .../hosts/nuke/plugins/publish/help/validate_knobs.xml | 2 ++ openpype/hosts/nuke/plugins/publish/validate_knobs.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml b/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml index cb5494729b..76c184f653 100644 --- a/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml +++ b/openpype/hosts/nuke/plugins/publish/help/validate_knobs.xml @@ -7,6 +7,8 @@ Following node knobs needs to be repaired: +{invalid_items} + ### How to repair? 1. Use Repair button. diff --git a/openpype/hosts/nuke/plugins/publish/validate_knobs.py b/openpype/hosts/nuke/plugins/publish/validate_knobs.py index 24723edc7a..6df0afd5ba 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_knobs.py +++ b/openpype/hosts/nuke/plugins/publish/validate_knobs.py @@ -30,10 +30,16 @@ class ValidateKnobs(pyblish.api.ContextPlugin): def process(self, context): invalid = self.get_invalid(context, compute=True) if invalid: + invalid_items = [ + ("Node __{node_name}__ with knob _{label}_ " + "expecting _{expected}_, " + "but is set to _{current}_").format(**i) + for i in invalid] raise PublishXmlValidationError( self, "Found knobs with invalid values:\n{}".format(invalid), - formatting_data={} + formatting_data={ + "invalid_items": "\n".join(invalid_items)} ) @classmethod @@ -85,6 +91,7 @@ class ValidateKnobs(pyblish.api.ContextPlugin): if node[knob].value() != expected: invalid_knobs.append( { + "node_name": node.name(), "knob": node[knob], "name": node[knob].name(), "label": node[knob].label(), From d5f5ba5bbfe361349503b0b24c4b0bc06450f796 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 3 Jan 2022 17:40:46 +0100 Subject: [PATCH 11/80] nuke validate output resolution with help --- .../help/validate_output_resolution.xml | 16 +++ .../publish/validate_output_resolution.py | 123 ++++++++++-------- 2 files changed, 85 insertions(+), 54 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_output_resolution.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_output_resolution.xml b/openpype/hosts/nuke/plugins/publish/help/validate_output_resolution.xml new file mode 100644 index 0000000000..08a88a993e --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_output_resolution.xml @@ -0,0 +1,16 @@ + + + + Output format + +## Invalid format setting + +Either the Reformat node inside of the render group is missing or the Reformat node output format knob is not set to `root.format`. + +### How to repair? + +1. Use Repair button. +2. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py index 27094b8d74..dbd388c2e6 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py +++ b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py @@ -1,43 +1,9 @@ -import nuke import pyblish.api - - -class RepairWriteResolutionDifference(pyblish.api.Action): - - label = "Repair" - icon = "wrench" - on = "failed" - - def process(self, context, plugin): - - # Get the errored instances - failed = [] - for result in context.data["results"]: - if (result["error"] is not None and result["instance"] is not None - and result["instance"] not in failed): - failed.append(result["instance"]) - - # Apply pyblish.logic to get the instances for the plug-in - instances = pyblish.api.instances_by_plugin(failed, plugin) - - for instance in instances: - reformat = instance[0].dependencies()[0] - if reformat.Class() != "Reformat": - reformat = nuke.nodes.Reformat(inputs=[instance[0].input(0)]) - - xpos = instance[0].xpos() - ypos = instance[0].ypos() - 26 - - dependent_ypos = instance[0].dependencies()[0].ypos() - if (instance[0].ypos() - dependent_ypos) <= 51: - xpos += 110 - - reformat.setXYpos(xpos, ypos) - - instance[0].setInput(0, reformat) - - reformat["resize"].setValue("none") +import openpype.api +import avalon.nuke.lib as anlib +from openpype.pipeline import PublishXmlValidationError +import nuke class ValidateOutputResolution(pyblish.api.InstancePlugin): @@ -52,27 +18,76 @@ class ValidateOutputResolution(pyblish.api.InstancePlugin): families = ["render", "render.local", "render.farm"] label = "Write Resolution" hosts = ["nuke"] - actions = [RepairWriteResolutionDifference] + actions = [openpype.api.RepairAction] + + missing_msg = "Missing Reformat node in render group node" + resolution_msg = "Reformat is set to wrong format" def process(self, instance): + invalid = self.get_invalid(instance) + if invalid: + raise PublishXmlValidationError(self, invalid) - # Skip bounding box check if a reformat node exists. - if instance[0].dependencies()[0].Class() == "Reformat": - return + @classmethod + def get_reformat(cls, instance): + reformat = None + for inode in instance: + if inode.Class() != "Reformat": + continue + reformat = inode - msg = "Bounding box is outside the format." - assert self.check_resolution(instance), msg + return reformat - def check_resolution(self, instance): - node = instance[0] + @classmethod + def get_invalid(cls, instance): + def _check_resolution(instance, reformat): + root_width = instance.data["resolutionWidth"] + root_height = instance.data["resolutionHeight"] - root_width = instance.data["resolutionWidth"] - root_height = instance.data["resolutionHeight"] + write_width = reformat.format().width() + write_height = reformat.format().height() - write_width = node.format().width() - write_height = node.format().height() + if (root_width != write_width) or (root_height != write_height): + return None + else: + return True - if (root_width != write_width) or (root_height != write_height): - return None - else: - return True + # check if reformat is in render node + reformat = cls.get_reformat(instance) + if not reformat: + return cls.missing_msg + + # check if reformat is set to correct root format + correct_format = _check_resolution(instance, reformat) + if not correct_format: + return cls.resolution_msg + + + @classmethod + def repair(cls, instance): + invalid = cls.get_invalid(instance) + grp_node = instance[0] + + if cls.missing_msg == invalid: + # make sure we are inside of the group node + with grp_node: + # find input node and select it + input = None + for inode in instance: + if inode.Class() != "Input": + continue + input = inode + + # add reformat node under it + with anlib.maintained_selection(): + input['selected'].setValue(True) + _rfn = nuke.createNode("Reformat", "name Reformat01") + _rfn["resize"].setValue(0) + _rfn["black_outside"].setValue(1) + + cls.log.info("I am adding reformat node") + + if cls.resolution_msg == invalid: + reformat = cls.get_reformat(instance) + reformat["format"].setValue(nuke.root()["format"].value()) + cls.log.info("I am fixing reformat to root.format") \ No newline at end of file From 01ba3d1c8a21b7ae2dd13daff67b30969997b8b5 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 May 2022 21:10:41 +0200 Subject: [PATCH 12/80] Nuke: validator ditch avalon dependency --- .../plugins/publish/validate_output_resolution.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py index dbd388c2e6..710adde069 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py +++ b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py @@ -1,7 +1,7 @@ import pyblish.api import openpype.api -import avalon.nuke.lib as anlib +from openpype.hosts.nuke.api import maintained_selection from openpype.pipeline import PublishXmlValidationError import nuke @@ -62,7 +62,6 @@ class ValidateOutputResolution(pyblish.api.InstancePlugin): if not correct_format: return cls.resolution_msg - @classmethod def repair(cls, instance): invalid = cls.get_invalid(instance) @@ -72,15 +71,15 @@ class ValidateOutputResolution(pyblish.api.InstancePlugin): # make sure we are inside of the group node with grp_node: # find input node and select it - input = None + _input = None for inode in instance: if inode.Class() != "Input": continue - input = inode + _input = inode # add reformat node under it - with anlib.maintained_selection(): - input['selected'].setValue(True) + with maintained_selection(): + _input['selected'].setValue(True) _rfn = nuke.createNode("Reformat", "name Reformat01") _rfn["resize"].setValue(0) _rfn["black_outside"].setValue(1) From b9e16e87f9d2b26861b5a33e1bba502ffdfafeba Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 13:07:39 +0200 Subject: [PATCH 13/80] Nuke: proxy mode validator refactory to new publisher --- .../plugins/publish/help/validate_proxy_mode.xml | 16 ++++++++++++++++ .../nuke/plugins/publish/validate_proxy_mode.py | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_proxy_mode.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_proxy_mode.xml b/openpype/hosts/nuke/plugins/publish/help/validate_proxy_mode.xml new file mode 100644 index 0000000000..6fe5d5d43e --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_proxy_mode.xml @@ -0,0 +1,16 @@ + + + + Proxy mode + +## Invalid proxy mode value + +Nuke is set to use Proxy. This is not supported by publisher. + +### How to repair? + +1. Use Repair button. +2. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py b/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py index 9c6ca03ffd..e5f3ae9800 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py +++ b/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py @@ -1,5 +1,6 @@ import pyblish import nuke +from openpype.pipeline import PublishXmlValidationError class FixProxyMode(pyblish.api.Action): @@ -7,7 +8,7 @@ class FixProxyMode(pyblish.api.Action): Togger off proxy switch OFF """ - label = "Proxy toggle to OFF" + label = "Repair" icon = "wrench" on = "failed" @@ -30,4 +31,7 @@ class ValidateProxyMode(pyblish.api.ContextPlugin): rootNode = nuke.root() isProxy = rootNode["proxy"].value() - assert not isProxy, "Proxy mode should be toggled OFF" + if not isProxy: + raise PublishXmlValidationError( + self, "Proxy mode should be toggled OFF" + ) From 44655d25156b1d8a1587ca898cc5aaf2a17e898e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 13:08:18 +0200 Subject: [PATCH 14/80] Nuke: improving code readability --- .../nuke/plugins/publish/validate_knobs.py | 11 ++-- .../plugins/publish/validate_write_nodes.py | 62 ++++++++++--------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_knobs.py b/openpype/hosts/nuke/plugins/publish/validate_knobs.py index 6df0afd5ba..573c25f3fe 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_knobs.py +++ b/openpype/hosts/nuke/plugins/publish/validate_knobs.py @@ -31,10 +31,13 @@ class ValidateKnobs(pyblish.api.ContextPlugin): invalid = self.get_invalid(context, compute=True) if invalid: invalid_items = [ - ("Node __{node_name}__ with knob _{label}_ " - "expecting _{expected}_, " - "but is set to _{current}_").format(**i) - for i in invalid] + ( + "Node __{node_name}__ with knob _{label}_ " + "expecting _{expected}_, " + "but is set to _{current}_" + ).format(**i) + for i in invalid + ] raise PublishXmlValidationError( self, "Found knobs with invalid values:\n{}".format(invalid), diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index f8d8393730..320307d09f 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -19,8 +19,8 @@ class RepairNukeWriteNodeAction(pyblish.api.Action): for instance in instances: node = instance[1] correct_data = get_write_node_template_attr(node) - for k, v in correct_data.items(): - node[k].setValue(v) + for key, value in correct_data.items(): + node[key].setValue(value) self.log.info("Node attributes were fixed") @@ -41,40 +41,44 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): def process(self, instance): node = instance[1] - correct_data = get_write_node_template_attr(node) + write_group_node = instance[0] + correct_data = get_write_node_template_attr(write_group_node) check = [] - for k, v in correct_data.items(): - if k is 'file': - padding = len(v.split('#')) - ref_path = get_node_path(v, padding) - n_path = get_node_path(node[k].value(), padding) - isnt = False - for i, p in enumerate(ref_path): - if str(n_path[i]) not in str(p): - if not isnt: - isnt = True - else: - continue - if isnt: - check.append([k, v, node[k].value()]) - else: - if str(node[k].value()) not in str(v): - check.append([k, v, node[k].value()]) + for key, value in correct_data.items(): + if key == 'file': + padding = len(value.split('#')) + ref_path = get_node_path(value, padding) + n_path = get_node_path(node[key].value(), padding) + is_not = False + for i, path in enumerate(ref_path): + if ( + str(n_path[i]) != str(path) + and not is_not + ): + is_not = True + if is_not: + check.append([key, value, node[key].value()]) + + elif str(node[key].value()) != str(value): + check.append([key, value, node[key].value()]) self.log.info(check) + if check: + self._make_error(check) + + def _make_error(self, check): msg = "Write node's knobs values are not correct!\n" + dbg_msg = msg msg_add = "Knob `{0}` Correct: `{1}` Wrong: `{2}` \n" xml_msg = "" - if check: - dbg_msg = msg - for item in check: - _msg_add = msg_add.format(item[0], item[1], item[2]) - dbg_msg += _msg_add - xml_msg += _msg_add + for item in check: + _msg_add = msg_add.format(item[0], item[1], item[2]) + dbg_msg += _msg_add + xml_msg += _msg_add - raise PublishXmlValidationError( - self, dbg_msg, formatting_data={"xml_msg": xml_msg} - ) + raise PublishXmlValidationError( + self, dbg_msg, formatting_data={"xml_msg": xml_msg} + ) From e33c06f95e58db53eb27edbfb85909e6643679dd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 17:26:30 +0200 Subject: [PATCH 15/80] Nuke: abstracting function for correct knob type value --- openpype/hosts/nuke/api/__init__.py | 4 ++- openpype/hosts/nuke/api/lib.py | 43 +++++++++++++++++------------ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/openpype/hosts/nuke/api/__init__.py b/openpype/hosts/nuke/api/__init__.py index b571c4098c..f7288bb287 100644 --- a/openpype/hosts/nuke/api/__init__.py +++ b/openpype/hosts/nuke/api/__init__.py @@ -26,7 +26,8 @@ from .pipeline import ( update_container, ) from .lib import ( - maintained_selection + maintained_selection, + convert_knob_value_to_correct_type ) from .utils import ( @@ -58,6 +59,7 @@ __all__ = ( "update_container", "maintained_selection", + "convert_knob_value_to_correct_type", "colorspace_exists_on_node", "get_colorspace_list" diff --git a/openpype/hosts/nuke/api/lib.py b/openpype/hosts/nuke/api/lib.py index ba8aa7a8db..457e5d851f 100644 --- a/openpype/hosts/nuke/api/lib.py +++ b/openpype/hosts/nuke/api/lib.py @@ -1528,28 +1528,35 @@ def set_node_knobs_from_settings(node, knob_settings, **kwargs): if not knob_value: continue - # first convert string types to string - # just to ditch unicode - if isinstance(knob_value, six.text_type): - knob_value = str(knob_value) - - # set correctly knob types - if knob_type == "bool": - knob_value = bool(knob_value) - elif knob_type == "decimal_number": - knob_value = float(knob_value) - elif knob_type == "number": - knob_value = int(knob_value) - elif knob_type == "text": - knob_value = knob_value - elif knob_type == "color_gui": - knob_value = color_gui_to_int(knob_value) - elif knob_type in ["2d_vector", "3d_vector", "color"]: - knob_value = [float(v) for v in knob_value] + knob_value = convert_knob_value_to_correct_type( + knob_type, knob_value) node[knob_name].setValue(knob_value) +def convert_knob_value_to_correct_type(knob_type, knob_value): + # first convert string types to string + # just to ditch unicode + if isinstance(knob_value, six.text_type): + knob_value = str(knob_value) + + # set correctly knob types + if knob_type == "bool": + knob_value = bool(knob_value) + elif knob_type == "decimal_number": + knob_value = float(knob_value) + elif knob_type == "number": + knob_value = int(knob_value) + elif knob_type == "text": + knob_value = knob_value + elif knob_type == "color_gui": + knob_value = color_gui_to_int(knob_value) + elif knob_type in ["2d_vector", "3d_vector", "color"]: + knob_value = [float(v) for v in knob_value] + + return knob_value + + def color_gui_to_int(color_gui): hex_value = ( "0x{0:0>2x}{1:0>2x}{2:0>2x}{3:0>2x}").format(*color_gui) From f8b45e078c74c64a9fc549e35429649db42b667a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 17:28:04 +0200 Subject: [PATCH 16/80] Nuke: removing obsolete code fixing family to families as Write is stored there in avalon data on node --- openpype/hosts/nuke/api/lib.py | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/openpype/hosts/nuke/api/lib.py b/openpype/hosts/nuke/api/lib.py index 457e5d851f..5d66cb2b89 100644 --- a/openpype/hosts/nuke/api/lib.py +++ b/openpype/hosts/nuke/api/lib.py @@ -836,29 +836,6 @@ def check_subsetname_exists(nodes, subset_name): if subset_name in read_avalon_data(n).get("subset", "")), False) - -def get_render_path(node): - ''' Generate Render path from presets regarding avalon knob data - ''' - avalon_knob_data = read_avalon_data(node) - data = {'avalon': avalon_knob_data} - - nuke_imageio_writes = get_imageio_node_setting( - node_class=avalon_knob_data["family"], - plugin_name=avalon_knob_data["creator"], - subset=avalon_knob_data["subset"] - ) - host_name = os.environ.get("AVALON_APP") - - data.update({ - "app": host_name, - "nuke_imageio_writes": nuke_imageio_writes - }) - - anatomy_filled = format_anatomy(data) - return anatomy_filled["render"]["path"].replace("\\", "/") - - def format_anatomy(data): ''' Helping function for formatting of anatomy paths @@ -2185,15 +2162,14 @@ def get_write_node_template_attr(node): avalon_knob_data = read_avalon_data(node) # get template data nuke_imageio_writes = get_imageio_node_setting( - node_class=avalon_knob_data["family"], + node_class=avalon_knob_data["families"], plugin_name=avalon_knob_data["creator"], subset=avalon_knob_data["subset"] ) + # collecting correct data - correct_data = OrderedDict({ - "file": get_render_path(node) - }) + correct_data = OrderedDict() # adding imageio knob presets for k, v in nuke_imageio_writes.items(): From 1686eaeadad906ddc8e4f20574f4af6c9e697674 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 17:29:07 +0200 Subject: [PATCH 17/80] Nuke: refactory write node validator --- .../publish/help/validate_write_nodes.xml | 1 + .../plugins/publish/validate_write_nodes.py | 86 ++++++++++++------- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml index c1f59a94f8..cdf85102bc 100644 --- a/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml +++ b/openpype/hosts/nuke/plugins/publish/help/validate_write_nodes.xml @@ -6,6 +6,7 @@ ## Invalid node's knobs values Following write node knobs needs to be repaired: + {xml_msg} ### How to repair? diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 320307d09f..f0a7f01dfb 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -1,8 +1,10 @@ import pyblish.api from openpype.api import get_errored_instances_from_context +import openpype.hosts.nuke.api.lib as nlib from openpype.hosts.nuke.api.lib import ( get_write_node_template_attr, - get_node_path + set_node_knobs_from_settings + ) from openpype.pipeline import PublishXmlValidationError @@ -17,10 +19,17 @@ class RepairNukeWriteNodeAction(pyblish.api.Action): instances = get_errored_instances_from_context(context) for instance in instances: - node = instance[1] - correct_data = get_write_node_template_attr(node) - for key, value in correct_data.items(): - node[key].setValue(value) + write_group_node = instance[0] + # get write node from inside of group + write_node = None + for x in instance: + if x.Class() == "Write": + write_node = x + + correct_data = get_write_node_template_attr(write_group_node) + + set_node_knobs_from_settings(write_node, correct_data["knobs"]) + self.log.info("Node attributes were fixed") @@ -39,29 +48,41 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): hosts = ["nuke"] def process(self, instance): - - node = instance[1] write_group_node = instance[0] + + # get write node from inside of group + write_node = None + for x in instance: + if x.Class() == "Write": + write_node = x + + if write_node is None: + return + correct_data = get_write_node_template_attr(write_group_node) - check = [] - for key, value in correct_data.items(): - if key == 'file': - padding = len(value.split('#')) - ref_path = get_node_path(value, padding) - n_path = get_node_path(node[key].value(), padding) - is_not = False - for i, path in enumerate(ref_path): - if ( - str(n_path[i]) != str(path) - and not is_not - ): - is_not = True - if is_not: - check.append([key, value, node[key].value()]) + if correct_data: + check_knobs = correct_data["knobs"] + else: + return - elif str(node[key].value()) != str(value): - check.append([key, value, node[key].value()]) + check = [] + self.log.debug("__ write_node: {}".format( + write_node + )) + + for knob_data in check_knobs: + key = knob_data["name"] + value = knob_data["value"] + self.log.debug("__ key: {} | value: {}".format( + key, value + )) + if ( + str(write_node[key].value()) != str(value) + and key != "file" + and key != "tile_color" + ): + check.append([key, value, write_node[key].value()]) self.log.info(check) @@ -69,15 +90,16 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): self._make_error(check) def _make_error(self, check): - msg = "Write node's knobs values are not correct!\n" - dbg_msg = msg - msg_add = "Knob `{0}` Correct: `{1}` Wrong: `{2}` \n" - xml_msg = "" + # sourcery skip: merge-assign-and-aug-assign, move-assign-in-block + dbg_msg = "Write node's knobs values are not correct!\n" + msg_add = "Knob '{0}' > Correct: `{1}` > Wrong: `{2}`" - for item in check: - _msg_add = msg_add.format(item[0], item[1], item[2]) - dbg_msg += _msg_add - xml_msg += _msg_add + details = [ + msg_add.format(item[0], item[1], item[2]) + for item in check + ] + xml_msg = "
".join(details) + dbg_msg += "\n\t".join(details) raise PublishXmlValidationError( self, dbg_msg, formatting_data={"xml_msg": xml_msg} From e71653fb1672a579dfb96b2843ac55eca4de7985 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 27 May 2022 17:29:21 +0200 Subject: [PATCH 18/80] Nuke: fixing proxy mode validator --- openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py b/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py index e5f3ae9800..dac240ad19 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py +++ b/openpype/hosts/nuke/plugins/publish/validate_proxy_mode.py @@ -31,7 +31,7 @@ class ValidateProxyMode(pyblish.api.ContextPlugin): rootNode = nuke.root() isProxy = rootNode["proxy"].value() - if not isProxy: + if isProxy: raise PublishXmlValidationError( self, "Proxy mode should be toggled OFF" ) From c5258fb295df6c65d81a77a13063034a3b733db0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 12:28:43 +0200 Subject: [PATCH 19/80] nuke: refactory Missing frames validator --- .../publish/help/validate_rendered_frames.xml | 17 +++++++ .../publish/validate_rendered_frames.py | 48 ++++++++++++------- 2 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_rendered_frames.xml diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_rendered_frames.xml b/openpype/hosts/nuke/plugins/publish/help/validate_rendered_frames.xml new file mode 100644 index 0000000000..434081c269 --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_rendered_frames.xml @@ -0,0 +1,17 @@ + + + + Rendered Frames + +## Missing Rendered Frames + +Render node "{node_name}" is set to "Use existing frames", but frames are missing. + +### How to repair? + +1. Use Repair button. +2. Set different target. +2. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py b/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py index af5e8e9d27..f8e128cd26 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py +++ b/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py @@ -1,7 +1,7 @@ import os import pyblish.api -from openpype.api import ValidationException import clique +from openpype.pipeline import PublishXmlValidationError @pyblish.api.log @@ -36,7 +36,7 @@ class RepairActionBase(pyblish.api.Action): class RepairCollectionActionToLocal(RepairActionBase): - label = "Repair > rerender with `Local` machine" + label = "Repair - rerender with \"Local\"" def process(self, context, plugin): instances = self.get_instance(context, plugin) @@ -44,7 +44,7 @@ class RepairCollectionActionToLocal(RepairActionBase): class RepairCollectionActionToFarm(RepairActionBase): - label = "Repair > rerender `On farm` with remote machines" + label = "Repair - rerender with \"On farm\"" def process(self, context, plugin): instances = self.get_instance(context, plugin) @@ -63,6 +63,10 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): def process(self, instance): + f_data = { + "node_name": instance[0]["name"].value() + } + for repre in instance.data["representations"]: if not repre.get("files"): @@ -71,7 +75,8 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): "Check properties of write node (group) and" "select 'Local' option in 'Publish' dropdown.") self.log.error(msg) - raise ValidationException(msg) + raise PublishXmlValidationError( + self, msg, formatting_data=f_data) if isinstance(repre["files"], str): return @@ -82,30 +87,33 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): collection = collections[0] - fstartH = instance.data["frameStartHandle"] - fendH = instance.data["frameEndHandle"] + f_start_h = instance.data["frameStartHandle"] + f_end_h = instance.data["frameEndHandle"] - frame_length = int(fendH - fstartH + 1) + frame_length = int(f_end_h - f_start_h + 1) if frame_length != 1: if len(collections) != 1: msg = "There are multiple collections in the folder" self.log.error(msg) - raise ValidationException(msg) + raise PublishXmlValidationError( + self, msg, formatting_data=f_data) if not collection.is_contiguous(): msg = "Some frames appear to be missing" self.log.error(msg) - raise ValidationException(msg) + raise PublishXmlValidationError( + self, msg, formatting_data=f_data) - collected_frames_len = int(len(collection.indexes)) + collected_frames_len = len(collection.indexes) coll_start = min(collection.indexes) coll_end = max(collection.indexes) self.log.info("frame_length: {}".format(frame_length)) self.log.info("collected_frames_len: {}".format( collected_frames_len)) - self.log.info("fstartH-fendH: {}-{}".format(fstartH, fendH)) + self.log.info("f_start_h-f_end_h: {}-{}".format( + f_start_h, f_end_h)) self.log.info( "coll_start-coll_end: {}-{}".format(coll_start, coll_end)) @@ -116,13 +124,19 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): if ("slate" in instance.data["families"]) \ and (frame_length != collected_frames_len): collected_frames_len -= 1 - fstartH += 1 + f_start_h += 1 - assert ((collected_frames_len >= frame_length) - and (coll_start <= fstartH) - and (coll_end >= fendH)), ( - "{} missing frames. Use repair to render all frames" - ).format(__name__) + if ( + collected_frames_len >= frame_length + and coll_start <= f_start_h + and coll_end >= f_end_h + ): + raise PublishXmlValidationError( + self, ( + "{} missing frames. Use repair to " + "render all frames" + ).format(__name__), formatting_data=f_data + ) instance.data["collection"] = collection From d3e982ebcf757a07d731760734ac51315454449b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 16:49:11 +0200 Subject: [PATCH 20/80] nuke: [wip] validate script attributes --- .../help/validate_script_attributes.xml | 18 ++ .../nuke/plugins/publish/validate_script.py | 156 ------------------ .../publish/validate_script_attributes.py | 129 +++++++++++++++ .../plugins/publish/validate_write_nodes.py | 2 - 4 files changed, 147 insertions(+), 158 deletions(-) create mode 100644 openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml delete mode 100644 openpype/hosts/nuke/plugins/publish/validate_script.py create mode 100644 openpype/hosts/nuke/plugins/publish/validate_script_attributes.py diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml b/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml new file mode 100644 index 0000000000..96f8ab5d38 --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml @@ -0,0 +1,18 @@ + + + + Script attributes + +## Invalid Script attributes + +Following script root attributes need to be fixed: +{missing_attributes} + +### How to repair? + +1. Either use Repair or Select button. +2. If you chose Select then rename asset knob to correct name. +3. Hit Reload button on the publisher. + + + \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_script.py b/openpype/hosts/nuke/plugins/publish/validate_script.py deleted file mode 100644 index b8d7494b9d..0000000000 --- a/openpype/hosts/nuke/plugins/publish/validate_script.py +++ /dev/null @@ -1,156 +0,0 @@ -import pyblish.api - -from openpype.client import get_project, get_asset_by_id, get_asset_by_name -from openpype.pipeline import legacy_io - - -@pyblish.api.log -class ValidateScript(pyblish.api.InstancePlugin): - """ Validates file output. """ - - order = pyblish.api.ValidatorOrder + 0.1 - families = ["workfile"] - label = "Check script settings" - hosts = ["nuke"] - optional = True - - def process(self, instance): - ctx_data = instance.context.data - project_name = legacy_io.active_project() - asset_name = ctx_data["asset"] - # TODO repace query with using 'instance.data["assetEntity"]' - asset = get_asset_by_name(project_name, asset_name) - asset_data = asset["data"] - - # These attributes will be checked - attributes = [ - "fps", - "frameStart", - "frameEnd", - "resolutionWidth", - "resolutionHeight", - "handleStart", - "handleEnd" - ] - - # Value of these attributes can be found on parents - hierarchical_attributes = [ - "fps", - "resolutionWidth", - "resolutionHeight", - "pixelAspect", - "handleStart", - "handleEnd" - ] - - missing_attributes = [] - asset_attributes = {} - for attr in attributes: - if attr in asset_data: - asset_attributes[attr] = asset_data[attr] - - elif attr in hierarchical_attributes: - # TODO this should be probably removed - # Hierarchical attributes is not a thing since Pype 2? - - # Try to find attribute on parent - parent_id = asset['parent'] - parent_type = "project" - if asset_data['visualParent'] is not None: - parent_type = "asset" - parent_id = asset_data['visualParent'] - - value = self.check_parent_hierarchical( - project_name, parent_type, parent_id, attr - ) - if value is None: - missing_attributes.append(attr) - else: - asset_attributes[attr] = value - else: - missing_attributes.append(attr) - - # Raise error if attributes weren't found on asset in database - if len(missing_attributes) > 0: - atr = ", ".join(missing_attributes) - msg = 'Missing attributes "{}" in asset "{}"' - message = msg.format(atr, asset_name) - raise ValueError(message) - - # Get handles from database, Default is 0 (if not found) - handle_start = 0 - handle_end = 0 - if "handleStart" in asset_attributes: - handle_start = asset_attributes["handleStart"] - if "handleEnd" in asset_attributes: - handle_end = asset_attributes["handleEnd"] - - asset_attributes["fps"] = float("{0:.4f}".format( - asset_attributes["fps"])) - - # Get values from nukescript - script_attributes = { - "handleStart": ctx_data["handleStart"], - "handleEnd": ctx_data["handleEnd"], - "fps": float("{0:.4f}".format(ctx_data["fps"])), - "frameStart": ctx_data["frameStart"], - "frameEnd": ctx_data["frameEnd"], - "resolutionWidth": ctx_data["resolutionWidth"], - "resolutionHeight": ctx_data["resolutionHeight"], - "pixelAspect": ctx_data["pixelAspect"] - } - - # Compare asset's values Nukescript X Database - not_matching = [] - for attr in attributes: - self.log.debug("asset vs script attribute \"{}\": {}, {}".format( - attr, asset_attributes[attr], script_attributes[attr]) - ) - if asset_attributes[attr] != script_attributes[attr]: - not_matching.append(attr) - - # Raise error if not matching - if len(not_matching) > 0: - msg = "Attributes '{}' are not set correctly" - # Alert user that handles are set if Frame start/end not match - if ( - (("frameStart" in not_matching) or ("frameEnd" in not_matching)) and - ((handle_start > 0) or (handle_end > 0)) - ): - msg += " (`handle_start` are set to {})".format(handle_start) - msg += " (`handle_end` are set to {})".format(handle_end) - message = msg.format(", ".join(not_matching)) - raise ValueError(message) - - def check_parent_hierarchical( - self, project_name, parent_type, parent_id, attr - ): - if parent_id is None: - return None - - doc = None - if parent_type == "project": - doc = get_project(project_name) - elif parent_type == "asset": - doc = get_asset_by_id(project_name, parent_id) - - if not doc: - return None - - doc_data = doc["data"] - if attr in doc_data: - self.log.info(attr) - return doc_data[attr] - - if parent_type == "project": - return None - - parent_id = doc_data.get("visualParent") - new_parent_type = "asset" - if parent_id is None: - parent_id = doc["parent"] - new_parent_type = "project" - - return self.check_parent_hierarchical( - project_name, new_parent_type, parent_id, attr - ) diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py new file mode 100644 index 0000000000..2411c7fe4e --- /dev/null +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -0,0 +1,129 @@ +from pprint import pformat +import pyblish.api + +from openpype.client import get_project, get_asset_by_id, get_asset_by_name +from openpype.pipeline import legacy_io +from openpype.pipeline import PublishXmlValidationError +import nuke + + +@pyblish.api.log +class ValidateScriptAttributes(pyblish.api.InstancePlugin): + """ Validates file output. """ + + order = pyblish.api.ValidatorOrder + 0.1 + families = ["workfile"] + label = "Validatte script attributes" + hosts = ["nuke"] + optional = True + + def process(self, instance): + ctx_data = instance.context.data + project_name = legacy_io.active_project() + asset_name = ctx_data["asset"] + asset = get_asset_by_name(project_name, asset_name) + asset_data = asset["data"] + + # These attributes will be checked + attributes = [ + "fps", + "frameStart", + "frameEnd", + "resolutionWidth", + "resolutionHeight", + "handleStart", + "handleEnd" + ] + + asset_attributes = { + attr: asset_data[attr] + for attr in attributes + if attr in asset_data + } + + self.log.debug(pformat( + asset_attributes + )) + + handle_start = asset_attributes["handleStart"] + handle_end = asset_attributes["handleEnd"] + asset_attributes["fps"] = float("{0:.4f}".format( + asset_attributes["fps"])) + + root = nuke.root() + # Get values from nukescript + script_attributes = { + "handleStart": ctx_data["handleStart"], + "handleEnd": ctx_data["handleEnd"], + "fps": float("{0:.4f}".format(ctx_data["fps"])), + "frameStart": int(root["first_frame"].getValue()), + "frameEnd": int(root["last_frame"].getValue()), + "resolutionWidth": ctx_data["resolutionWidth"], + "resolutionHeight": ctx_data["resolutionHeight"], + "pixelAspect": ctx_data["pixelAspect"] + } + self.log.debug(pformat( + script_attributes + )) + # Compare asset's values Nukescript X Database + not_matching = [] + for attr in attributes: + self.log.debug( + "Asset vs Script attribute \"{}\": {}, {}".format( + attr, + asset_attributes[attr], + script_attributes[attr] + ) + ) + if asset_attributes[attr] != script_attributes[attr]: + not_matching.append({ + "name": attr, + "expected": asset_attributes[attr], + "actual": script_attributes[attr] + }) + + # Raise error if not matching + if not_matching: + msg = "Attributes '{}' are not set correctly" + # Alert user that handles are set if Frame start/end not match + message = msg.format(", ".join( + [at["name"] for at in not_matching])) + raise PublishXmlValidationError( + self, message, + formatting_data={ + "missing_attributes": not_matching + } + ) + + def check_parent_hierarchical( + self, project_name, parent_type, parent_id, attr + ): + if parent_id is None: + return None + + doc = None + if parent_type == "project": + doc = get_project(project_name) + elif parent_type == "asset": + doc = get_asset_by_id(project_name, parent_id) + + if not doc: + return None + + doc_data = doc["data"] + if attr in doc_data: + self.log.info(attr) + return doc_data[attr] + + if parent_type == "project": + return None + + parent_id = doc_data.get("visualParent") + new_parent_type = "asset" + if parent_id is None: + parent_id = doc["parent"] + new_parent_type = "project" + + return self.check_parent_hierarchical( + project_name, new_parent_type, parent_id, attr + ) diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index f0a7f01dfb..48dce623a9 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -1,10 +1,8 @@ import pyblish.api from openpype.api import get_errored_instances_from_context -import openpype.hosts.nuke.api.lib as nlib from openpype.hosts.nuke.api.lib import ( get_write_node_template_attr, set_node_knobs_from_settings - ) from openpype.pipeline import PublishXmlValidationError From b4b4725b5790b5ffee3b258375ef1bbfe7d17d45 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 17:57:13 +0200 Subject: [PATCH 21/80] nuke: getting testing values from script dirrectly --- .../publish/validate_script_attributes.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index 2411c7fe4e..d9b9a35ece 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -4,6 +4,9 @@ import pyblish.api from openpype.client import get_project, get_asset_by_id, get_asset_by_name from openpype.pipeline import legacy_io from openpype.pipeline import PublishXmlValidationError +from openpype.hosts.nuke.api.lib import ( + get_avalon_knob_data +) import nuke @@ -45,22 +48,32 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): asset_attributes )) - handle_start = asset_attributes["handleStart"] - handle_end = asset_attributes["handleEnd"] asset_attributes["fps"] = float("{0:.4f}".format( asset_attributes["fps"])) root = nuke.root() + knob_data = get_avalon_knob_data(root) + + # Get frame range + first_frame = int(root["first_frame"].getValue()) + last_frame = int(root["last_frame"].getValue()) + + handle_start = int(knob_data["handleStart"]) + handle_end = int(knob_data["handleEnd"]) + + # Get format + _format = root["format"].value() + # Get values from nukescript script_attributes = { - "handleStart": ctx_data["handleStart"], - "handleEnd": ctx_data["handleEnd"], - "fps": float("{0:.4f}".format(ctx_data["fps"])), - "frameStart": int(root["first_frame"].getValue()), - "frameEnd": int(root["last_frame"].getValue()), - "resolutionWidth": ctx_data["resolutionWidth"], - "resolutionHeight": ctx_data["resolutionHeight"], - "pixelAspect": ctx_data["pixelAspect"] + "handleStart": handle_start, + "handleEnd": handle_end, + "fps": float("{0:.4f}".format(root['fps'].value())), + "frameStart": first_frame + handle_start, + "frameEnd": last_frame - handle_end, + "resolutionWidth": _format.width(), + "resolutionHeight": _format.height(), + "pixelAspect": _format.pixelAspect() } self.log.debug(pformat( script_attributes From 62a7b1f713e636d9ffbf66de6d33d9ea0c6c32a0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 18:04:01 +0200 Subject: [PATCH 22/80] nuke: adding repair action to script attribute validator --- .../publish/validate_script_attributes.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index d9b9a35ece..605145149d 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -4,12 +4,27 @@ import pyblish.api from openpype.client import get_project, get_asset_by_id, get_asset_by_name from openpype.pipeline import legacy_io from openpype.pipeline import PublishXmlValidationError +from openpype.api import get_errored_instances_from_context from openpype.hosts.nuke.api.lib import ( - get_avalon_knob_data + get_avalon_knob_data, + WorkfileSettings ) import nuke +@pyblish.api.log +class RepairScriptAttributes(pyblish.api.Action): + label = "Repair" + on = "failed" + icon = "wrench" + + def process(self, context, plugin): + instances = get_errored_instances_from_context(context) + + self.log.debug(instances) + WorkfileSettings().set_context_settings() + + @pyblish.api.log class ValidateScriptAttributes(pyblish.api.InstancePlugin): """ Validates file output. """ @@ -19,6 +34,7 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): label = "Validatte script attributes" hosts = ["nuke"] optional = True + actions = [RepairScriptAttributes] def process(self, instance): ctx_data = instance.context.data From 907500c9e9354c47d945a1abd785f5520be26bc2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 21:26:13 +0200 Subject: [PATCH 23/80] nuke: validate script attributes finish --- .../help/validate_script_attributes.xml | 8 +- .../publish/validate_script_attributes.py | 107 ++++++++---------- 2 files changed, 49 insertions(+), 66 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml b/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml index 96f8ab5d38..871fc629ce 100644 --- a/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml +++ b/openpype/hosts/nuke/plugins/publish/help/validate_script_attributes.xml @@ -6,13 +6,13 @@ ## Invalid Script attributes Following script root attributes need to be fixed: -{missing_attributes} + +{failed_attributes} ### How to repair? -1. Either use Repair or Select button. -2. If you chose Select then rename asset knob to correct name. -3. Hit Reload button on the publisher. +1. Use Repair. +2. Hit Reload button on the publisher. \ No newline at end of file diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index 605145149d..ef89d71c5b 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -37,11 +37,18 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): actions = [RepairScriptAttributes] def process(self, instance): - ctx_data = instance.context.data + root = nuke.root() + knob_data = get_avalon_knob_data(root) project_name = legacy_io.active_project() - asset_name = ctx_data["asset"] - asset = get_asset_by_name(project_name, asset_name) - asset_data = asset["data"] + asset = get_asset_by_name( + project_name, + instance.context.data["asset"] + ) + # get asset data frame values + frame_start = asset["data"]["frameStart"] + frame_end = asset["data"]["frameEnd"] + handle_start = asset["data"]["handleStart"] + handle_end = asset["data"]["handleEnd"] # These attributes will be checked attributes = [ @@ -54,39 +61,36 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): "handleEnd" ] + # get only defined attributes from asset data asset_attributes = { - attr: asset_data[attr] + attr: asset["data"][attr] for attr in attributes - if attr in asset_data + if attr in asset["data"] } + # fix float to max 4 digints (only for evaluating) + fps_data = float("{0:.4f}".format( + asset_attributes["fps"])) + # fix frame values to include handles + asset_attributes.update({ + "frameStart": frame_start - handle_start, + "frameEnd": frame_end + handle_end, + "fps": fps_data + }) self.log.debug(pformat( asset_attributes )) - asset_attributes["fps"] = float("{0:.4f}".format( - asset_attributes["fps"])) - - root = nuke.root() - knob_data = get_avalon_knob_data(root) - - # Get frame range - first_frame = int(root["first_frame"].getValue()) - last_frame = int(root["last_frame"].getValue()) - - handle_start = int(knob_data["handleStart"]) - handle_end = int(knob_data["handleEnd"]) - # Get format _format = root["format"].value() # Get values from nukescript script_attributes = { - "handleStart": handle_start, - "handleEnd": handle_end, + "handleStart": int(knob_data["handleStart"]), + "handleEnd": int(knob_data["handleEnd"]), "fps": float("{0:.4f}".format(root['fps'].value())), - "frameStart": first_frame + handle_start, - "frameEnd": last_frame - handle_end, + "frameStart": int(root["first_frame"].getValue()), + "frameEnd": int(root["last_frame"].getValue()), "resolutionWidth": _format.width(), "resolutionHeight": _format.height(), "pixelAspect": _format.pixelAspect() @@ -94,6 +98,7 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): self.log.debug(pformat( script_attributes )) + # Compare asset's values Nukescript X Database not_matching = [] for attr in attributes: @@ -113,46 +118,24 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): # Raise error if not matching if not_matching: - msg = "Attributes '{}' are not set correctly" - # Alert user that handles are set if Frame start/end not match - message = msg.format(", ".join( - [at["name"] for at in not_matching])) + msg = "Following attributes are not set correctly: \n{}" + attrs_wrong_str = "\n".join([ + ( + "`{0}` is set to `{1}`, " + "but should be set to `{2}`" + ).format(at["name"], at["actual"], at["expected"]) + for at in not_matching + ]) + attrs_wrong_html = "
".join([ + ( + "-- __{0}__ is set to __{1}__, " + "but should be set to __{2}__" + ).format(at["name"], at["actual"], at["expected"]) + for at in not_matching + ]) raise PublishXmlValidationError( - self, message, + self, msg.format(attrs_wrong_str), formatting_data={ - "missing_attributes": not_matching + "failed_attributes": attrs_wrong_html } ) - - def check_parent_hierarchical( - self, project_name, parent_type, parent_id, attr - ): - if parent_id is None: - return None - - doc = None - if parent_type == "project": - doc = get_project(project_name) - elif parent_type == "asset": - doc = get_asset_by_id(project_name, parent_id) - - if not doc: - return None - - doc_data = doc["data"] - if attr in doc_data: - self.log.info(attr) - return doc_data[attr] - - if parent_type == "project": - return None - - parent_id = doc_data.get("visualParent") - new_parent_type = "asset" - if parent_id is None: - parent_id = doc["parent"] - new_parent_type = "project" - - return self.check_parent_hierarchical( - project_name, new_parent_type, parent_id, attr - ) From 06f338a95bf33644d84e365d01a3c4f6a68ac344 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 25 Jul 2022 21:47:47 +0200 Subject: [PATCH 24/80] nuke: making validator code nicer --- .../publish/validate_script_attributes.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index ef89d71c5b..d16660f272 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -1,10 +1,10 @@ from pprint import pformat import pyblish.api -from openpype.client import get_project, get_asset_by_id, get_asset_by_name +import openpype.api +from openpype.client import get_asset_by_name from openpype.pipeline import legacy_io from openpype.pipeline import PublishXmlValidationError -from openpype.api import get_errored_instances_from_context from openpype.hosts.nuke.api.lib import ( get_avalon_knob_data, WorkfileSettings @@ -12,19 +12,6 @@ from openpype.hosts.nuke.api.lib import ( import nuke -@pyblish.api.log -class RepairScriptAttributes(pyblish.api.Action): - label = "Repair" - on = "failed" - icon = "wrench" - - def process(self, context, plugin): - instances = get_errored_instances_from_context(context) - - self.log.debug(instances) - WorkfileSettings().set_context_settings() - - @pyblish.api.log class ValidateScriptAttributes(pyblish.api.InstancePlugin): """ Validates file output. """ @@ -34,7 +21,7 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): label = "Validatte script attributes" hosts = ["nuke"] optional = True - actions = [RepairScriptAttributes] + actions = [openpype.api.RepairAction] def process(self, instance): root = nuke.root() @@ -139,3 +126,8 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): "failed_attributes": attrs_wrong_html } ) + + @classmethod + def repair(cls, instance): + cls.log.debug("__ repairing instance: {}".format(instance)) + WorkfileSettings().set_context_settings() From 4a18e50352cda46e3b8de09bd7a40df15ea1384d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 28 Jul 2022 17:42:52 +0200 Subject: [PATCH 25/80] Update openpype/hosts/nuke/plugins/publish/validate_script_attributes.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../nuke/plugins/publish/validate_script_attributes.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index d16660f272..3907f40991 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -26,11 +26,7 @@ class ValidateScriptAttributes(pyblish.api.InstancePlugin): def process(self, instance): root = nuke.root() knob_data = get_avalon_knob_data(root) - project_name = legacy_io.active_project() - asset = get_asset_by_name( - project_name, - instance.context.data["asset"] - ) + asset = instance.data["assetEntity"] # get asset data frame values frame_start = asset["data"]["frameStart"] frame_end = asset["data"]["frameEnd"] From 6a51b4e7891829761b2c49353d17a7fc3423edcc Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 28 Jul 2022 17:49:29 +0200 Subject: [PATCH 26/80] hound suggestions --- .../hosts/nuke/plugins/publish/validate_output_resolution.py | 2 +- .../hosts/nuke/plugins/publish/validate_script_attributes.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py index 710adde069..fc07e9b83b 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py +++ b/openpype/hosts/nuke/plugins/publish/validate_output_resolution.py @@ -89,4 +89,4 @@ class ValidateOutputResolution(pyblish.api.InstancePlugin): if cls.resolution_msg == invalid: reformat = cls.get_reformat(instance) reformat["format"].setValue(nuke.root()["format"].value()) - cls.log.info("I am fixing reformat to root.format") \ No newline at end of file + cls.log.info("I am fixing reformat to root.format") diff --git a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py index 3907f40991..106d7a2524 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_script_attributes.py @@ -2,8 +2,6 @@ from pprint import pformat import pyblish.api import openpype.api -from openpype.client import get_asset_by_name -from openpype.pipeline import legacy_io from openpype.pipeline import PublishXmlValidationError from openpype.hosts.nuke.api.lib import ( get_avalon_knob_data, From cf0ac3f8b56c5d0f63ab6fa00966fcfe6b76ee08 Mon Sep 17 00:00:00 2001 From: Kaa Maurice Date: Thu, 11 Aug 2022 17:10:02 +0200 Subject: [PATCH 27/80] blender ops refresh manager after process events --- openpype/hosts/blender/api/lib.py | 2 +- openpype/hosts/blender/api/ops.py | 17 ++++++++++++----- .../hosts/blender/blender_addon/startup/init.py | 8 +++++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/blender/api/lib.py b/openpype/hosts/blender/api/lib.py index 20098c0fe8..9cd1ace821 100644 --- a/openpype/hosts/blender/api/lib.py +++ b/openpype/hosts/blender/api/lib.py @@ -234,7 +234,7 @@ def lsattrs(attrs: Dict) -> List: def read(node: bpy.types.bpy_struct_meta_idprop): """Return user-defined attributes from `node`""" - data = dict(node.get(pipeline.AVALON_PROPERTY)) + data = dict(node.get(pipeline.AVALON_PROPERTY, {})) # Ignore hidden/internal data data = { diff --git a/openpype/hosts/blender/api/ops.py b/openpype/hosts/blender/api/ops.py index 4f8410da74..e0e09277df 100644 --- a/openpype/hosts/blender/api/ops.py +++ b/openpype/hosts/blender/api/ops.py @@ -26,7 +26,7 @@ PREVIEW_COLLECTIONS: Dict = dict() # This seems like a good value to keep the Qt app responsive and doesn't slow # down Blender. At least on macOS I the interace of Blender gets very laggy if # you make it smaller. -TIMER_INTERVAL: float = 0.01 +TIMER_INTERVAL: float = 0.01 if platform.system() == "Windows" else 0.1 class BlenderApplication(QtWidgets.QApplication): @@ -164,6 +164,12 @@ def _process_app_events() -> Optional[float]: dialog.setDetailedText(detail) dialog.exec_() + # Refresh Manager + if GlobalClass.app: + manager = GlobalClass.app.get_window("WM_OT_avalon_manager") + if manager: + manager.refresh() + if not GlobalClass.is_windows: if OpenFileCacher.opening_file: return TIMER_INTERVAL @@ -192,10 +198,11 @@ class LaunchQtApp(bpy.types.Operator): self._app = BlenderApplication.get_app() GlobalClass.app = self._app - bpy.app.timers.register( - _process_app_events, - persistent=True - ) + if not bpy.app.timers.is_registered(_process_app_events): + bpy.app.timers.register( + _process_app_events, + persistent=True + ) def execute(self, context): """Execute the operator. diff --git a/openpype/hosts/blender/blender_addon/startup/init.py b/openpype/hosts/blender/blender_addon/startup/init.py index 13a4b8a7a1..8dbff8a91d 100644 --- a/openpype/hosts/blender/blender_addon/startup/init.py +++ b/openpype/hosts/blender/blender_addon/startup/init.py @@ -1,4 +1,10 @@ from openpype.pipeline import install_host from openpype.hosts.blender import api -install_host(api) + +def register(): + install_host(api) + + +def unregister(): + pass From dd2becdb7964bf43d71368c89f7a9fdae48ce4a0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 11:55:16 +0200 Subject: [PATCH 28/80] nuke: collect workfile adding KnownPublishErrorl when untitled --- openpype/hosts/nuke/plugins/publish/precollect_workfile.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openpype/hosts/nuke/plugins/publish/precollect_workfile.py b/openpype/hosts/nuke/plugins/publish/precollect_workfile.py index 7349a8f424..822f405a6f 100644 --- a/openpype/hosts/nuke/plugins/publish/precollect_workfile.py +++ b/openpype/hosts/nuke/plugins/publish/precollect_workfile.py @@ -8,6 +8,7 @@ from openpype.hosts.nuke.api.lib import ( add_publish_knob, get_avalon_knob_data ) +from openpype.pipeline import KnownPublishError class CollectWorkfile(pyblish.api.ContextPlugin): @@ -22,6 +23,12 @@ class CollectWorkfile(pyblish.api.ContextPlugin): current_file = os.path.normpath(nuke.root().name()) + if current_file.lower() == "root": + raise KnownPublishError( + "Workfile is not correct file name. \n" + "Use workfile tool to manage the name correctly." + ) + knob_data = get_avalon_knob_data(root) add_publish_knob(root) From 4bd375409e11822978d352cae20fd1081615aa55 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 11:55:37 +0200 Subject: [PATCH 29/80] nuke: fixing validate rendered frames --- openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py b/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py index f8e128cd26..237ff423e5 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py +++ b/openpype/hosts/nuke/plugins/publish/validate_rendered_frames.py @@ -127,7 +127,7 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): f_start_h += 1 if ( - collected_frames_len >= frame_length + collected_frames_len != frame_length and coll_start <= f_start_h and coll_end >= f_end_h ): From bcb15c2fc5b28ff38ea3d5a9beca41186dec1615 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 15 Aug 2022 21:01:30 +0200 Subject: [PATCH 30/80] nuke: validation type mishmash wip --- .../nuke/plugins/publish/validate_write_nodes.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 48dce623a9..9c9b8babaa 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -1,3 +1,4 @@ +import six import pyblish.api from openpype.api import get_errored_instances_from_context from openpype.hosts.nuke.api.lib import ( @@ -72,11 +73,21 @@ class ValidateNukeWriteNode(pyblish.api.InstancePlugin): for knob_data in check_knobs: key = knob_data["name"] value = knob_data["value"] + node_value = write_node[key].value() + + # fix type differences + if type(node_value) in (int, float): + value = float(value) + node_value = float(node_value) + else: + value = str(value) + node_value = str(node_value) + self.log.debug("__ key: {} | value: {}".format( key, value )) if ( - str(write_node[key].value()) != str(value) + node_value != value and key != "file" and key != "tile_color" ): From 46726e5afedc88ef22b31f515e70b08308643acd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 16 Aug 2022 14:31:18 +0200 Subject: [PATCH 31/80] OP-3713 - changed type to tri-state Customer wants to have more granularity, they want to create flatten 'image', but not separate 'image' per layer. --- .../settings/defaults/project_settings/photoshop.json | 2 +- .../projects_schema/schema_project_photoshop.json | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index d9b7a8083f..b08e73f1ee 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -8,7 +8,7 @@ }, "publish": { "CollectColorCodedInstances": { - "create_flatten_image": false, + "create_flatten_image": "no", "flatten_subset_template": "", "color_code_mapping": [] }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index badf94229b..6935ec8e5e 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -45,9 +45,15 @@ "label": "Set color for publishable layers, set its resulting family and template for subset name. \nCan create flatten image from published instances.(Applicable only for remote publishing!)" }, { - "type": "boolean", "key": "create_flatten_image", - "label": "Create flatten image" + "label": "Create flatten image", + "type": "enum", + "multiselection": false, + "enum_items": [ + { "yes": "Yes" }, + { "no": "No" }, + { "only": "Only flatten" } + ] }, { "type": "text", From 9bfc1447b7945cd11d11414df3ffad42f6014292 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 16 Aug 2022 14:32:47 +0200 Subject: [PATCH 32/80] OP-3713 - implement tri-state logic for create_flatten_image Customer wants to have more granularity, they want to create flatten 'image', but not separate 'image' per layer. --- .../publish/collect_color_coded_instances.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py index 71bd2cd854..9adc16d0fd 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py @@ -32,7 +32,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): # TODO check if could be set globally, probably doesn't make sense when # flattened template cannot subset_template_name = "" - create_flatten_image = False + create_flatten_image = "no" # probably not possible to configure this globally flatten_subset_template = "" @@ -98,13 +98,16 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): "Subset {} already created, skipping.".format(subset)) continue - instance = self._create_instance(context, layer, resolved_family, - asset_name, subset, task_name) + if self.create_flatten_image != "only": + instance = self._create_instance(context, layer, + resolved_family, + asset_name, subset, task_name) + created_instances.append(instance) + existing_subset_names.append(subset) publishable_layers.append(layer) - created_instances.append(instance) - if self.create_flatten_image and publishable_layers: + if self.create_flatten_image != "no" and publishable_layers: self.log.debug("create_flatten_image") if not self.flatten_subset_template: self.log.warning("No template for flatten image") @@ -116,7 +119,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): first_layer = publishable_layers[0] # dummy layer first_layer.name = subset - family = created_instances[0].data["family"] # inherit family + family = resolved_family # inherit family instance = self._create_instance(context, first_layer, family, asset_name, subset, task_name) From 141b275fc614aa1456c97bbe16706497524cb0f0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 16 Aug 2022 17:02:37 +0200 Subject: [PATCH 33/80] OP-3713 - fix missing family Resulted in failure in integrate --- .../plugins/publish/collect_color_coded_instances.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py index 9adc16d0fd..7d78140c5b 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py @@ -62,6 +62,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): publishable_layers = [] created_instances = [] + family_from_settings = None for layer in layers: self.log.debug("Layer:: {}".format(layer)) if layer.parents: @@ -80,6 +81,9 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): self.log.debug("!!! Not found family or template, skip") continue + if not family_from_settings: + family_from_settings = resolved_family + fill_pairs = { "variant": variant, "family": resolved_family, @@ -119,7 +123,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): first_layer = publishable_layers[0] # dummy layer first_layer.name = subset - family = resolved_family # inherit family + family = family_from_settings # inherit family instance = self._create_instance(context, first_layer, family, asset_name, subset, task_name) From 45368a7ba83f0bdb59e9b4e591e6a1fee0736fff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 16 Aug 2022 17:15:09 +0200 Subject: [PATCH 34/80] OP-3713 - added more documentation --- .../publish/collect_color_coded_instances.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py index 7d78140c5b..f93ba51574 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py @@ -9,14 +9,22 @@ from openpype.settings import get_project_settings class CollectColorCodedInstances(pyblish.api.ContextPlugin): - """Creates instances for configured color code of a layer. + """Creates instances for layers marked by configurable color. Used in remote publishing when artists marks publishable layers by color- - coding. + coding. Top level layers (group) must be marked by specific color to be + published as an instance of 'image' family. Can add group for all publishable layers to allow creation of flattened image. (Cannot contain special background layer as it cannot be grouped!) + Based on value `create_flatten_image` from Settings: + - "yes": create flattened 'image' subset of all publishable layers + create + 'image' subset per publishable layer + - "only": create ONLY flattened 'image' subset of all publishable layers + - "no": do not create flattened 'image' subset at all, + only separate subsets per marked layer. + Identifier: id (str): "pyblish.avalon.instance" """ @@ -33,7 +41,6 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): # flattened template cannot subset_template_name = "" create_flatten_image = "no" - # probably not possible to configure this globally flatten_subset_template = "" def process(self, context): From 4dbca722bac4a918fa992a0860361460338ef970 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 17 Aug 2022 11:14:29 +0200 Subject: [PATCH 35/80] OP-3713 - refactored keys and labels --- .../plugins/publish/collect_color_coded_instances.py | 2 +- .../schemas/projects_schema/schema_project_photoshop.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py index f93ba51574..c157c932fd 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_color_coded_instances.py @@ -109,7 +109,7 @@ class CollectColorCodedInstances(pyblish.api.ContextPlugin): "Subset {} already created, skipping.".format(subset)) continue - if self.create_flatten_image != "only": + if self.create_flatten_image != "flatten_only": instance = self._create_instance(context, layer, resolved_family, asset_name, subset, task_name) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index 6935ec8e5e..db06147a51 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -50,9 +50,9 @@ "type": "enum", "multiselection": false, "enum_items": [ - { "yes": "Yes" }, - { "no": "No" }, - { "only": "Only flatten" } + { "flatten_with_images": "Flatten with images" }, + { "flatten_only": "Flatten only" }, + { "no": "No" } ] }, { From b178bb538496123fe748d54dc3271ebabc019cfe Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 17 Aug 2022 11:49:50 +0200 Subject: [PATCH 36/80] OP-3722 - added check for empty context --- openpype/pype_commands.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index a447aa916b..b266479bb1 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -232,6 +232,11 @@ class PypeCommands: fail_batch(_id, dbcon, msg) print("Another batch running, probably stuck, ask admin for help") + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + asset_name, task_name, task_type = get_batch_asset_task_info( task_data["context"]) From 309a272a1833ad73badeab5235ece8707c904c33 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 17:27:47 +0200 Subject: [PATCH 37/80] nuke: fixing validate knobs --- openpype/hosts/nuke/plugins/publish/validate_knobs.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_knobs.py b/openpype/hosts/nuke/plugins/publish/validate_knobs.py index 573c25f3fe..e2b11892e5 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_knobs.py +++ b/openpype/hosts/nuke/plugins/publish/validate_knobs.py @@ -65,13 +65,22 @@ class ValidateKnobs(pyblish.api.ContextPlugin): # Filter families. families = [instance.data["family"]] families += instance.data.get("families", []) - families = list(set(families) & set(cls.knobs.keys())) + if not families: continue # Get all knobs to validate. knobs = {} for family in families: + # check if dot in family + if "." in family: + family = family.split(".")[0] + + # avoid families not in settings + if family not in cls.knobs: + continue + + # get presets of knobs for preset in cls.knobs[family]: knobs[preset] = cls.knobs[family][preset] From eb897ac579e0993103cc2d12c82d574181e55754 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 23 Aug 2022 17:28:03 +0200 Subject: [PATCH 38/80] remove unused import --- openpype/hosts/nuke/plugins/publish/validate_write_nodes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py index 9c9b8babaa..362ff31174 100644 --- a/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py +++ b/openpype/hosts/nuke/plugins/publish/validate_write_nodes.py @@ -1,4 +1,3 @@ -import six import pyblish.api from openpype.api import get_errored_instances_from_context from openpype.hosts.nuke.api.lib import ( From deec5e5c936abd1fca49cc586875e0d4f671e761 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 12:30:29 +0200 Subject: [PATCH 39/80] nuke: fixing setting colorspace --- openpype/hosts/nuke/api/lib.py | 38 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/openpype/hosts/nuke/api/lib.py b/openpype/hosts/nuke/api/lib.py index a53d932db1..10ddfca51e 100644 --- a/openpype/hosts/nuke/api/lib.py +++ b/openpype/hosts/nuke/api/lib.py @@ -1945,15 +1945,25 @@ class WorkfileSettings(object): if not write_node: return - # write all knobs to node - for knob in nuke_imageio_writes["knobs"]: - value = knob["value"] - if isinstance(value, six.text_type): - value = str(value) - if str(value).startswith("0x"): - value = int(value, 16) + try: + # write all knobs to node + for knob in nuke_imageio_writes["knobs"]: + value = knob["value"] + if isinstance(value, six.text_type): + value = str(value) + if str(value).startswith("0x"): + value = int(value, 16) - write_node[knob["name"]].setValue(value) + log.debug("knob: {}| value: {}".format( + knob["name"], value + )) + write_node[knob["name"]].setValue(value) + except TypeError: + log.warning( + "Legacy workflow didnt work, switching to current") + + set_node_knobs_from_settings( + write_node, nuke_imageio_writes["knobs"]) def set_reads_colorspace(self, read_clrs_inputs): """ Setting colorspace to Read nodes @@ -2010,12 +2020,14 @@ class WorkfileSettings(object): # get imageio nuke_colorspace = get_nuke_imageio_settings() + log.info("Setting colorspace to workfile...") try: self.set_root_colorspace(nuke_colorspace["workfile"]) except AttributeError: msg = "set_colorspace(): missing `workfile` settings in template" nuke.message(msg) + log.info("Setting colorspace to viewers...") try: self.set_viewers_colorspace(nuke_colorspace["viewer"]) except AttributeError: @@ -2023,24 +2035,18 @@ class WorkfileSettings(object): nuke.message(msg) log.error(msg) + log.info("Setting colorspace to write nodes...") try: self.set_writes_colorspace() except AttributeError as _error: nuke.message(_error) log.error(_error) + log.info("Setting colorspace to read nodes...") read_clrs_inputs = nuke_colorspace["regexInputs"].get("inputs", []) if read_clrs_inputs: self.set_reads_colorspace(read_clrs_inputs) - try: - for key in nuke_colorspace: - log.debug("Preset's colorspace key: {}".format(key)) - except TypeError: - msg = "Nuke is not in templates! Contact your supervisor!" - nuke.message(msg) - log.error(msg) - def reset_frame_range_handles(self): """Set frame range to current asset""" From 3ab974f98c96c823953e349fe499b48c5e4a0a7c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 24 Aug 2022 13:46:19 +0200 Subject: [PATCH 40/80] Removed submodule vendor/configs/OpenColorIO-Configs --- vendor/configs/OpenColorIO-Configs | 1 - 1 file changed, 1 deletion(-) delete mode 160000 vendor/configs/OpenColorIO-Configs diff --git a/vendor/configs/OpenColorIO-Configs b/vendor/configs/OpenColorIO-Configs deleted file mode 160000 index 0bb079c08b..0000000000 --- a/vendor/configs/OpenColorIO-Configs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0bb079c08be410030669cbf5f19ff869b88af953 From 882e00baefda3be849c5b92be897d0b9d27ad3e1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 15:52:47 +0200 Subject: [PATCH 41/80] use Logger instead of PypeLogger --- openpype/client/entities.py | 2 +- openpype/hosts/maya/api/plugin.py | 5 +-- openpype/hosts/tvpaint/worker/worker_job.py | 4 +- .../webserver_service/webpublish_routes.py | 6 +-- .../webserver_service/webserver_cli.py | 4 +- openpype/lib/applications.py | 10 ++--- openpype/lib/execute.py | 4 +- openpype/lib/path_templates.py | 5 --- openpype/lib/remote_publish.py | 4 +- openpype/modules/base.py | 12 ++--- openpype/modules/deadline/deadline_module.py | 4 +- .../ftrack/ftrack_server/ftrack_server.py | 21 ++++----- openpype/modules/log_viewer/tray/models.py | 14 +++--- openpype/modules/royalrender/api.py | 12 +++-- openpype/modules/sync_server/sync_server.py | 5 +-- .../modules/sync_server/sync_server_module.py | 4 +- openpype/modules/sync_server/tray/app.py | 3 -- .../modules/sync_server/tray/delegates.py | 5 +-- openpype/modules/sync_server/tray/lib.py | 5 --- openpype/modules/sync_server/tray/models.py | 5 +-- openpype/modules/sync_server/tray/widgets.py | 5 +-- .../modules/timers_manager/idle_threads.py | 4 +- openpype/modules/webserver/server.py | 45 ++++++++++++------- openpype/pipeline/anatomy.py | 4 +- openpype/pype_commands.py | 24 +++++----- openpype/settings/entities/base_entity.py | 4 +- 26 files changed, 109 insertions(+), 111 deletions(-) diff --git a/openpype/client/entities.py b/openpype/client/entities.py index f1f1d30214..3d2730a17c 100644 --- a/openpype/client/entities.py +++ b/openpype/client/entities.py @@ -1455,7 +1455,7 @@ def get_workfile_info( """ ## Custom data storage: - Settings - OP settings overrides and local settings -- Logging - logs from PypeLogger +- Logging - logs from Logger - Webpublisher - jobs - Ftrack - events - Maya - Shaders diff --git a/openpype/hosts/maya/api/plugin.py b/openpype/hosts/maya/api/plugin.py index e50ebfccad..39d821f620 100644 --- a/openpype/hosts/maya/api/plugin.py +++ b/openpype/hosts/maya/api/plugin.py @@ -4,6 +4,7 @@ from maya import cmds import qargparse +from openpype.lib import Logger from openpype.pipeline import ( LegacyCreator, LoaderPlugin, @@ -50,9 +51,7 @@ def get_reference_node(members, log=None): # Warn the user when we're taking the highest reference node if len(references) > 1: if not log: - from openpype.lib import PypeLogger - - log = PypeLogger().get_logger(__name__) + log = Logger.get_logger(__name__) log.warning("More than one reference node found in " "container, using highest reference node: " diff --git a/openpype/hosts/tvpaint/worker/worker_job.py b/openpype/hosts/tvpaint/worker/worker_job.py index 1c785ab2ee..95c0a678bc 100644 --- a/openpype/hosts/tvpaint/worker/worker_job.py +++ b/openpype/hosts/tvpaint/worker/worker_job.py @@ -9,7 +9,7 @@ from abc import ABCMeta, abstractmethod, abstractproperty import six -from openpype.api import PypeLogger +from openpype.lib import Logger from openpype.modules import ModulesManager @@ -328,7 +328,7 @@ class TVPaintCommands: def log(self): """Access to logger object.""" if self._log is None: - self._log = PypeLogger.get_logger(self.__class__.__name__) + self._log = Logger.get_logger(self.__class__.__name__) return self._log @property diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 6444a5191d..2e9d460a98 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -12,9 +12,7 @@ from openpype.client import ( get_assets, OpenPypeMongoConnection, ) -from openpype.lib import ( - PypeLogger, -) +from openpype.lib import Logger from openpype.lib.remote_publish import ( get_task_data, ERROR_STATUS, @@ -23,7 +21,7 @@ from openpype.lib.remote_publish import ( from openpype.settings import get_project_settings from openpype_modules.webserver.base_routes import RestApiEndpoint -log = PypeLogger.get_logger("WebpublishRoutes") +log = Logger.get_logger("WebpublishRoutes") class ResourceRestApiEndpoint(RestApiEndpoint): diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 6620e5d5cf..936bd9735f 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -7,7 +7,7 @@ import json import subprocess from openpype.client import OpenPypeMongoConnection -from openpype.lib import PypeLogger +from openpype.lib import Logger from .webpublish_routes import ( RestApiResource, @@ -28,7 +28,7 @@ from openpype.lib.remote_publish import ( ) -log = PypeLogger.get_logger("webserver_gui") +log = Logger.get_logger("webserver_gui") def run_webserver(*args, **kwargs): diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 074e815160..73f9607835 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -24,7 +24,7 @@ from openpype.settings.constants import ( METADATA_KEYS, M_DYNAMIC_KEY_LABEL ) -from . import PypeLogger +from .log import Logger from .profiles_filtering import filter_profiles from .local_settings import get_openpype_username @@ -138,7 +138,7 @@ def get_logger(): """Global lib.applications logger getter.""" global _logger if _logger is None: - _logger = PypeLogger.get_logger(__name__) + _logger = Logger.get_logger(__name__) return _logger @@ -373,7 +373,7 @@ class ApplicationManager: """ def __init__(self, system_settings=None): - self.log = PypeLogger.get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) self.app_groups = {} self.applications = {} @@ -735,7 +735,7 @@ class LaunchHook: Always should be called """ - self.log = PypeLogger().get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) self.launch_context = launch_context @@ -877,7 +877,7 @@ class ApplicationLaunchContext: # Logger logger_name = "{}-{}".format(self.__class__.__name__, self.app_name) - self.log = PypeLogger.get_logger(logger_name) + self.log = Logger.get_logger(logger_name) self.executable = executable diff --git a/openpype/lib/execute.py b/openpype/lib/execute.py index c3e35772f3..f1f2a4fa0a 100644 --- a/openpype/lib/execute.py +++ b/openpype/lib/execute.py @@ -5,7 +5,7 @@ import platform import json import tempfile -from .log import PypeLogger as Logger +from .log import Logger from .vendor_bin_utils import find_executable # MSDN process creation flag (Windows only) @@ -40,7 +40,7 @@ def execute(args, log_levels = ['DEBUG:', 'INFO:', 'ERROR:', 'WARNING:', 'CRITICAL:'] - log = Logger().get_logger('execute') + log = Logger.get_logger('execute') log.info("Executing ({})".format(" ".join(args))) popen = subprocess.Popen( args, diff --git a/openpype/lib/path_templates.py b/openpype/lib/path_templates.py index e4b18ec258..b160054e38 100644 --- a/openpype/lib/path_templates.py +++ b/openpype/lib/path_templates.py @@ -6,11 +6,6 @@ import collections import six -from .log import PypeLogger - -log = PypeLogger.get_logger(__name__) - - KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})") KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+") SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)") diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py index b4b05c053b..2a901544cc 100644 --- a/openpype/lib/remote_publish.py +++ b/openpype/lib/remote_publish.py @@ -66,7 +66,7 @@ def publish(log, close_plugin_name=None, raise_error=False): """Loops through all plugins, logs to console. Used for tests. Args: - log (OpenPypeLogger) + log (openpype.lib.Logger) close_plugin_name (str): name of plugin with responsibility to close host app """ @@ -98,7 +98,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): Args: dbcon (OpenPypeMongoConnection) _id (str) - id of current job in DB - log (OpenPypeLogger) + log (openpype.lib.Logger) batch_id (str) - id sent from frontend close_plugin_name (str): name of plugin with responsibility to close host app diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 1316d7f734..7fc848af2d 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -26,7 +26,7 @@ from openpype.settings.lib import ( get_studio_system_settings_overrides, load_json_file ) -from openpype.lib import PypeLogger +from openpype.lib import Logger # Files that will be always ignored on modules import IGNORED_FILENAMES = ( @@ -93,7 +93,7 @@ class _ModuleClass(object): def log(self): if self._log is None: super(_ModuleClass, self).__setattr__( - "_log", PypeLogger.get_logger(self.name) + "_log", Logger.get_logger(self.name) ) return self._log @@ -290,7 +290,7 @@ def _load_modules(): # Change `sys.modules` sys.modules[modules_key] = openpype_modules = _ModuleClass(modules_key) - log = PypeLogger.get_logger("ModulesLoader") + log = Logger.get_logger("ModulesLoader") # Look for OpenPype modules in paths defined with `get_module_dirs` # - dynamically imported OpenPype modules and addons @@ -440,7 +440,7 @@ class OpenPypeModule: def __init__(self, manager, settings): self.manager = manager - self.log = PypeLogger.get_logger(self.name) + self.log = Logger.get_logger(self.name) self.initialize(settings) @@ -1059,7 +1059,7 @@ class TrayModulesManager(ModulesManager): ) def __init__(self): - self.log = PypeLogger.get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) self.modules = [] self.modules_by_id = {} @@ -1235,7 +1235,7 @@ def get_module_settings_defs(): settings_defs = [] - log = PypeLogger.get_logger("ModuleSettingsLoad") + log = Logger.get_logger("ModuleSettingsLoad") for raw_module in openpype_modules: for attr_name in dir(raw_module): diff --git a/openpype/modules/deadline/deadline_module.py b/openpype/modules/deadline/deadline_module.py index c30db75188..bbd0f74e8a 100644 --- a/openpype/modules/deadline/deadline_module.py +++ b/openpype/modules/deadline/deadline_module.py @@ -3,7 +3,7 @@ import requests import six import sys -from openpype.lib import requests_get, PypeLogger +from openpype.lib import requests_get, Logger from openpype.modules import OpenPypeModule from openpype_interfaces import IPluginPaths @@ -58,7 +58,7 @@ class DeadlineModule(OpenPypeModule, IPluginPaths): """ if not log: - log = PypeLogger.get_logger(__name__) + log = Logger.get_logger(__name__) argument = "{}/api/pools?NamesOnly=true".format(webservice) try: diff --git a/openpype/modules/ftrack/ftrack_server/ftrack_server.py b/openpype/modules/ftrack/ftrack_server/ftrack_server.py index 8944591b71..c75b8f7172 100644 --- a/openpype/modules/ftrack/ftrack_server/ftrack_server.py +++ b/openpype/modules/ftrack/ftrack_server/ftrack_server.py @@ -7,12 +7,10 @@ import traceback import ftrack_api from openpype.lib import ( - PypeLogger, + Logger, modules_from_path ) -log = PypeLogger.get_logger(__name__) - """ # Required - Needed for connection to Ftrack FTRACK_SERVER # Ftrack server e.g. "https://myFtrack.ftrackapp.com" @@ -43,10 +41,13 @@ class FtrackServer: server.run_server() .. """ + # set Ftrack logging to Warning only - OPTIONAL ftrack_log = logging.getLogger("ftrack_api") ftrack_log.setLevel(logging.WARNING) + self.log = Logger.get_logger(__name__) + self.stopped = True self.is_running = False @@ -72,7 +73,7 @@ class FtrackServer: # Get all modules with functions modules, crashed = modules_from_path(path) for filepath, exc_info in crashed: - log.warning("Filepath load crashed {}.\n{}".format( + self.log.warning("Filepath load crashed {}.\n{}".format( filepath, traceback.format_exception(*exc_info) )) @@ -87,7 +88,7 @@ class FtrackServer: break if not register_function: - log.warning( + self.log.warning( "\"{}\" - Missing register method".format(filepath) ) continue @@ -97,7 +98,7 @@ class FtrackServer: ) if not register_functions: - log.warning(( + self.log.warning(( "There are no events with `register` function" " in registered paths: \"{}\"" ).format("| ".join(paths))) @@ -106,7 +107,7 @@ class FtrackServer: try: register_func(self.session) except Exception: - log.warning( + self.log.warning( "\"{}\" - register was not successful".format(filepath), exc_info=True ) @@ -141,7 +142,7 @@ class FtrackServer: self.session = session if load_files: if not self.handler_paths: - log.warning(( + self.log.warning(( "Paths to event handlers are not set." " Ftrack server won't launch." )) @@ -151,8 +152,8 @@ class FtrackServer: self.set_files(self.handler_paths) msg = "Registration of event handlers has finished!" - log.info(len(msg) * "*") - log.info(msg) + self.log.info(len(msg) * "*") + self.log.info(msg) # keep event_hub on session running self.session.event_hub.wait() diff --git a/openpype/modules/log_viewer/tray/models.py b/openpype/modules/log_viewer/tray/models.py index aea62c381b..d369ffeb64 100644 --- a/openpype/modules/log_viewer/tray/models.py +++ b/openpype/modules/log_viewer/tray/models.py @@ -1,6 +1,6 @@ import collections from Qt import QtCore, QtGui -from openpype.lib.log import PypeLogger +from openpype.lib import Logger class LogModel(QtGui.QStandardItemModel): @@ -41,14 +41,14 @@ class LogModel(QtGui.QStandardItemModel): self.dbcon = None # Crash if connection is not possible to skip this module - if not PypeLogger.initialized: - PypeLogger.initialize() + if not Logger.initialized: + Logger.initialize() - connection = PypeLogger.get_log_mongo_connection() + connection = Logger.get_log_mongo_connection() if connection: - PypeLogger.bootstrap_mongo_log() - database = connection[PypeLogger.log_database_name] - self.dbcon = database[PypeLogger.log_collection_name] + Logger.bootstrap_mongo_log() + database = connection[Logger.log_database_name] + self.dbcon = database[Logger.log_collection_name] def headerData(self, section, orientation, role): if ( diff --git a/openpype/modules/royalrender/api.py b/openpype/modules/royalrender/api.py index ed9e71f240..de1dba8724 100644 --- a/openpype/modules/royalrender/api.py +++ b/openpype/modules/royalrender/api.py @@ -5,13 +5,10 @@ import os from openpype.settings import get_project_settings from openpype.lib.local_settings import OpenPypeSettingsRegistry -from openpype.lib import PypeLogger, run_subprocess +from openpype.lib import Logger, run_subprocess from .rr_job import RRJob, SubmitFile, SubmitterParameter -log = PypeLogger.get_logger("RoyalRender") - - class Api: _settings = None @@ -19,6 +16,7 @@ class Api: RR_SUBMIT_API = 2 def __init__(self, settings, project=None): + self.log = Logger.get_logger("RoyalRender") self._settings = settings self._initialize_rr(project) @@ -137,7 +135,7 @@ class Api: rr_console += ".exe" args = [rr_console, file] - run_subprocess(" ".join(args), logger=log) + run_subprocess(" ".join(args), logger=self.log) def _submit_using_api(self, file): # type: (SubmitFile) -> None @@ -159,11 +157,11 @@ class Api: rr_server = tcp.getRRServer() if len(rr_server) == 0: - log.info("Got RR IP address {}".format(rr_server)) + self.log.info("Got RR IP address {}".format(rr_server)) # TODO: Port is hardcoded in RR? If not, move it to Settings if not tcp.setServer(rr_server, 7773): - log.error( + self.log.error( "Can not set RR server: {}".format(tcp.errorMessage())) raise RoyalRenderException(tcp.errorMessage()) diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index 97538fcd4e..d01ab1d3a0 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -6,12 +6,11 @@ import concurrent.futures from concurrent.futures._base import CancelledError from .providers import lib -from openpype.lib import PypeLogger +from openpype.lib import Logger from .utils import SyncStatus, ResumableError - -log = PypeLogger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer") async def upload(module, project_name, file, representation, provider_name, diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index c7f9484e55..c72b310907 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -13,7 +13,7 @@ from openpype.settings import ( get_project_settings, get_system_settings, ) -from openpype.lib import PypeLogger, get_local_site_id +from openpype.lib import Logger, get_local_site_id from openpype.pipeline import AvalonMongoDB, Anatomy from openpype.settings.lib import ( get_default_anatomy_settings, @@ -28,7 +28,7 @@ from .utils import time_function, SyncStatus, SiteAlreadyPresentError from openpype.client import get_representations, get_representation_by_id -log = PypeLogger.get_logger("SyncServer") +log = Logger.get_logger("SyncServer") class SyncServerModule(OpenPypeModule, ITrayModule): diff --git a/openpype/modules/sync_server/tray/app.py b/openpype/modules/sync_server/tray/app.py index 96fad6a247..9b9768327e 100644 --- a/openpype/modules/sync_server/tray/app.py +++ b/openpype/modules/sync_server/tray/app.py @@ -2,7 +2,6 @@ from Qt import QtWidgets, QtCore, QtGui from openpype.tools.settings import style -from openpype.lib import PypeLogger from openpype import resources from .widgets import ( @@ -10,8 +9,6 @@ from .widgets import ( SyncRepresentationSummaryWidget ) -log = PypeLogger().get_logger("SyncServer") - class SyncServerWindow(QtWidgets.QDialog): """ diff --git a/openpype/modules/sync_server/tray/delegates.py b/openpype/modules/sync_server/tray/delegates.py index 5ab809a816..988eb40d28 100644 --- a/openpype/modules/sync_server/tray/delegates.py +++ b/openpype/modules/sync_server/tray/delegates.py @@ -1,8 +1,7 @@ import os from Qt import QtCore, QtWidgets, QtGui -from openpype.lib import PypeLogger -from . import lib +from openpype.lib import Logger from openpype.tools.utils.constants import ( LOCAL_PROVIDER_ROLE, @@ -16,7 +15,7 @@ from openpype.tools.utils.constants import ( EDIT_ICON_ROLE ) -log = PypeLogger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer") class PriorityDelegate(QtWidgets.QStyledItemDelegate): diff --git a/openpype/modules/sync_server/tray/lib.py b/openpype/modules/sync_server/tray/lib.py index 87344be634..ff93815639 100644 --- a/openpype/modules/sync_server/tray/lib.py +++ b/openpype/modules/sync_server/tray/lib.py @@ -2,11 +2,6 @@ import attr import abc import six -from openpype.lib import PypeLogger - - -log = PypeLogger().get_logger("SyncServer") - STATUS = { 0: 'In Progress', 1: 'Queued', diff --git a/openpype/modules/sync_server/tray/models.py b/openpype/modules/sync_server/tray/models.py index 629c4cbbf1..d63d046508 100644 --- a/openpype/modules/sync_server/tray/models.py +++ b/openpype/modules/sync_server/tray/models.py @@ -9,8 +9,7 @@ import qtawesome from openpype.tools.utils.delegates import pretty_timestamp -from openpype.lib import PypeLogger -from openpype.api import get_local_site_id +from openpype.lib import Logger, get_local_site_id from openpype.client import get_representation_by_id from . import lib @@ -33,7 +32,7 @@ from openpype.tools.utils.constants import ( ) -log = PypeLogger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer") class _SyncRepresentationModel(QtCore.QAbstractTableModel): diff --git a/openpype/modules/sync_server/tray/widgets.py b/openpype/modules/sync_server/tray/widgets.py index b4ee447ac4..c40aa98f24 100644 --- a/openpype/modules/sync_server/tray/widgets.py +++ b/openpype/modules/sync_server/tray/widgets.py @@ -9,8 +9,7 @@ import qtawesome from openpype.tools.settings import style -from openpype.api import get_local_site_id -from openpype.lib import PypeLogger +from openpype.lib import Logger, get_local_site_id from openpype.tools.utils.delegates import pretty_timestamp @@ -36,7 +35,7 @@ from openpype.tools.utils.constants import ( TRIES_ROLE ) -log = PypeLogger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer") class SyncProjectListWidget(QtWidgets.QWidget): diff --git a/openpype/modules/timers_manager/idle_threads.py b/openpype/modules/timers_manager/idle_threads.py index 9ec27e659b..7242761143 100644 --- a/openpype/modules/timers_manager/idle_threads.py +++ b/openpype/modules/timers_manager/idle_threads.py @@ -2,7 +2,7 @@ import time from Qt import QtCore from pynput import mouse, keyboard -from openpype.lib import PypeLogger +from openpype.lib import Logger class IdleItem: @@ -31,7 +31,7 @@ class IdleManager(QtCore.QThread): def __init__(self): super(IdleManager, self).__init__() - self.log = PypeLogger.get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) self.signal_reset_timer.connect(self._reset_time) self.idle_item = IdleItem() diff --git a/openpype/modules/webserver/server.py b/openpype/modules/webserver/server.py index 82b681f406..44b14acbb6 100644 --- a/openpype/modules/webserver/server.py +++ b/openpype/modules/webserver/server.py @@ -4,16 +4,16 @@ import asyncio from aiohttp import web -from openpype.lib import PypeLogger +from openpype.lib import Logger from .cors_middleware import cors_middleware -log = PypeLogger.get_logger("WebServer") - class WebServerManager: """Manger that care about web server thread.""" def __init__(self, port=None, host=None): + self._log = None + self.port = port or 8079 self.host = host or "localhost" @@ -33,6 +33,12 @@ class WebServerManager: self.webserver_thread = WebServerThread(self) + @property + def log(self): + if self._log is None: + self._log = Logger.get_logger(self.__class__.__name__) + return self._log + @property def url(self): return "http://{}:{}".format(self.host, self.port) @@ -51,12 +57,12 @@ class WebServerManager: if not self.is_running: return try: - log.debug("Stopping Web server") + self.log.debug("Stopping Web server") self.webserver_thread.is_running = False self.webserver_thread.stop() except Exception: - log.warning( + self.log.warning( "Error has happened during Killing Web server", exc_info=True ) @@ -74,7 +80,10 @@ class WebServerManager: class WebServerThread(threading.Thread): """ Listener for requests in thread.""" + def __init__(self, manager): + self._log = None + super(WebServerThread, self).__init__() self.is_running = False @@ -84,6 +93,12 @@ class WebServerThread(threading.Thread): self.site = None self.tasks = [] + @property + def log(self): + if self._log is None: + self._log = Logger.get_logger(self.__class__.__name__) + return self._log + @property def port(self): return self.manager.port @@ -96,13 +111,13 @@ class WebServerThread(threading.Thread): self.is_running = True try: - log.info("Starting WebServer server") + self.log.info("Starting WebServer server") self.loop = asyncio.new_event_loop() # create new loop for thread asyncio.set_event_loop(self.loop) self.loop.run_until_complete(self.start_server()) - log.debug( + self.log.debug( "Running Web server on URL: \"localhost:{}\"".format(self.port) ) @@ -110,7 +125,7 @@ class WebServerThread(threading.Thread): self.loop.run_forever() except Exception: - log.warning( + self.log.warning( "Web Server service has failed", exc_info=True ) finally: @@ -118,7 +133,7 @@ class WebServerThread(threading.Thread): self.is_running = False self.manager.thread_stopped() - log.info("Web server stopped") + self.log.info("Web server stopped") async def start_server(self): """ Starts runner and TCPsite """ @@ -138,17 +153,17 @@ class WebServerThread(threading.Thread): while self.is_running: while self.tasks: task = self.tasks.pop(0) - log.debug("waiting for task {}".format(task)) + self.log.debug("waiting for task {}".format(task)) await task - log.debug("returned value {}".format(task.result)) + self.log.debug("returned value {}".format(task.result)) await asyncio.sleep(0.5) - log.debug("Starting shutdown") + self.log.debug("Starting shutdown") await self.site.stop() - log.debug("Site stopped") + self.log.debug("Site stopped") await self.runner.cleanup() - log.debug("Runner stopped") + self.log.debug("Runner stopped") tasks = [ task for task in asyncio.all_tasks() @@ -156,7 +171,7 @@ class WebServerThread(threading.Thread): ] list(map(lambda task: task.cancel(), tasks)) # cancel all the tasks results = await asyncio.gather(*tasks, return_exceptions=True) - log.debug(f'Finished awaiting cancelled tasks, results: {results}...') + self.log.debug(f'Finished awaiting cancelled tasks, results: {results}...') await self.loop.shutdown_asyncgens() # to really make sure everything else has time to stop await asyncio.sleep(0.07) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index 08db4749b3..cb6e07154b 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -14,9 +14,9 @@ from openpype.lib.path_templates import ( TemplatesDict, FormatObject, ) -from openpype.lib.log import PypeLogger +from openpype.lib.log import Logger -log = PypeLogger.get_logger(__name__) +log = Logger.get_logger(__name__) class ProjectNotSet(Exception): diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 66bf5e9bb4..71fa7fb6c0 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -5,7 +5,6 @@ import sys import json import time -from openpype.lib import PypeLogger from openpype.api import get_app_environments_for_context from openpype.lib.plugin_tools import get_batch_asset_task_info from openpype.lib.remote_publish import ( @@ -27,10 +26,11 @@ class PypeCommands: """ @staticmethod def launch_tray(): - PypeLogger.set_process_name("Tray") - + from openpype.lib import Logger from openpype.tools import tray + Logger.set_process_name("Tray") + tray.main() @staticmethod @@ -47,10 +47,12 @@ class PypeCommands: @staticmethod def add_modules(click_func): """Modules/Addons can add their cli commands dynamically.""" + + from openpype.lib import Logger from openpype.modules import ModulesManager manager = ModulesManager() - log = PypeLogger.get_logger("AddModulesCLI") + log = Logger.get_logger("CLI-AddModules") for module in manager.modules: try: module.cli(click_func) @@ -96,10 +98,10 @@ class PypeCommands: Raises: RuntimeError: When there is no path to process. """ + + from openpype.lib import Logger from openpype.modules import ModulesManager from openpype.pipeline import install_openpype_plugins - - from openpype.api import Logger from openpype.tools.utils.host_tools import show_publish from openpype.tools.utils.lib import qt_app_context @@ -107,7 +109,7 @@ class PypeCommands: import pyblish.api import pyblish.util - log = Logger.get_logger() + log = Logger.get_logger("CLI-publish") install_openpype_plugins() @@ -195,11 +197,12 @@ class PypeCommands: targets (list): Pyblish targets (to choose validator for example) """ + import pyblish.api - from openpype.api import Logger from openpype.lib import ApplicationManager - log = Logger.get_logger() + from openpype.lib import Logger + log = Logger.get_logger("CLI-remotepublishfromapp") log.info("remotepublishphotoshop command") @@ -311,10 +314,11 @@ class PypeCommands: import pyblish.api import pyblish.util + from openpype.lib import Logger from openpype.pipeline import install_host from openpype.hosts.webpublisher import api as webpublisher - log = PypeLogger.get_logger() + log = Logger.get_logger("remotepublish") log.info("remotepublish command") diff --git a/openpype/settings/entities/base_entity.py b/openpype/settings/entities/base_entity.py index 741f13c49b..f28fefdf5a 100644 --- a/openpype/settings/entities/base_entity.py +++ b/openpype/settings/entities/base_entity.py @@ -15,7 +15,7 @@ from .exceptions import ( EntitySchemaError ) -from openpype.lib import PypeLogger +from openpype.lib import Logger @six.add_metaclass(ABCMeta) @@ -478,7 +478,7 @@ class BaseItemEntity(BaseEntity): def log(self): """Auto created logger for debugging or warnings.""" if self._log is None: - self._log = PypeLogger.get_logger(self.__class__.__name__) + self._log = Logger.get_logger(self.__class__.__name__) return self._log @abstractproperty From 3cdfc5b350dd1ccc3e940967f13cdda42c987739 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 15:54:07 +0200 Subject: [PATCH 42/80] use class log attribues instead of global loggers --- openpype/modules/sync_server/sync_server.py | 61 ++++++++-------- .../modules/sync_server/sync_server_module.py | 69 +++++++++++-------- 2 files changed, 73 insertions(+), 57 deletions(-) diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index d01ab1d3a0..8b11055e65 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -10,8 +10,6 @@ from openpype.lib import Logger from .utils import SyncStatus, ResumableError -log = Logger.get_logger("SyncServer") - async def upload(module, project_name, file, representation, provider_name, remote_site_name, tree=None, preset=None): @@ -237,6 +235,7 @@ class SyncServerThread(threading.Thread): Stopped when tray is closed. """ def __init__(self, module): + self.log = Logger.get_logger(self.__class__.__name__) super(SyncServerThread, self).__init__() self.module = module self.loop = None @@ -248,17 +247,17 @@ class SyncServerThread(threading.Thread): self.is_running = True try: - log.info("Starting Sync Server") + self.log.info("Starting Sync Server") self.loop = asyncio.new_event_loop() # create new loop for thread asyncio.set_event_loop(self.loop) self.loop.set_default_executor(self.executor) asyncio.ensure_future(self.check_shutdown(), loop=self.loop) asyncio.ensure_future(self.sync_loop(), loop=self.loop) - log.info("Sync Server Started") + self.log.info("Sync Server Started") self.loop.run_forever() except Exception: - log.warning( + self.log.warning( "Sync Server service has failed", exc_info=True ) finally: @@ -378,8 +377,9 @@ class SyncServerThread(threading.Thread): )) processed_file_path.add(file_path) - log.debug("Sync tasks count {}". - format(len(task_files_to_process))) + self.log.debug("Sync tasks count {}".format( + len(task_files_to_process) + )) files_created = await asyncio.gather( *task_files_to_process, return_exceptions=True) @@ -398,28 +398,31 @@ class SyncServerThread(threading.Thread): error) duration = time.time() - start_time - log.debug("One loop took {:.2f}s".format(duration)) + self.log.debug("One loop took {:.2f}s".format(duration)) delay = self.module.get_loop_delay(project_name) - log.debug("Waiting for {} seconds to new loop".format(delay)) + self.log.debug( + "Waiting for {} seconds to new loop".format(delay) + ) self.timer = asyncio.create_task(self.run_timer(delay)) await asyncio.gather(self.timer) except ConnectionResetError: - log.warning("ConnectionResetError in sync loop, " - "trying next loop", - exc_info=True) + self.log.warning( + "ConnectionResetError in sync loop, trying next loop", + exc_info=True) except CancelledError: # just stopping server pass except ResumableError: - log.warning("ResumableError in sync loop, " - "trying next loop", - exc_info=True) + self.log.warning( + "ResumableError in sync loop, trying next loop", + exc_info=True) except Exception: self.stop() - log.warning("Unhandled except. in sync loop, stopping server", - exc_info=True) + self.log.warning( + "Unhandled except. in sync loop, stopping server", + exc_info=True) def stop(self): """Sets is_running flag to false, 'check_shutdown' shuts server down""" @@ -432,16 +435,17 @@ class SyncServerThread(threading.Thread): while self.is_running: if self.module.long_running_tasks: task = self.module.long_running_tasks.pop() - log.info("starting long running") + self.log.info("starting long running") await self.loop.run_in_executor(None, task["func"]) - log.info("finished long running") + self.log.info("finished long running") self.module.projects_processed.remove(task["project_name"]) await asyncio.sleep(0.5) tasks = [task for task in asyncio.all_tasks() if task is not asyncio.current_task()] list(map(lambda task: task.cancel(), tasks)) # cancel all the tasks results = await asyncio.gather(*tasks, return_exceptions=True) - log.debug(f'Finished awaiting cancelled tasks, results: {results}...') + self.log.debug( + f'Finished awaiting cancelled tasks, results: {results}...') await self.loop.shutdown_asyncgens() # to really make sure everything else has time to stop self.executor.shutdown(wait=True) @@ -454,29 +458,32 @@ class SyncServerThread(threading.Thread): def reset_timer(self): """Called when waiting for next loop should be skipped""" - log.debug("Resetting timer") + self.log.debug("Resetting timer") if self.timer: self.timer.cancel() self.timer = None def _working_sites(self, project_name): if self.module.is_project_paused(project_name): - log.debug("Both sites same, skipping") + self.log.debug("Both sites same, skipping") return None, None local_site = self.module.get_active_site(project_name) remote_site = self.module.get_remote_site(project_name) if local_site == remote_site: - log.debug("{}-{} sites same, skipping".format(local_site, - remote_site)) + self.log.debug("{}-{} sites same, skipping".format( + local_site, remote_site)) return None, None configured_sites = _get_configured_sites(self.module, project_name) if not all([local_site in configured_sites, remote_site in configured_sites]): - log.debug("Some of the sites {} - {} is not ".format(local_site, - remote_site) + - "working properly") + self.log.debug( + "Some of the sites {} - {} is not working properly".format( + local_site, remote_site + ) + ) + return None, None return local_site, remote_site diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index c72b310907..3ef680c5a6 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -462,7 +462,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation_id (string): MongoDB objectId value site_name (string): 'gdrive', 'studio' etc. """ - log.info("Pausing SyncServer for {}".format(representation_id)) + self.log.info("Pausing SyncServer for {}".format(representation_id)) self._paused_representations.add(representation_id) self.reset_site_on_representation(project_name, representation_id, site_name=site_name, pause=True) @@ -479,7 +479,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation_id (string): MongoDB objectId value site_name (string): 'gdrive', 'studio' etc. """ - log.info("Unpausing SyncServer for {}".format(representation_id)) + self.log.info("Unpausing SyncServer for {}".format(representation_id)) try: self._paused_representations.remove(representation_id) except KeyError: @@ -518,7 +518,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Args: project_name (string): project_name name """ - log.info("Pausing SyncServer for {}".format(project_name)) + self.log.info("Pausing SyncServer for {}".format(project_name)) self._paused_projects.add(project_name) def unpause_project(self, project_name): @@ -530,7 +530,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Args: project_name (string): """ - log.info("Unpausing SyncServer for {}".format(project_name)) + self.log.info("Unpausing SyncServer for {}".format(project_name)) try: self._paused_projects.remove(project_name) except KeyError: @@ -558,14 +558,14 @@ class SyncServerModule(OpenPypeModule, ITrayModule): It won't check anything, not uploading/downloading... """ - log.info("Pausing SyncServer") + self.log.info("Pausing SyncServer") self._paused = True def unpause_server(self): """ Unpause server """ - log.info("Unpausing SyncServer") + self.log.info("Unpausing SyncServer") self._paused = False def is_paused(self): @@ -876,7 +876,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): # val = val[platform.system().lower()] # except KeyError: # st = "{}'s field value {} should be".format(key, val) # noqa: E501 - # log.error(st + " multiplatform dict") + # self.log.error(st + " multiplatform dict") # # item["namespace"] = item["namespace"].replace('{site}', # site_name) @@ -1148,7 +1148,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if self.enabled: self.sync_server_thread.start() else: - log.info("No presets or active providers. " + + self.log.info("No presets or active providers. " + "Synchronization not possible.") def tray_exit(self): @@ -1166,12 +1166,12 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if not self.is_running: return try: - log.info("Stopping sync server server") + self.log.info("Stopping sync server server") self.sync_server_thread.is_running = False self.sync_server_thread.stop() - log.info("Sync server stopped") + self.log.info("Sync server stopped") except Exception: - log.warning( + self.log.warning( "Error has happened during Killing sync server", exc_info=True ) @@ -1256,7 +1256,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): sync_project_settings[project_name] = proj_settings if not sync_project_settings: - log.info("No enabled and configured projects for sync.") + self.log.info("No enabled and configured projects for sync.") return sync_project_settings def get_sync_project_setting(self, project_name, exclude_locals=False, @@ -1387,7 +1387,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Returns: (list) of dictionaries """ - log.debug("Check representations for : {}".format(project_name)) + self.log.debug("Check representations for : {}".format(project_name)) self.connection.Session["AVALON_PROJECT"] = project_name # retry_cnt - number of attempts to sync specific file before giving up retries_arr = self._get_retries_arr(project_name) @@ -1466,9 +1466,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule): }}, {"$sort": {'priority': -1, '_id': 1}}, ] - log.debug("active_site:{} - remote_site:{}".format(active_site, - remote_site)) - log.debug("query: {}".format(aggr)) + self.log.debug("active_site:{} - remote_site:{}".format( + active_site, remote_site + )) + self.log.debug("query: {}".format(aggr)) representations = self.connection.aggregate(aggr) return representations @@ -1503,7 +1504,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if get_local_site_id() not in (local_site, remote_site): # don't do upload/download for studio sites - log.debug("No local site {} - {}".format(local_site, remote_site)) + self.log.debug("No local site {} - {}".format(local_site, remote_site)) return SyncStatus.DO_NOTHING _, remote_rec = self._get_site_rec(sites, remote_site) or {} @@ -1594,11 +1595,16 @@ class SyncServerModule(OpenPypeModule, ITrayModule): error_str = '' source_file = file.get("path", "") - log.debug("File for {} - {source_file} process {status} {error_str}". - format(representation_id, - status=status, - source_file=source_file, - error_str=error_str)) + self.log.debug( + ( + "File for {} - {source_file} process {status} {error_str}" + ).format( + representation_id, + status=status, + source_file=source_file, + error_str=error_str + ) + ) def _get_file_info(self, files, _id): """ @@ -1772,7 +1778,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): break if not found: msg = "Site {} not found".format(site_name) - log.info(msg) + self.log.info(msg) raise ValueError(msg) update = { @@ -1799,7 +1805,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): break if not found: msg = "Site {} not found".format(site_name) - log.info(msg) + self.log.info(msg) raise ValueError(msg) if pause: @@ -1834,7 +1840,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): reset_existing = False files = representation.get("files", []) if not files: - log.debug("No files for {}".format(representation_id)) + self.log.debug("No files for {}".format(representation_id)) return for repre_file in files: @@ -1851,7 +1857,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): reset_existing = True else: msg = "Site {} already present".format(site_name) - log.info(msg) + self.log.info(msg) raise SiteAlreadyPresentError(msg) if reset_existing: @@ -1951,16 +1957,19 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.widget = SyncServerWindow(self) no_errors = True except ValueError: - log.info("No system setting for sync. Not syncing.", exc_info=True) + self.log.info( + "No system setting for sync. Not syncing.", exc_info=True + ) except KeyError: - log.info(( + self.log.info(( "There are not set presets for SyncServer OR " "Credentials provided are invalid, " "no syncing possible"). format(str(self.sync_project_settings)), exc_info=True) except: - log.error("Uncaught exception durin start of SyncServer", - exc_info=True) + self.log.error( + "Uncaught exception durin start of SyncServer", + exc_info=True) self.enabled = no_errors self.widget.show() From c352ae5bcc713405794a3234ca38065624b9c119 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 15:57:52 +0200 Subject: [PATCH 43/80] add deprecation warning for PypeLogger and added docstring about deprecation --- openpype/lib/log.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/openpype/lib/log.py b/openpype/lib/log.py index e77edea0e9..26dcd86eec 100644 --- a/openpype/lib/log.py +++ b/openpype/lib/log.py @@ -486,12 +486,18 @@ class Logger: class PypeLogger(Logger): + """Duplicate of 'Logger'. + + Deprecated: + Class will be removed after release version 3.16.* + """ + @classmethod def get_logger(cls, *args, **kwargs): logger = Logger.get_logger(*args, **kwargs) # TODO uncomment when replaced most of places - # logger.warning(( - # "'openpype.lib.PypeLogger' is deprecated class." - # " Please use 'openpype.lib.Logger' instead." - # )) + logger.warning(( + "'openpype.lib.PypeLogger' is deprecated class." + " Please use 'openpype.lib.Logger' instead." + )) return logger From 08efc477caa31e3ee064ce755ff7336322a9bc2b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 16:21:04 +0200 Subject: [PATCH 44/80] small tweaks in usage of Logger --- openpype/hosts/celaction/api/cli.py | 2 +- openpype/hosts/fusion/api/pipeline.py | 4 ++-- .../fusion/utility_scripts/__OpenPype_Menu__.py | 5 ++--- openpype/hosts/hiero/api/events.py | 5 ++--- openpype/hosts/hiero/api/lib.py | 4 ++-- openpype/hosts/hiero/api/pipeline.py | 4 ++-- openpype/hosts/hiero/api/plugin.py | 3 ++- openpype/hosts/nuke/plugins/load/actions.py | 4 ++-- openpype/hosts/nuke/startup/clear_rendered.py | 5 +++-- openpype/hosts/nuke/startup/write_to_read.py | 4 ++-- .../modules/ftrack/ftrack_server/socket_thread.py | 6 +++--- openpype/modules/ftrack/lib/ftrack_base_handler.py | 4 ++-- .../modules/ftrack/scripts/sub_event_processor.py | 11 +++++------ .../modules/ftrack/scripts/sub_legacy_server.py | 4 ++-- openpype/modules/ftrack/scripts/sub_user_server.py | 5 ++--- openpype/modules/ftrack/tray/ftrack_tray.py | 5 +++-- .../sync_server/providers/abstract_provider.py | 4 ++-- openpype/modules/sync_server/utils.py | 6 ++++-- openpype/modules/timers_manager/rest_api.py | 14 +++++++++----- 19 files changed, 52 insertions(+), 47 deletions(-) diff --git a/openpype/hosts/celaction/api/cli.py b/openpype/hosts/celaction/api/cli.py index 8c7b3a2e74..eb91def090 100644 --- a/openpype/hosts/celaction/api/cli.py +++ b/openpype/hosts/celaction/api/cli.py @@ -14,7 +14,7 @@ from openpype.tools.utils import host_tools from openpype.pipeline import install_openpype_plugins -log = Logger().get_logger("Celaction_cli_publisher") +log = Logger.get_logger("Celaction_cli_publisher") publish_host = "celaction" diff --git a/openpype/hosts/fusion/api/pipeline.py b/openpype/hosts/fusion/api/pipeline.py index 54002f9f51..54a6c94b60 100644 --- a/openpype/hosts/fusion/api/pipeline.py +++ b/openpype/hosts/fusion/api/pipeline.py @@ -8,7 +8,7 @@ import contextlib import pyblish.api -from openpype.api import Logger +from openpype.lib import Logger from openpype.pipeline import ( register_loader_plugin_path, register_creator_plugin_path, @@ -20,7 +20,7 @@ from openpype.pipeline import ( ) import openpype.hosts.fusion -log = Logger().get_logger(__name__) +log = Logger.get_logger(__name__) HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.fusion.__file__)) PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") diff --git a/openpype/hosts/fusion/utility_scripts/__OpenPype_Menu__.py b/openpype/hosts/fusion/utility_scripts/__OpenPype_Menu__.py index de8fc4b3b4..870e74280a 100644 --- a/openpype/hosts/fusion/utility_scripts/__OpenPype_Menu__.py +++ b/openpype/hosts/fusion/utility_scripts/__OpenPype_Menu__.py @@ -1,14 +1,12 @@ import os import sys -from openpype.api import Logger +from openpype.lib import Logger from openpype.pipeline import ( install_host, registered_host, ) -log = Logger().get_logger(__name__) - def main(env): from openpype.hosts.fusion import api @@ -17,6 +15,7 @@ def main(env): # activate resolve from pype install_host(api) + log = Logger.get_logger(__name__) log.info(f"Registered host: {registered_host()}") menu.launch_openpype_menu() diff --git a/openpype/hosts/hiero/api/events.py b/openpype/hosts/hiero/api/events.py index 59fd278a81..862a2607c1 100644 --- a/openpype/hosts/hiero/api/events.py +++ b/openpype/hosts/hiero/api/events.py @@ -1,7 +1,6 @@ import os import hiero.core.events -from openpype.api import Logger -from openpype.lib import register_event_callback +from openpype.lib import Logger, register_event_callback from .lib import ( sync_avalon_data_to_workfile, launch_workfiles_app, @@ -11,7 +10,7 @@ from .lib import ( from .tags import add_tags_to_workfile from .menu import update_menu_task_label -log = Logger().get_logger(__name__) +log = Logger.get_logger(__name__) def startupCompleted(event): diff --git a/openpype/hosts/hiero/api/lib.py b/openpype/hosts/hiero/api/lib.py index 2f66f3ddd7..e288cea2b1 100644 --- a/openpype/hosts/hiero/api/lib.py +++ b/openpype/hosts/hiero/api/lib.py @@ -21,7 +21,7 @@ from openpype.client import ( ) from openpype.settings import get_anatomy_settings from openpype.pipeline import legacy_io, Anatomy -from openpype.api import Logger +from openpype.lib import Logger from . import tags try: @@ -34,7 +34,7 @@ except ImportError: # from opentimelineio import opentime # from pprint import pformat -log = Logger().get_logger(__name__) +log = Logger.get_logger(__name__) self = sys.modules[__name__] self._has_been_setup = False diff --git a/openpype/hosts/hiero/api/pipeline.py b/openpype/hosts/hiero/api/pipeline.py index b243a38b06..dacfd338bb 100644 --- a/openpype/hosts/hiero/api/pipeline.py +++ b/openpype/hosts/hiero/api/pipeline.py @@ -6,7 +6,7 @@ import contextlib from collections import OrderedDict from pyblish import api as pyblish -from openpype.api import Logger +from openpype.lib import Logger from openpype.pipeline import ( schema, register_creator_plugin_path, @@ -18,7 +18,7 @@ from openpype.pipeline import ( from openpype.tools.utils import host_tools from . import lib, menu, events -log = Logger().get_logger(__name__) +log = Logger.get_logger(__name__) # plugin paths API_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/openpype/hosts/hiero/api/plugin.py b/openpype/hosts/hiero/api/plugin.py index 28a9dfb492..77fedbbbdc 100644 --- a/openpype/hosts/hiero/api/plugin.py +++ b/openpype/hosts/hiero/api/plugin.py @@ -9,11 +9,12 @@ from Qt import QtWidgets, QtCore import qargparse import openpype.api as openpype +from openpype.lib import Logger from openpype.pipeline import LoaderPlugin, LegacyCreator from openpype.pipeline.context_tools import get_current_project_asset from . import lib -log = openpype.Logger().get_logger(__name__) +log = Logger.get_logger(__name__) def load_stylesheet(): diff --git a/openpype/hosts/nuke/plugins/load/actions.py b/openpype/hosts/nuke/plugins/load/actions.py index d364a4f3a1..69f56c7305 100644 --- a/openpype/hosts/nuke/plugins/load/actions.py +++ b/openpype/hosts/nuke/plugins/load/actions.py @@ -2,10 +2,10 @@ """ -from openpype.api import Logger +from openpype.lib import Logger from openpype.pipeline import load -log = Logger().get_logger(__name__) +log = Logger.get_logger(__name__) class SetFrameRangeLoader(load.LoaderPlugin): diff --git a/openpype/hosts/nuke/startup/clear_rendered.py b/openpype/hosts/nuke/startup/clear_rendered.py index cf1d8ce170..744af71034 100644 --- a/openpype/hosts/nuke/startup/clear_rendered.py +++ b/openpype/hosts/nuke/startup/clear_rendered.py @@ -1,10 +1,11 @@ import os -from openpype.api import Logger -log = Logger().get_logger(__name__) +from openpype.lib import Logger def clear_rendered(dir_path): + log = Logger.get_logger(__name__) + for _f in os.listdir(dir_path): _f_path = os.path.join(dir_path, _f) log.info("Removing: `{}`".format(_f_path)) diff --git a/openpype/hosts/nuke/startup/write_to_read.py b/openpype/hosts/nuke/startup/write_to_read.py index f5cf66b357..b7add40f47 100644 --- a/openpype/hosts/nuke/startup/write_to_read.py +++ b/openpype/hosts/nuke/startup/write_to_read.py @@ -2,8 +2,8 @@ import re import os import glob import nuke -from openpype.api import Logger -log = Logger().get_logger(__name__) +from openpype.lib import Logger +log = Logger.get_logger(__name__) SINGLE_FILE_FORMATS = ['avi', 'mp4', 'mxf', 'mov', 'mpg', 'mpeg', 'wmv', 'm4v', 'm2v'] diff --git a/openpype/modules/ftrack/ftrack_server/socket_thread.py b/openpype/modules/ftrack/ftrack_server/socket_thread.py index f49ca5557e..3ef55f8daa 100644 --- a/openpype/modules/ftrack/ftrack_server/socket_thread.py +++ b/openpype/modules/ftrack/ftrack_server/socket_thread.py @@ -5,8 +5,8 @@ import socket import threading import traceback import subprocess -from openpype.api import Logger -from openpype.lib import get_openpype_execute_args + +from openpype.lib import get_openpype_execute_args, Logger class SocketThread(threading.Thread): @@ -16,7 +16,7 @@ class SocketThread(threading.Thread): def __init__(self, name, port, filepath, additional_args=[]): super(SocketThread, self).__init__() - self.log = Logger().get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) self.setName(name) self.name = name self.port = port diff --git a/openpype/modules/ftrack/lib/ftrack_base_handler.py b/openpype/modules/ftrack/lib/ftrack_base_handler.py index c0fad6aadc..c0b03f8a41 100644 --- a/openpype/modules/ftrack/lib/ftrack_base_handler.py +++ b/openpype/modules/ftrack/lib/ftrack_base_handler.py @@ -6,7 +6,7 @@ import uuid import datetime import traceback import time -from openpype.api import Logger +from openpype.lib import Logger from openpype.settings import get_project_settings import ftrack_api @@ -52,7 +52,7 @@ class BaseHandler(object): def __init__(self, session): '''Expects a ftrack_api.Session instance''' - self.log = Logger().get_logger(self.__class__.__name__) + self.log = Logger.get_logger(self.__class__.__name__) if not( isinstance(session, ftrack_api.session.Session) or isinstance(session, ftrack_server.lib.SocketSession) diff --git a/openpype/modules/ftrack/scripts/sub_event_processor.py b/openpype/modules/ftrack/scripts/sub_event_processor.py index d1e2e3aaeb..a5ce0511b8 100644 --- a/openpype/modules/ftrack/scripts/sub_event_processor.py +++ b/openpype/modules/ftrack/scripts/sub_event_processor.py @@ -4,6 +4,8 @@ import signal import socket import datetime +import ftrack_api + from openpype_modules.ftrack.ftrack_server.ftrack_server import FtrackServer from openpype_modules.ftrack.ftrack_server.lib import ( SocketSession, @@ -12,17 +14,12 @@ from openpype_modules.ftrack.ftrack_server.lib import ( ) from openpype.modules import ModulesManager -from openpype.api import Logger from openpype.lib import ( + Logger, get_openpype_version, get_build_version ) - -import ftrack_api - -log = Logger().get_logger("Event processor") - subprocess_started = datetime.datetime.now() @@ -68,6 +65,8 @@ def register(session): def main(args): + log = Logger.get_logger("Event processor") + port = int(args[-1]) # Create a TCP/IP socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/openpype/modules/ftrack/scripts/sub_legacy_server.py b/openpype/modules/ftrack/scripts/sub_legacy_server.py index e3a623c376..1f0fc1b369 100644 --- a/openpype/modules/ftrack/scripts/sub_legacy_server.py +++ b/openpype/modules/ftrack/scripts/sub_legacy_server.py @@ -5,11 +5,11 @@ import signal import threading import ftrack_api -from openpype.api import Logger +from openpype.lib import Logger from openpype.modules import ModulesManager from openpype_modules.ftrack.ftrack_server.ftrack_server import FtrackServer -log = Logger().get_logger("Event Server Legacy") +log = Logger.get_logger("Event Server Legacy") class TimerChecker(threading.Thread): diff --git a/openpype/modules/ftrack/scripts/sub_user_server.py b/openpype/modules/ftrack/scripts/sub_user_server.py index a3701a0950..930a2d51e2 100644 --- a/openpype/modules/ftrack/scripts/sub_user_server.py +++ b/openpype/modules/ftrack/scripts/sub_user_server.py @@ -2,6 +2,7 @@ import sys import signal import socket +from openpype.lib import Logger from openpype_modules.ftrack.ftrack_server.ftrack_server import FtrackServer from openpype_modules.ftrack.ftrack_server.lib import ( SocketSession, @@ -9,9 +10,7 @@ from openpype_modules.ftrack.ftrack_server.lib import ( ) from openpype.modules import ModulesManager -from openpype.api import Logger - -log = Logger().get_logger("FtrackUserServer") +log = Logger.get_logger("FtrackUserServer") def main(args): diff --git a/openpype/modules/ftrack/tray/ftrack_tray.py b/openpype/modules/ftrack/tray/ftrack_tray.py index 2919ae22fb..501d837a4c 100644 --- a/openpype/modules/ftrack/tray/ftrack_tray.py +++ b/openpype/modules/ftrack/tray/ftrack_tray.py @@ -12,10 +12,11 @@ from ..lib import credentials from ..ftrack_module import FTRACK_MODULE_DIR from . import login_dialog -from openpype.api import Logger, resources +from openpype import resources +from openpype.lib import Logger -log = Logger().get_logger("FtrackModule") +log = Logger.get_logger("FtrackModule") class FtrackTrayWrapper: diff --git a/openpype/modules/sync_server/providers/abstract_provider.py b/openpype/modules/sync_server/providers/abstract_provider.py index 8c2fe1cad9..9c808dc80e 100644 --- a/openpype/modules/sync_server/providers/abstract_provider.py +++ b/openpype/modules/sync_server/providers/abstract_provider.py @@ -1,8 +1,8 @@ import abc import six -from openpype.api import Logger +from openpype.lib import Logger -log = Logger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer") @six.add_metaclass(abc.ABCMeta) diff --git a/openpype/modules/sync_server/utils.py b/openpype/modules/sync_server/utils.py index 03f362202f..4caa01e9d7 100644 --- a/openpype/modules/sync_server/utils.py +++ b/openpype/modules/sync_server/utils.py @@ -1,6 +1,8 @@ import time -from openpype.api import Logger -log = Logger().get_logger("SyncServer") + +from openpype.lib import Logger + +log = Logger.get_logger("SyncServer") class ResumableError(Exception): diff --git a/openpype/modules/timers_manager/rest_api.py b/openpype/modules/timers_manager/rest_api.py index f16cb316c3..9bde19aec9 100644 --- a/openpype/modules/timers_manager/rest_api.py +++ b/openpype/modules/timers_manager/rest_api.py @@ -1,9 +1,7 @@ import json from aiohttp.web_response import Response -from openpype.api import Logger - -log = Logger().get_logger("Event processor") +from openpype.lib import Logger class TimersManagerModuleRestApi: @@ -12,6 +10,7 @@ class TimersManagerModuleRestApi: happens in Workfile app. """ def __init__(self, user_module, server_manager): + self.log = None self.module = user_module self.server_manager = server_manager @@ -19,6 +18,11 @@ class TimersManagerModuleRestApi: self.register() + def log(self): + if self._log is None: + self._log = Logger.get_logger(self.__ckass__.__name__) + return self._log + def register(self): self.server_manager.add_route( "POST", @@ -47,7 +51,7 @@ class TimersManagerModuleRestApi: "Payload must contain fields 'project_name," " 'asset_name' and 'task_name'" ) - log.error(msg) + self.log.error(msg) return Response(status=400, message=msg) self.module.stop_timers() @@ -73,7 +77,7 @@ class TimersManagerModuleRestApi: "Payload must contain fields 'project_name, 'asset_name'," " 'task_name'" ) - log.warning(message) + self.log.warning(message) return Response(text=message, status=404) time = self.module.get_task_time(project_name, asset_name, task_name) From 840fbaa38086b110f9ad72584aa3e37fa1bf1178 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 16:21:17 +0200 Subject: [PATCH 45/80] cleanup imports in modules --- openpype/modules/base.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7fc848af2d..8ac4e7ddac 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -13,7 +13,6 @@ from uuid import uuid4 from abc import ABCMeta, abstractmethod import six -import openpype from openpype.settings import ( get_system_settings, SYSTEM_SETTINGS_KEY, @@ -26,7 +25,12 @@ from openpype.settings.lib import ( get_studio_system_settings_overrides, load_json_file ) -from openpype.lib import Logger + +from openpype.lib import ( + Logger, + import_filepath, + import_module_from_dirpath +) # Files that will be always ignored on modules import IGNORED_FILENAMES = ( @@ -278,12 +282,6 @@ def load_modules(force=False): def _load_modules(): - # Import helper functions from lib - from openpype.lib import ( - import_filepath, - import_module_from_dirpath - ) - # Key under which will be modules imported in `sys.modules` modules_key = "openpype_modules" From bf8e2207e07dd18f1fc8e2d8026ec719886666ba Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 16:58:38 +0200 Subject: [PATCH 46/80] fix property --- openpype/modules/timers_manager/rest_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/modules/timers_manager/rest_api.py b/openpype/modules/timers_manager/rest_api.py index 9bde19aec9..6686407350 100644 --- a/openpype/modules/timers_manager/rest_api.py +++ b/openpype/modules/timers_manager/rest_api.py @@ -18,6 +18,7 @@ class TimersManagerModuleRestApi: self.register() + @property def log(self): if self._log is None: self._log = Logger.get_logger(self.__ckass__.__name__) From 310e9bf50f59a3f39adf54d18047eb7a422c5843 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 24 Aug 2022 17:19:01 +0200 Subject: [PATCH 47/80] fix line lengths --- openpype/modules/sync_server/sync_server_module.py | 4 +++- openpype/modules/webserver/server.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 3ef680c5a6..634b68c55f 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -1504,7 +1504,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if get_local_site_id() not in (local_site, remote_site): # don't do upload/download for studio sites - self.log.debug("No local site {} - {}".format(local_site, remote_site)) + self.log.debug( + "No local site {} - {}".format(local_site, remote_site) + ) return SyncStatus.DO_NOTHING _, remote_rec = self._get_site_rec(sites, remote_site) or {} diff --git a/openpype/modules/webserver/server.py b/openpype/modules/webserver/server.py index 44b14acbb6..120925a362 100644 --- a/openpype/modules/webserver/server.py +++ b/openpype/modules/webserver/server.py @@ -171,7 +171,9 @@ class WebServerThread(threading.Thread): ] list(map(lambda task: task.cancel(), tasks)) # cancel all the tasks results = await asyncio.gather(*tasks, return_exceptions=True) - self.log.debug(f'Finished awaiting cancelled tasks, results: {results}...') + self.log.debug( + f'Finished awaiting cancelled tasks, results: {results}...' + ) await self.loop.shutdown_asyncgens() # to really make sure everything else has time to stop await asyncio.sleep(0.07) From 2f2cbd41465c882519ef6831e5ad81373f7a3615 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Aug 2022 11:24:53 +0200 Subject: [PATCH 48/80] OP-3722 - added check for empty context to basic publish --- openpype/pype_commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index b266479bb1..391244d185 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -230,7 +230,6 @@ class PypeCommands: format("\n".join(running_batches)) msg += "Ask admin to check them and reprocess current batch" fail_batch(_id, dbcon, msg) - print("Another batch running, probably stuck, ask admin for help") if not task_data["context"]: msg = "Batch manifest must contain context data" @@ -351,6 +350,12 @@ class PypeCommands: dbcon = get_webpublish_conn() _id = start_webpublish_log(dbcon, batch_id, user_email) + task_data = get_task_data(batch_path) + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + publish_and_log(dbcon, _id, log, batch_id=batch_id) log.info("Publish finished.") From 5372c016eadfe2dd09cfc1803a6984cfca24d61b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 12:19:47 +0200 Subject: [PATCH 49/80] moved 'OpenPypeInterface' into interfaces.py --- openpype/modules/__init__.py | 2 -- openpype/modules/base.py | 42 +++++++--------------------------- openpype/modules/interfaces.py | 29 +++++++++++++++++++++-- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/openpype/modules/__init__.py b/openpype/modules/__init__.py index 68b5f6c247..02e7dc13ab 100644 --- a/openpype/modules/__init__.py +++ b/openpype/modules/__init__.py @@ -2,7 +2,6 @@ from .base import ( OpenPypeModule, OpenPypeAddOn, - OpenPypeInterface, load_modules, @@ -20,7 +19,6 @@ from .base import ( __all__ = ( "OpenPypeModule", "OpenPypeAddOn", - "OpenPypeInterface", "load_modules", diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 1316d7f734..1b8cf5d769 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -28,6 +28,14 @@ from openpype.settings.lib import ( ) from openpype.lib import PypeLogger +from .interfaces import ( + OpenPypeInterface, + IPluginPaths, + IHostModule, + ITrayModule, + ITrayService +) + # Files that will be always ignored on modules import IGNORED_FILENAMES = ( "__pycache__", @@ -391,29 +399,7 @@ def _load_modules(): log.error(msg, exc_info=True) -class _OpenPypeInterfaceMeta(ABCMeta): - """OpenPypeInterface meta class to print proper string.""" - def __str__(self): - return "<'OpenPypeInterface.{}'>".format(self.__name__) - - def __repr__(self): - return str(self) - - -@six.add_metaclass(_OpenPypeInterfaceMeta) -class OpenPypeInterface: - """Base class of Interface that can be used as Mixin with abstract parts. - - This is way how OpenPype module or addon can tell that has implementation - for specific part or for other module/addon. - - Child classes of OpenPypeInterface may be used as mixin in different - OpenPype modules which means they have to have implemented methods defined - in the interface. By default interface does not have any abstract parts. - """ - - pass @six.add_metaclass(ABCMeta) @@ -749,8 +735,6 @@ class ModulesManager: and "actions" each containing list of paths. """ # Output structure - from openpype_interfaces import IPluginPaths - output = { "publish": [], "create": [], @@ -807,8 +791,6 @@ class ModulesManager: list: List of creator plugin paths. """ # Output structure - from openpype_interfaces import IPluginPaths - output = [] for module in self.get_enabled_modules(): # Skip module that do not inherit from `IPluginPaths` @@ -897,8 +879,6 @@ class ModulesManager: host name set to passed 'host_name'. """ - from openpype_interfaces import IHostModule - for module in self.get_enabled_modules(): if ( isinstance(module, IHostModule) @@ -915,8 +895,6 @@ class ModulesManager: inheriting 'IHostModule'. """ - from openpype_interfaces import IHostModule - host_names = { module.host_name for module in self.get_enabled_modules() @@ -1098,8 +1076,6 @@ class TrayModulesManager(ModulesManager): self.tray_menu(tray_menu) def get_enabled_tray_modules(self): - from openpype_interfaces import ITrayModule - output = [] for module in self.modules: if module.enabled and isinstance(module, ITrayModule): @@ -1175,8 +1151,6 @@ class TrayModulesManager(ModulesManager): self._report["Tray menu"] = report def start_modules(self): - from openpype_interfaces import ITrayService - report = {} time_start = time.time() prev_start_time = time_start diff --git a/openpype/modules/interfaces.py b/openpype/modules/interfaces.py index 14f49204ee..8221db4d05 100644 --- a/openpype/modules/interfaces.py +++ b/openpype/modules/interfaces.py @@ -1,8 +1,33 @@ -from abc import abstractmethod, abstractproperty +from abc import ABCMeta, abstractmethod, abstractproperty + +import six from openpype import resources -from openpype.modules import OpenPypeInterface + +class _OpenPypeInterfaceMeta(ABCMeta): + """OpenPypeInterface meta class to print proper string.""" + + def __str__(self): + return "<'OpenPypeInterface.{}'>".format(self.__name__) + + def __repr__(self): + return str(self) + + +@six.add_metaclass(_OpenPypeInterfaceMeta) +class OpenPypeInterface: + """Base class of Interface that can be used as Mixin with abstract parts. + + This is way how OpenPype module or addon can tell OpenPype that contain + implementation for specific functionality. + + Child classes of OpenPypeInterface may be used as mixin in different + OpenPype modules which means they have to have implemented methods defined + in the interface. By default interface does not have any abstract parts. + """ + + pass class IPluginPaths(OpenPypeInterface): From 8cc6086e92a6c5135428898717e6d9057f567e8c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 12:22:19 +0200 Subject: [PATCH 50/80] removed usage of 'ILaunchHookPaths' --- openpype/modules/ftrack/ftrack_module.py | 5 ++--- openpype/modules/shotgrid/shotgrid_module.py | 5 +---- openpype/modules/slack/slack_module.py | 10 ++++------ openpype/modules/timers_manager/timers_manager.py | 11 ++++------- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/openpype/modules/ftrack/ftrack_module.py b/openpype/modules/ftrack/ftrack_module.py index f99e189082..cb4f204523 100644 --- a/openpype/modules/ftrack/ftrack_module.py +++ b/openpype/modules/ftrack/ftrack_module.py @@ -9,7 +9,6 @@ from openpype.modules import OpenPypeModule from openpype_interfaces import ( ITrayModule, IPluginPaths, - ILaunchHookPaths, ISettingsChangeListener ) from openpype.settings import SaveWarningExc @@ -21,7 +20,6 @@ class FtrackModule( OpenPypeModule, ITrayModule, IPluginPaths, - ILaunchHookPaths, ISettingsChangeListener ): name = "ftrack" @@ -85,7 +83,8 @@ class FtrackModule( } def get_launch_hook_paths(self): - """Implementation of `ILaunchHookPaths`.""" + """Implementation for applications launch hooks.""" + return os.path.join(FTRACK_MODULE_DIR, "launch_hooks") def modify_application_launch_arguments(self, application, env): diff --git a/openpype/modules/shotgrid/shotgrid_module.py b/openpype/modules/shotgrid/shotgrid_module.py index 5644f0c35f..281c6fdcad 100644 --- a/openpype/modules/shotgrid/shotgrid_module.py +++ b/openpype/modules/shotgrid/shotgrid_module.py @@ -3,7 +3,6 @@ import os from openpype_interfaces import ( ITrayModule, IPluginPaths, - ILaunchHookPaths, ) from openpype.modules import OpenPypeModule @@ -11,9 +10,7 @@ from openpype.modules import OpenPypeModule SHOTGRID_MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) -class ShotgridModule( - OpenPypeModule, ITrayModule, IPluginPaths, ILaunchHookPaths -): +class ShotgridModule(OpenPypeModule, ITrayModule, IPluginPaths): leecher_manager_url = None name = "shotgrid" enabled = False diff --git a/openpype/modules/slack/slack_module.py b/openpype/modules/slack/slack_module.py index 9b2976d766..499c1c19ce 100644 --- a/openpype/modules/slack/slack_module.py +++ b/openpype/modules/slack/slack_module.py @@ -1,14 +1,11 @@ import os from openpype.modules import OpenPypeModule -from openpype_interfaces import ( - IPluginPaths, - ILaunchHookPaths -) +from openpype.modules.interfaces import IPluginPaths SLACK_MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) -class SlackIntegrationModule(OpenPypeModule, IPluginPaths, ILaunchHookPaths): +class SlackIntegrationModule(OpenPypeModule, IPluginPaths): """Allows sending notification to Slack channels during publishing.""" name = "slack" @@ -18,7 +15,8 @@ class SlackIntegrationModule(OpenPypeModule, IPluginPaths, ILaunchHookPaths): self.enabled = slack_settings["enabled"] def get_launch_hook_paths(self): - """Implementation of `ILaunchHookPaths`.""" + """Implementation for applications launch hooks.""" + return os.path.join(SLACK_MODULE_DIR, "launch_hooks") def get_plugin_paths(self): diff --git a/openpype/modules/timers_manager/timers_manager.py b/openpype/modules/timers_manager/timers_manager.py index 93332ace4f..c168e9534d 100644 --- a/openpype/modules/timers_manager/timers_manager.py +++ b/openpype/modules/timers_manager/timers_manager.py @@ -6,7 +6,6 @@ from openpype.client import get_asset_by_name from openpype.modules import OpenPypeModule from openpype_interfaces import ( ITrayService, - ILaunchHookPaths, IPluginPaths ) from openpype.lib.events import register_event_callback @@ -79,7 +78,6 @@ class ExampleTimersManagerConnector: class TimersManager( OpenPypeModule, ITrayService, - ILaunchHookPaths, IPluginPaths ): """ Handles about Timers. @@ -185,12 +183,11 @@ class TimersManager( ) def get_launch_hook_paths(self): - """Implementation of `ILaunchHookPaths`.""" + """Implementation for applications launch hooks.""" - return os.path.join( - TIMER_MODULE_DIR, - "launch_hooks" - ) + return [ + os.path.join(TIMER_MODULE_DIR, "launch_hooks") + ] def get_plugin_paths(self): """Implementation of `IPluginPaths`.""" From 8acb96c572a58c779b544dac8b79aab9d71b6663 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 12:27:02 +0200 Subject: [PATCH 51/80] added deprecation warning to 'ILaunchHookPaths' --- openpype/modules/interfaces.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openpype/modules/interfaces.py b/openpype/modules/interfaces.py index 8221db4d05..13655773dd 100644 --- a/openpype/modules/interfaces.py +++ b/openpype/modules/interfaces.py @@ -81,6 +81,13 @@ class ILaunchHookPaths(OpenPypeInterface): Expected result is list of paths. ["path/to/launch_hooks_dir"] + + Deprecated: + This interface is not needed since OpenPype 3.14.*. Addon just have to + implement 'get_launch_hook_paths' which can expect Application object + or nothing as argument. + + Interface class will be removed after 3.16.*. """ @abstractmethod From 7356fc666d7aa4cca7251849f4eaf4d91dfc0e75 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 12:31:47 +0200 Subject: [PATCH 52/80] moved collection of launch hooks from modules into applications logic --- openpype/lib/applications.py | 61 +++++++++++++++++++++++++++++++++-- openpype/modules/base.py | 62 ------------------------------------ 2 files changed, 58 insertions(+), 65 deletions(-) diff --git a/openpype/lib/applications.py b/openpype/lib/applications.py index 074e815160..b389bc2539 100644 --- a/openpype/lib/applications.py +++ b/openpype/lib/applications.py @@ -950,6 +950,63 @@ class ApplicationLaunchContext: ) self.kwargs["env"] = value + def _collect_addons_launch_hook_paths(self): + """Helper to collect application launch hooks from addons. + + Module have to have implemented 'get_launch_hook_paths' method which + can expect appliction as argument or nothing. + + Returns: + List[str]: Paths to launch hook directories. + """ + + expected_types = (list, tuple, set) + + output = [] + for module in self.modules_manager.get_enabled_modules(): + # Skip module if does not have implemented 'get_launch_hook_paths' + func = getattr(module, "get_launch_hook_paths", None) + if func is None: + continue + + func = module.get_launch_hook_paths + if hasattr(inspect, "signature"): + sig = inspect.signature(func) + expect_args = len(sig.parameters) > 0 + else: + expect_args = len(inspect.getargspec(func)[0]) > 0 + + # Pass application argument if method expect it. + try: + if expect_args: + hook_paths = func(self.application) + else: + hook_paths = func() + except Exception: + self.log.warning( + "Failed to call 'get_launch_hook_paths'", + exc_info=True + ) + continue + + if not hook_paths: + continue + + # Convert string to list + if isinstance(hook_paths, six.string_types): + hook_paths = [hook_paths] + + # Skip invalid types + if not isinstance(hook_paths, expected_types): + self.log.warning(( + "Result of `get_launch_hook_paths`" + " has invalid type {}. Expected {}" + ).format(type(hook_paths), expected_types)) + continue + + output.extend(hook_paths) + return output + def paths_to_launch_hooks(self): """Directory paths where to look for launch hooks.""" # This method has potential to be part of application manager (maybe). @@ -983,9 +1040,7 @@ class ApplicationLaunchContext: paths.append(path) # Load modules paths - paths.extend( - self.modules_manager.collect_launch_hook_paths(self.application) - ) + paths.extend(self._collect_addons_launch_hook_paths()) return paths diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 1b8cf5d769..25355cbd9c 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -805,68 +805,6 @@ class ModulesManager: output.extend(paths) return output - def collect_launch_hook_paths(self, app): - """Helper to collect application launch hooks. - - It used to be based on 'ILaunchHookPaths' which is not true anymore. - Module just have to have implemented 'get_launch_hook_paths' method. - - Args: - app (Application): Application object which can be used for - filtering of which launch hook paths are returned. - - Returns: - list: Paths to launch hook directories. - """ - - str_type = type("") - expected_types = (list, tuple, set) - - output = [] - for module in self.get_enabled_modules(): - # Skip module if does not have implemented 'get_launch_hook_paths' - func = getattr(module, "get_launch_hook_paths", None) - if func is None: - continue - - func = module.get_launch_hook_paths - if hasattr(inspect, "signature"): - sig = inspect.signature(func) - expect_args = len(sig.parameters) > 0 - else: - expect_args = len(inspect.getargspec(func)[0]) > 0 - - # Pass application argument if method expect it. - try: - if expect_args: - hook_paths = func(app) - else: - hook_paths = func() - except Exception: - self.log.warning( - "Failed to call 'get_launch_hook_paths'", - exc_info=True - ) - continue - - if not hook_paths: - continue - - # Convert string to list - if isinstance(hook_paths, str_type): - hook_paths = [hook_paths] - - # Skip invalid types - if not isinstance(hook_paths, expected_types): - self.log.warning(( - "Result of `get_launch_hook_paths`" - " has invalid type {}. Expected {}" - ).format(type(hook_paths), expected_types)) - continue - - output.extend(hook_paths) - return output - def get_host_module(self, host_name): """Find host module by host name. From c1268ec253a1c9b4ee7e3d8dc5acb76712a8a035 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:55:09 +0200 Subject: [PATCH 53/80] implemented hamrony addon --- openpype/hosts/harmony/__init__.py | 17 ++++++++--------- openpype/hosts/harmony/addon.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 openpype/hosts/harmony/addon.py diff --git a/openpype/hosts/harmony/__init__.py b/openpype/hosts/harmony/__init__.py index d2f710d83d..9177eaa285 100644 --- a/openpype/hosts/harmony/__init__.py +++ b/openpype/hosts/harmony/__init__.py @@ -1,11 +1,10 @@ -import os +from .addon import ( + HARMONY_HOST_DIR, + HarmonyAddon, +) -def add_implementation_envs(env, _app): - """Modify environments to contain all required for implementation.""" - openharmony_path = os.path.join( - os.environ["OPENPYPE_REPOS_ROOT"], "openpype", "hosts", - "harmony", "vendor", "OpenHarmony" - ) - # TODO check if is already set? What to do if is already set? - env["LIB_OPENHARMONY_PATH"] = openharmony_path +__all__ = ( + "HARMONY_HOST_DIR", + "HarmonyAddon", +) diff --git a/openpype/hosts/harmony/addon.py b/openpype/hosts/harmony/addon.py new file mode 100644 index 0000000000..b051d68abb --- /dev/null +++ b/openpype/hosts/harmony/addon.py @@ -0,0 +1,24 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +HARMONY_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class HarmonyAddon(OpenPypeModule, IHostModule): + name = "harmony" + host_name = "harmony" + + def initialize(self, module_settings): + self.enabled = True + + def add_implementation_envs(self, env, _app): + """Modify environments to contain all required for implementation.""" + openharmony_path = os.path.join( + HARMONY_HOST_DIR, "vendor", "OpenHarmony" + ) + # TODO check if is already set? What to do if is already set? + env["LIB_OPENHARMONY_PATH"] = openharmony_path + + def get_workfile_extensions(self): + return [".zip"] From a5ce719e58eadaa9c936086e7fdbd37f4eaba7fb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:55:19 +0200 Subject: [PATCH 54/80] removed usage of HOST_WORKFILE_EXTENSIONS --- openpype/hosts/harmony/api/workio.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openpype/hosts/harmony/api/workio.py b/openpype/hosts/harmony/api/workio.py index ab1cb9b1a9..8df5ede917 100644 --- a/openpype/hosts/harmony/api/workio.py +++ b/openpype/hosts/harmony/api/workio.py @@ -2,8 +2,6 @@ import os import shutil -from openpype.pipeline import HOST_WORKFILE_EXTENSIONS - from .lib import ( ProcessContext, get_local_harmony_path, @@ -16,7 +14,7 @@ save_disabled = False def file_extensions(): - return HOST_WORKFILE_EXTENSIONS["harmony"] + return [".zip"] def has_unsaved_changes(): From bdedea41d67bd72ecca676c066d24a83120b3215 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 15:55:30 +0200 Subject: [PATCH 55/80] reuse 'HARMONY_HOST_DIR' from public api --- openpype/hosts/harmony/api/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/harmony/api/pipeline.py b/openpype/hosts/harmony/api/pipeline.py index 4d71b9380d..4b9849c190 100644 --- a/openpype/hosts/harmony/api/pipeline.py +++ b/openpype/hosts/harmony/api/pipeline.py @@ -14,14 +14,14 @@ from openpype.pipeline import ( ) from openpype.pipeline.load import get_outdated_containers from openpype.pipeline.context_tools import get_current_project_asset -import openpype.hosts.harmony + +from openpype.hosts.harmony import HARMONY_HOST_DIR import openpype.hosts.harmony.api as harmony log = logging.getLogger("openpype.hosts.harmony") -HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.harmony.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PLUGINS_DIR = os.path.join(HARMONY_HOST_DIR, "plugins") PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") LOAD_PATH = os.path.join(PLUGINS_DIR, "load") CREATE_PATH = os.path.join(PLUGINS_DIR, "create") From 1810d757856093726e37b9f4eee8eb50ebfb5934 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:23:53 +0200 Subject: [PATCH 56/80] implemented photoshop addon --- openpype/hosts/photoshop/__init__.py | 19 ++++++++++--------- openpype/hosts/photoshop/addon.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 openpype/hosts/photoshop/addon.py diff --git a/openpype/hosts/photoshop/__init__.py b/openpype/hosts/photoshop/__init__.py index a91e0a65ff..b3f66ea35c 100644 --- a/openpype/hosts/photoshop/__init__.py +++ b/openpype/hosts/photoshop/__init__.py @@ -1,9 +1,10 @@ -def add_implementation_envs(env, _app): - """Modify environments to contain all required for implementation.""" - defaults = { - "OPENPYPE_LOG_NO_COLORS": "True", - "WEBSOCKET_URL": "ws://localhost:8099/ws/" - } - for key, value in defaults.items(): - if not env.get(key): - env[key] = value +from .module import ( + PhotoshopAddon, + PHOTOSHOP_HOST_DIR, +) + + +__all__ = ( + "PhotoshopAddon", + "PHOTOSHOP_HOST_DIR", +) diff --git a/openpype/hosts/photoshop/addon.py b/openpype/hosts/photoshop/addon.py new file mode 100644 index 0000000000..18899d4de8 --- /dev/null +++ b/openpype/hosts/photoshop/addon.py @@ -0,0 +1,26 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +PHOTOSHOP_HOST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class PhotoshopAddon(OpenPypeModule, IHostModule): + name = "photoshop" + host_name = "photoshop" + + def initialize(self, module_settings): + self.enabled = True + + def add_implementation_envs(self, env, _app): + """Modify environments to contain all required for implementation.""" + defaults = { + "OPENPYPE_LOG_NO_COLORS": "True", + "WEBSOCKET_URL": "ws://localhost:8099/ws/" + } + for key, value in defaults.items(): + if not env.get(key): + env[key] = value + + def get_workfile_extensions(self): + return [".psd", ".psb"] From 355edb24f55cc9dc9ff9cc9c7d9c6aaf612efadb Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:24:04 +0200 Subject: [PATCH 57/80] reuse PHOTOSHOP_HOST_DIR from public api --- openpype/hosts/photoshop/api/pipeline.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/photoshop/api/pipeline.py b/openpype/hosts/photoshop/api/pipeline.py index ee150d1808..f660096630 100644 --- a/openpype/hosts/photoshop/api/pipeline.py +++ b/openpype/hosts/photoshop/api/pipeline.py @@ -14,14 +14,13 @@ from openpype.pipeline import ( AVALON_CONTAINER_ID, ) from openpype.pipeline.load import any_outdated_containers -import openpype.hosts.photoshop +from openpype.hosts.photoshop import PHOTOSHOP_HOST_DIR from . import lib log = Logger.get_logger(__name__) -HOST_DIR = os.path.dirname(os.path.abspath(openpype.hosts.photoshop.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") +PLUGINS_DIR = os.path.join(PHOTOSHOP_HOST_DIR, "plugins") PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") LOAD_PATH = os.path.join(PLUGINS_DIR, "load") CREATE_PATH = os.path.join(PLUGINS_DIR, "create") From f0af027faaa4a1c015ee91de0cb98a5c22152e05 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:24:12 +0200 Subject: [PATCH 58/80] removed usage of HOST_WORKFILE_EXTENSIONS --- openpype/hosts/photoshop/api/workio.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/photoshop/api/workio.py b/openpype/hosts/photoshop/api/workio.py index 951c5dbfff..35b44d6070 100644 --- a/openpype/hosts/photoshop/api/workio.py +++ b/openpype/hosts/photoshop/api/workio.py @@ -1,7 +1,6 @@ """Host API required Work Files tool""" import os -from openpype.pipeline import HOST_WORKFILE_EXTENSIONS from . import lib @@ -14,7 +13,7 @@ def _active_document(): def file_extensions(): - return HOST_WORKFILE_EXTENSIONS["photoshop"] + return [".psd", ".psb"] def has_unsaved_changes(): From caf9e014bdb8150ac2abcc8dd23cbe5cb88ab09d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:32:46 +0200 Subject: [PATCH 59/80] implemented webpublish addon --- openpype/hosts/webpublisher/__init__.py | 10 ++++++++++ openpype/hosts/webpublisher/addon.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 openpype/hosts/webpublisher/addon.py diff --git a/openpype/hosts/webpublisher/__init__.py b/openpype/hosts/webpublisher/__init__.py index e69de29bb2..4e918c5d7d 100644 --- a/openpype/hosts/webpublisher/__init__.py +++ b/openpype/hosts/webpublisher/__init__.py @@ -0,0 +1,10 @@ +from .addon import ( + WebpublisherAddon, + WEBPUBLISHER_ROOT_DIR, +) + + +__all__ = ( + "WebpublisherAddon", + "WEBPUBLISHER_ROOT_DIR", +) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py new file mode 100644 index 0000000000..3d76115df1 --- /dev/null +++ b/openpype/hosts/webpublisher/addon.py @@ -0,0 +1,13 @@ +import os +from openpype.modules import OpenPypeModule +from openpype.modules.interfaces import IHostModule + +WEBPUBLISHER_ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class WebpublisherAddon(OpenPypeModule, IHostModule): + name = "webpublisher" + host_name = "webpublisher" + + def initialize(self, module_settings): + self.enabled = True From b188afe97569a7141f6a3c3e14dc7966b1e3b853 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:39:19 +0200 Subject: [PATCH 60/80] reorganized imports in pype commands --- openpype/pype_commands.py | 43 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 67b0b8ad76..cb84fac3c7 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -5,19 +5,6 @@ import sys import json import time -from openpype.api import get_app_environments_for_context -from openpype.lib.plugin_tools import get_batch_asset_task_info -from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - publish_and_log, - fail_batch, - find_variant_key, - get_task_data, - get_timeout, - IN_PROGRESS_STATUS -) - class PypeCommands: """Class implementing commands used by Pype. @@ -100,6 +87,7 @@ class PypeCommands: """ from openpype.lib import Logger + from openpype.lib.applications import get_app_environments_for_context from openpype.modules import ModulesManager from openpype.pipeline import install_openpype_plugins from openpype.tools.utils.host_tools import show_publish @@ -199,9 +187,23 @@ class PypeCommands: """ import pyblish.api - from openpype.lib import ApplicationManager from openpype.lib import Logger + from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, + ) + from openpype.lib.plugin_tools import get_batch_asset_task_info + from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + fail_batch, + find_variant_key, + get_task_data, + get_timeout, + IN_PROGRESS_STATUS + ) + log = Logger.get_logger("CLI-remotepublishfromapp") log.info("remotepublishphotoshop command") @@ -318,9 +320,16 @@ class PypeCommands: import pyblish.api import pyblish.util - from openpype.lib import Logger from openpype.pipeline import install_host from openpype.hosts.webpublisher import api as webpublisher + from openpype.lib import Logger + from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + publish_and_log, + fail_batch, + get_task_data, + ) log = Logger.get_logger("remotepublish") @@ -366,8 +375,10 @@ class PypeCommands: Called by Deadline plugin to propagate environment into render jobs. """ + + from openpype.lib.applications import get_app_environments_for_context + if all((project, asset, task, app)): - from openpype.api import get_app_environments_for_context env = get_app_environments_for_context( project, asset, task, app, env_group ) From c2332507f49eb863aecd98540b9b71b421c2f1ec Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:41:56 +0200 Subject: [PATCH 61/80] implement webpublisher host with HostBase --- openpype/hosts/webpublisher/api/__init__.py | 36 ++++++++------------- openpype/pype_commands.py | 11 +++---- 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/openpype/hosts/webpublisher/api/__init__.py b/openpype/hosts/webpublisher/api/__init__.py index 18e3a16cf5..afea838e2c 100644 --- a/openpype/hosts/webpublisher/api/__init__.py +++ b/openpype/hosts/webpublisher/api/__init__.py @@ -1,31 +1,23 @@ import os import logging -from pyblish import api as pyblish -import openpype.hosts.webpublisher -from openpype.pipeline import legacy_io +import pyblish.api + +from openpype.host import HostBase +from openpype.hosts.webpublisher import WEBPUBLISHER_ROOT_DIR log = logging.getLogger("openpype.hosts.webpublisher") -HOST_DIR = os.path.dirname(os.path.abspath( - openpype.hosts.webpublisher.__file__)) -PLUGINS_DIR = os.path.join(HOST_DIR, "plugins") -PUBLISH_PATH = os.path.join(PLUGINS_DIR, "publish") +class WebpublisherHost(HostBase): + name = "webpublisher" -def install(): - print("Installing Pype config...") + def install(self): + print("Installing Pype config...") + pyblish.api.register_host(self.name) - pyblish.register_plugin_path(PUBLISH_PATH) - log.info(PUBLISH_PATH) - - legacy_io.install() - - -def uninstall(): - pyblish.deregister_plugin_path(PUBLISH_PATH) - - -# to have required methods for interface -def ls(): - pass + publish_plugin_dir = os.path.join( + WEBPUBLISHER_ROOT_DIR, "plugins", "publish" + ) + pyblish.api.register_plugin_path(publish_plugin_dir) + self.log.info(publish_plugin_dir) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index cb84fac3c7..6a65b78dfc 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -321,7 +321,7 @@ class PypeCommands: import pyblish.util from openpype.pipeline import install_host - from openpype.hosts.webpublisher import api as webpublisher + from openpype.hosts.webpublisher.api import WebpublisherHost from openpype.lib import Logger from openpype.lib.remote_publish import ( get_webpublish_conn, @@ -335,22 +335,21 @@ class PypeCommands: log.info("remotepublish command") - host_name = "webpublisher" + webpublisher_host = WebpublisherHost() + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = host_name + os.environ["AVALON_APP"] = webpublisher_host.name os.environ["USER_EMAIL"] = user_email os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - pyblish.api.register_host(host_name) - if targets: if isinstance(targets, str): targets = [targets] for target in targets: pyblish.api.register_target(target) - install_host(webpublisher) + install_host(webpublisher_host) log.info("Running publish ...") From 61690d84774268b49904789f0ea5cd8e5171caf7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:54:42 +0200 Subject: [PATCH 62/80] omved cli functions into webpublisher --- openpype/hosts/webpublisher/addon.py | 50 +++++ openpype/hosts/webpublisher/cli_functions.py | 204 +++++++++++++++++++ openpype/pype_commands.py | 159 +-------------- 3 files changed, 259 insertions(+), 154 deletions(-) create mode 100644 openpype/hosts/webpublisher/cli_functions.py diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 3d76115df1..1a4370c9a5 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -1,4 +1,7 @@ import os + +import click + from openpype.modules import OpenPypeModule from openpype.modules.interfaces import IHostModule @@ -11,3 +14,50 @@ class WebpublisherAddon(OpenPypeModule, IHostModule): def initialize(self, module_settings): self.enabled = True + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group( + WebpublisherAddon.name, + help="Webpublisher related commands.") +def cli_main(): + pass + + +@cli_main.command() +@click.argument("path") +@click.option("-u", "--user", help="User email address") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets", default=None, + multiple=True) +def publish(project, path, user=None, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + + from .cli_functions import publish + + publish(project, path, user, targets) + + +@cli_main.command() +@click.argument("path") +@click.option("-h", "--host", help="Host") +@click.option("-u", "--user", help="User email address") +@click.option("-p", "--project", help="Project") +@click.option("-t", "--targets", help="Targets", default=None, + multiple=True) +def publishfromapp(project, path, user=None, targets=None): + """Start CLI publishing. + + Publish collects json from paths provided as an argument. + More than one path is allowed. + """ + + from .cli_functions import publish_from_app + + publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py new file mode 100644 index 0000000000..cb2e59fac2 --- /dev/null +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -0,0 +1,204 @@ +import os +import time +import pyblish.api +import pyblish.util + +from openpype.lib import Logger +from openpype.lib.remote_publish import ( + get_webpublish_conn, + start_webpublish_log, + publish_and_log, + fail_batch, + find_variant_key, + get_task_data, + get_timeout, + IN_PROGRESS_STATUS +) +from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, +) +from openpype.lib.plugin_tools import get_batch_asset_task_info +from openpype.pipeline import install_host +from openpype.hosts.webpublisher.api import WebpublisherHost + + +def publish(project_name, batch_path, user_email, targets): + """Start headless publishing. + + Used to publish rendered assets, workfiles etc via Webpublisher. + Eventually should be yanked out to Webpublisher cli. + + Publish use json from passed paths argument. + + Args: + project_name (str): project to publish (only single context is + expected per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) + user_email (string): email address for webpublisher - used to + find Ftrack user with same email + targets (list): Pyblish targets + (to choose validator for example) + + Raises: + RuntimeError: When there is no path to process. + """ + + if not batch_path: + raise RuntimeError("No publish paths specified") + + log = Logger.get_logger("remotepublish") + log.info("remotepublish command") + + # Register target and host + webpublisher_host = WebpublisherHost() + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + os.environ["AVALON_PROJECT"] = project_name + os.environ["AVALON_APP"] = webpublisher_host.name + os.environ["USER_EMAIL"] = user_email + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib + + if targets: + if isinstance(targets, str): + targets = [targets] + for target in targets: + pyblish.api.register_target(target) + + install_host(webpublisher_host) + + log.info("Running publish ...") + + _, batch_id = os.path.split(batch_path) + dbcon = get_webpublish_conn() + _id = start_webpublish_log(dbcon, batch_id, user_email) + + task_data = get_task_data(batch_path) + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + + publish_and_log(dbcon, _id, log, batch_id=batch_id) + + log.info("Publish finished.") + + +def publish_from_app( + project_name, batch_path, host_name, user_email, targets +): + """Opens installed variant of 'host' and run remote publish there. + + Eventually should be yanked out to Webpublisher cli. + + Currently implemented and tested for Photoshop where customer + wants to process uploaded .psd file and publish collected layers + from there. Triggered by Webpublisher. + + Checks if no other batches are running (status =='in_progress). If + so, it sleeps for SLEEP (this is separate process), + waits for WAIT_FOR seconds altogether. + + Requires installed host application on the machine. + + Runs publish process as user would, in automatic fashion. + + Args: + project_name (str): project to publish (only single context is + expected per call of remotepublish + batch_path (str): Path batch folder. Contains subfolders with + resources (workfile, another subfolder 'renders' etc.) + host_name (str): 'photoshop' + user_email (string): email address for webpublisher - used to + find Ftrack user with same email + targets (list): Pyblish targets + (to choose validator for example) + """ + + log = Logger.get_logger("RemotePublishFromApp") + + log.info("remotepublishphotoshop command") + + task_data = get_task_data(batch_path) + + workfile_path = os.path.join(batch_path, + task_data["task"], + task_data["files"][0]) + + print("workfile_path {}".format(workfile_path)) + + batch_id = task_data["batch"] + dbcon = get_webpublish_conn() + # safer to start logging here, launch might be broken altogether + _id = start_webpublish_log(dbcon, batch_id, user_email) + + batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS})) + if len(batches_in_progress) > 1: + running_batches = [str(batch["_id"]) + for batch in batches_in_progress + if batch["_id"] != _id] + msg = "There are still running batches {}\n". \ + format("\n".join(running_batches)) + msg += "Ask admin to check them and reprocess current batch" + fail_batch(_id, dbcon, msg) + + if not task_data["context"]: + msg = "Batch manifest must contain context data" + msg += "Create new batch and set context properly." + fail_batch(_id, dbcon, msg) + + asset_name, task_name, task_type = get_batch_asset_task_info( + task_data["context"]) + + application_manager = ApplicationManager() + found_variant_key = find_variant_key(application_manager, host_name) + app_name = "{}/{}".format(host_name, found_variant_key) + + # must have for proper launch of app + env = get_app_environments_for_context( + project_name, + asset_name, + task_name, + app_name + ) + print("env:: {}".format(env)) + os.environ.update(env) + + os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path + # must pass identifier to update log lines for a batch + os.environ["BATCH_LOG_ID"] = str(_id) + os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib + os.environ["USER_EMAIL"] = user_email + + pyblish.api.register_host(host_name) + if targets: + if isinstance(targets, str): + targets = [targets] + current_targets = os.environ.get("PYBLISH_TARGETS", "").split( + os.pathsep) + for target in targets: + current_targets.append(target) + + os.environ["PYBLISH_TARGETS"] = os.pathsep.join( + set(current_targets)) + + data = { + "last_workfile_path": workfile_path, + "start_last_workfile": True, + "project_name": project_name, + "asset_name": asset_name, + "task_name": task_name + } + + launched_app = application_manager.launch(app_name, **data) + + timeout = get_timeout(project_name, host_name, task_type) + + time_start = time.time() + while launched_app.poll() is None: + time.sleep(0.5) + if time.time() - time_start > timeout: + launched_app.terminate() + msg = "Timeout reached" + fail_batch(_id, dbcon, msg) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 6a65b78dfc..1817724df1 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -186,110 +186,11 @@ class PypeCommands: (to choose validator for example) """ - import pyblish.api + from openpype.hosts.webpublisher.cli_functions import publish_from_app - from openpype.lib import Logger - from openpype.lib.applications import ( - ApplicationManager, - get_app_environments_for_context, + publish_from_app( + project_name, batch_path, host_name, user_email, targets ) - from openpype.lib.plugin_tools import get_batch_asset_task_info - from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - fail_batch, - find_variant_key, - get_task_data, - get_timeout, - IN_PROGRESS_STATUS - ) - - log = Logger.get_logger("CLI-remotepublishfromapp") - - log.info("remotepublishphotoshop command") - - task_data = get_task_data(batch_path) - - workfile_path = os.path.join(batch_path, - task_data["task"], - task_data["files"][0]) - - print("workfile_path {}".format(workfile_path)) - - batch_id = task_data["batch"] - dbcon = get_webpublish_conn() - # safer to start logging here, launch might be broken altogether - _id = start_webpublish_log(dbcon, batch_id, user_email) - - batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS})) - if len(batches_in_progress) > 1: - running_batches = [str(batch["_id"]) - for batch in batches_in_progress - if batch["_id"] != _id] - msg = "There are still running batches {}\n". \ - format("\n".join(running_batches)) - msg += "Ask admin to check them and reprocess current batch" - fail_batch(_id, dbcon, msg) - - if not task_data["context"]: - msg = "Batch manifest must contain context data" - msg += "Create new batch and set context properly." - fail_batch(_id, dbcon, msg) - - asset_name, task_name, task_type = get_batch_asset_task_info( - task_data["context"]) - - application_manager = ApplicationManager() - found_variant_key = find_variant_key(application_manager, host_name) - app_name = "{}/{}".format(host_name, found_variant_key) - - # must have for proper launch of app - env = get_app_environments_for_context( - project_name, - asset_name, - task_name, - app_name - ) - print("env:: {}".format(env)) - os.environ.update(env) - - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - # must pass identifier to update log lines for a batch - os.environ["BATCH_LOG_ID"] = str(_id) - os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - os.environ["USER_EMAIL"] = user_email - - pyblish.api.register_host(host_name) - if targets: - if isinstance(targets, str): - targets = [targets] - current_targets = os.environ.get("PYBLISH_TARGETS", "").split( - os.pathsep) - for target in targets: - current_targets.append(target) - - os.environ["PYBLISH_TARGETS"] = os.pathsep.join( - set(current_targets)) - - data = { - "last_workfile_path": workfile_path, - "start_last_workfile": True, - "project_name": project_name, - "asset_name": asset_name, - "task_name": task_name - } - - launched_app = application_manager.launch(app_name, **data) - - timeout = get_timeout(project_name, host_name, task_type) - - time_start = time.time() - while launched_app.poll() is None: - time.sleep(0.5) - if time.time() - time_start > timeout: - launched_app.terminate() - msg = "Timeout reached" - fail_batch(_id, dbcon, msg) @staticmethod def remotepublish(project, batch_path, user_email, targets=None): @@ -313,59 +214,10 @@ class PypeCommands: Raises: RuntimeError: When there is no path to process. """ - if not batch_path: - raise RuntimeError("No publish paths specified") - # Register target and host - import pyblish.api - import pyblish.util + from openpype.hosts.webpublisher.cli_functions import publish - from openpype.pipeline import install_host - from openpype.hosts.webpublisher.api import WebpublisherHost - from openpype.lib import Logger - from openpype.lib.remote_publish import ( - get_webpublish_conn, - start_webpublish_log, - publish_and_log, - fail_batch, - get_task_data, - ) - - log = Logger.get_logger("remotepublish") - - log.info("remotepublish command") - - webpublisher_host = WebpublisherHost() - - os.environ["OPENPYPE_PUBLISH_DATA"] = batch_path - os.environ["AVALON_PROJECT"] = project - os.environ["AVALON_APP"] = webpublisher_host.name - os.environ["USER_EMAIL"] = user_email - os.environ["HEADLESS_PUBLISH"] = 'true' # to use in app lib - - if targets: - if isinstance(targets, str): - targets = [targets] - for target in targets: - pyblish.api.register_target(target) - - install_host(webpublisher_host) - - log.info("Running publish ...") - - _, batch_id = os.path.split(batch_path) - dbcon = get_webpublish_conn() - _id = start_webpublish_log(dbcon, batch_id, user_email) - - task_data = get_task_data(batch_path) - if not task_data["context"]: - msg = "Batch manifest must contain context data" - msg += "Create new batch and set context properly." - fail_batch(_id, dbcon, msg) - - publish_and_log(dbcon, _id, log, batch_id=batch_id) - - log.info("Publish finished.") + publish(project, batch_path, user_email, targets) @staticmethod def extractenvironments(output_json_path, project, asset, task, app, @@ -479,7 +331,6 @@ class PypeCommands: sync_server_module.server_init() sync_server_module.server_start() - import time while True: time.sleep(1.0) From eed9789287adeb1ba000262544368344be353ff9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 16:59:20 +0200 Subject: [PATCH 63/80] changed function names --- openpype/hosts/webpublisher/addon.py | 8 ++++---- openpype/hosts/webpublisher/cli_functions.py | 4 ++-- openpype/pype_commands.py | 12 ++++++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 1a4370c9a5..9e63030fe2 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -39,9 +39,9 @@ def publish(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import publish + from .cli_functions import cli_publish - publish(project, path, user, targets) + cli_publish(project, path, user, targets) @cli_main.command() @@ -58,6 +58,6 @@ def publishfromapp(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import publish_from_app + from .cli_functions import cli_publish_from_app - publish_from_app(project, path, user, targets) + cli_publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py index cb2e59fac2..ad3bb596fb 100644 --- a/openpype/hosts/webpublisher/cli_functions.py +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -23,7 +23,7 @@ from openpype.pipeline import install_host from openpype.hosts.webpublisher.api import WebpublisherHost -def publish(project_name, batch_path, user_email, targets): +def cli_publish(project_name, batch_path, user_email, targets): """Start headless publishing. Used to publish rendered assets, workfiles etc via Webpublisher. @@ -85,7 +85,7 @@ def publish(project_name, batch_path, user_email, targets): log.info("Publish finished.") -def publish_from_app( +def cli_publish_from_app( project_name, batch_path, host_name, user_email, targets ): """Opens installed variant of 'host' and run remote publish there. diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 1817724df1..b6c1228ade 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -186,9 +186,11 @@ class PypeCommands: (to choose validator for example) """ - from openpype.hosts.webpublisher.cli_functions import publish_from_app + from openpype.hosts.webpublisher.cli_functions import ( + cli_publish_from_app + ) - publish_from_app( + cli_publish_from_app( project_name, batch_path, host_name, user_email, targets ) @@ -215,9 +217,11 @@ class PypeCommands: RuntimeError: When there is no path to process. """ - from openpype.hosts.webpublisher.cli_functions import publish + from openpype.hosts.webpublisher.cli_functions import ( + cli_publish + ) - publish(project, batch_path, user_email, targets) + cli_publish(project, batch_path, user_email, targets) @staticmethod def extractenvironments(output_json_path, project, asset, task, app, From dec6335ece01b37e6f0396807526ab16b5db1a6e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:41:18 +0200 Subject: [PATCH 64/80] move remote_publish funtion into pipeline publish --- openpype/pipeline/publish/lib.py | 40 ++++++++++++++++++++++++++++++ openpype/scripts/remote_publish.py | 7 +++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/openpype/pipeline/publish/lib.py b/openpype/pipeline/publish/lib.py index d5494cd8a4..9060a0bf4b 100644 --- a/openpype/pipeline/publish/lib.py +++ b/openpype/pipeline/publish/lib.py @@ -273,3 +273,43 @@ def filter_pyblish_plugins(plugins): option, value, plugin.__name__)) setattr(plugin, option, value) + + +def find_close_plugin(close_plugin_name, log): + if close_plugin_name: + plugins = pyblish.api.discover() + for plugin in plugins: + if plugin.__name__ == close_plugin_name: + return plugin + + log.debug("Close plugin not found, app might not close.") + + +def remote_publish(log, close_plugin_name=None, raise_error=False): + """Loops through all plugins, logs to console. Used for tests. + + Args: + log (openpype.lib.Logger) + close_plugin_name (str): name of plugin with responsibility to + close host app + """ + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" + + close_plugin = find_close_plugin(close_plugin_name, log) + + for result in pyblish.util.publish_iter(): + for record in result["records"]: + log.info("{}: {}".format( + result["plugin"].label, record.msg)) + + if result["error"]: + error_message = error_format.format(**result) + log.error(error_message) + if close_plugin: # close host app explicitly after error + context = pyblish.api.Context() + close_plugin().process(context) + if raise_error: + # Fatal Error is because of Deadline + error_message = "Fatal Error: " + error_format.format(**result) + raise RuntimeError(error_message) diff --git a/openpype/scripts/remote_publish.py b/openpype/scripts/remote_publish.py index d322f369d1..37df35e36c 100644 --- a/openpype/scripts/remote_publish.py +++ b/openpype/scripts/remote_publish.py @@ -1,11 +1,12 @@ try: - from openpype.api import Logger - import openpype.lib.remote_publish + from openpype.lib import Logger + from openpype.pipeline.publish.lib import remote_publish except ImportError as exc: # Ensure Deadline fails by output an error that contains "Fatal Error:" raise ImportError("Fatal Error: %s" % exc) + if __name__ == "__main__": # Perform remote publish with thorough error checking log = Logger.get_logger(__name__) - openpype.lib.remote_publish.publish(log, raise_error=True) + remote_publish(log, raise_error=True) From c1a7b9aff5024c4df92493169e2741f044558b2d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:49:24 +0200 Subject: [PATCH 65/80] moved webpublisher specific functions into webpublisher --- openpype/hosts/webpublisher/lib.py | 278 +++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 openpype/hosts/webpublisher/lib.py diff --git a/openpype/hosts/webpublisher/lib.py b/openpype/hosts/webpublisher/lib.py new file mode 100644 index 0000000000..dde875c934 --- /dev/null +++ b/openpype/hosts/webpublisher/lib.py @@ -0,0 +1,278 @@ +import os +from datetime import datetime +import collections +import json + +from bson.objectid import ObjectId + +import pyblish.util +import pyblish.api + +from openpype.client.mongo import OpenPypeMongoConnection +from openpype.settings import get_project_settings +from openpype.lib import Logger +from openpype.lib.profiles_filtering import filter_profiles + +ERROR_STATUS = "error" +IN_PROGRESS_STATUS = "in_progress" +REPROCESS_STATUS = "reprocess" +SENT_REPROCESSING_STATUS = "sent_for_reprocessing" +FINISHED_REPROCESS_STATUS = "republishing_finished" +FINISHED_OK_STATUS = "finished_ok" + +log = Logger.get_logger(__name__) + + +def parse_json(path): + """Parses json file at 'path' location + + Returns: + (dict) or None if unparsable + Raises: + AsssertionError if 'path' doesn't exist + """ + path = path.strip('\"') + assert os.path.isfile(path), ( + "Path to json file doesn't exist. \"{}\"".format(path) + ) + data = None + with open(path, "r") as json_file: + try: + data = json.load(json_file) + except Exception as exc: + log.error( + "Error loading json: {} - Exception: {}".format(path, exc) + ) + return data + + +def get_batch_asset_task_info(ctx): + """Parses context data from webpublisher's batch metadata + + Returns: + (tuple): asset, task_name (Optional), task_type + """ + task_type = "default_task_type" + task_name = None + asset = None + + if ctx["type"] == "task": + items = ctx["path"].split('/') + asset = items[-2] + task_name = ctx["name"] + task_type = ctx["attributes"]["type"] + else: + asset = ctx["name"] + + return asset, task_name, task_type + + +def get_webpublish_conn(): + """Get connection to OP 'webpublishes' collection.""" + mongo_client = OpenPypeMongoConnection.get_mongo_client() + database_name = os.environ["OPENPYPE_DATABASE_NAME"] + return mongo_client[database_name]["webpublishes"] + + +def start_webpublish_log(dbcon, batch_id, user): + """Start new log record for 'batch_id' + + Args: + dbcon (OpenPypeMongoConnection) + batch_id (str) + user (str) + Returns + (ObjectId) from DB + """ + return dbcon.insert_one({ + "batch_id": batch_id, + "start_date": datetime.now(), + "user": user, + "status": IN_PROGRESS_STATUS, + "progress": 0 # integer 0-100, percentage + }).inserted_id + + +def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): + """Loops through all plugins, logs ok and fails into OP DB. + + Args: + dbcon (OpenPypeMongoConnection) + _id (str) - id of current job in DB + log (openpype.lib.Logger) + batch_id (str) - id sent from frontend + close_plugin_name (str): name of plugin with responsibility to + close host app + """ + # Error exit as soon as any error occurs. + error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" + error_format += "-" * 80 + "\n" + + close_plugin = _get_close_plugin(close_plugin_name, log) + + if isinstance(_id, str): + _id = ObjectId(_id) + + log_lines = [] + processed = 0 + log_every = 5 + for result in pyblish.util.publish_iter(): + for record in result["records"]: + log_lines.append("{}: {}".format( + result["plugin"].label, record.msg)) + processed += 1 + + if result["error"]: + log.error(error_format.format(**result)) + log_lines = [error_format.format(**result)] + log_lines + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": ERROR_STATUS, + "log": os.linesep.join(log_lines) + + }} + ) + if close_plugin: # close host app explicitly after error + context = pyblish.api.Context() + close_plugin().process(context) + return + elif processed % log_every == 0: + # pyblish returns progress in 0.0 - 2.0 + progress = min(round(result["progress"] / 2 * 100), 99) + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "progress": progress, + "log": os.linesep.join(log_lines) + }} + ) + + # final update + if batch_id: + dbcon.update_many( + {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS}, + { + "$set": + { + "finish_date": datetime.now(), + "status": FINISHED_REPROCESS_STATUS, + } + } + ) + + dbcon.update_one( + {"_id": _id}, + { + "$set": + { + "finish_date": datetime.now(), + "status": FINISHED_OK_STATUS, + "progress": 100, + "log": os.linesep.join(log_lines) + } + } + ) + + +def fail_batch(_id, dbcon, msg): + """Set current batch as failed as there is some problem. + + Raises: + ValueError + """ + dbcon.update_one( + {"_id": _id}, + {"$set": + { + "finish_date": datetime.now(), + "status": ERROR_STATUS, + "log": msg + + }} + ) + raise ValueError(msg) + + +def find_variant_key(application_manager, host): + """Searches for latest installed variant for 'host' + + Args: + application_manager (ApplicationManager) + host (str) + Returns + (string) (optional) + Raises: + (ValueError) if no variant found + """ + app_group = application_manager.app_groups.get(host) + if not app_group or not app_group.enabled: + raise ValueError("No application {} configured".format(host)) + + found_variant_key = None + # finds most up-to-date variant if any installed + sorted_variants = collections.OrderedDict( + sorted(app_group.variants.items())) + for variant_key, variant in sorted_variants.items(): + for executable in variant.executables: + if executable.exists(): + found_variant_key = variant_key + + if not found_variant_key: + raise ValueError("No executable for {} found".format(host)) + + return found_variant_key + + +def _get_close_plugin(close_plugin_name, log): + if close_plugin_name: + plugins = pyblish.api.discover() + for plugin in plugins: + if plugin.__name__ == close_plugin_name: + return plugin + + log.debug("Close plugin not found, app might not close.") + + +def get_task_data(batch_dir): + """Return parsed data from first task manifest.json + + Used for `remotepublishfromapp` command where batch contains only + single task with publishable workfile. + + Returns: + (dict) + Throws: + (ValueError) if batch or task manifest not found or broken + """ + batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) + if not batch_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(batch_dir)) + task_dir_name = batch_data["tasks"][0] + task_data = parse_json(os.path.join(batch_dir, task_dir_name, + "manifest.json")) + if not task_data: + raise ValueError( + "Cannot parse batch meta in {} folder".format(task_data)) + + return task_data + + +def get_timeout(project_name, host_name, task_type): + """Returns timeout(seconds) from Setting profile.""" + filter_data = { + "task_types": task_type, + "hosts": host_name + } + timeout_profiles = (get_project_settings(project_name)["webpublisher"] + ["timeout_profiles"]) + matching_item = filter_profiles(timeout_profiles, filter_data) + timeout = 3600 + if matching_item: + timeout = matching_item["timeout"] + + return timeout From 6c330c48969bffba508a83aa0d98cf93d804142f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:50:54 +0200 Subject: [PATCH 66/80] use lib functions from webpublisher --- .../plugins/publish/collect_batch_data.py | 8 ++++---- openpype/hosts/webpublisher/cli_functions.py | 17 +++++++++-------- .../plugins/publish/collect_batch_data.py | 11 ++++++----- .../plugins/publish/collect_published_files.py | 6 ++---- .../publish/collect_tvpaint_workfile_data.py | 2 +- .../webserver_service/webpublish_routes.py | 12 +++++------- .../webserver_service/webserver_cli.py | 14 +++++++------- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py index 2881ef0ea6..5d50a78914 100644 --- a/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py +++ b/openpype/hosts/photoshop/plugins/publish/collect_batch_data.py @@ -17,11 +17,11 @@ import os import pyblish.api -from openpype.lib.plugin_tools import ( - parse_json, - get_batch_asset_task_info -) from openpype.pipeline import legacy_io +from openpype_modules.webpublisher.lib import ( + get_batch_asset_task_info, + parse_json +) class CollectBatchData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/cli_functions.py index ad3bb596fb..83f53ced68 100644 --- a/openpype/hosts/webpublisher/cli_functions.py +++ b/openpype/hosts/webpublisher/cli_functions.py @@ -4,7 +4,15 @@ import pyblish.api import pyblish.util from openpype.lib import Logger -from openpype.lib.remote_publish import ( +from openpype.lib.applications import ( + ApplicationManager, + get_app_environments_for_context, +) +from openpype.pipeline import install_host +from openpype.hosts.webpublisher.api import WebpublisherHost + +from .lib import ( + get_batch_asset_task_info, get_webpublish_conn, start_webpublish_log, publish_and_log, @@ -14,13 +22,6 @@ from openpype.lib.remote_publish import ( get_timeout, IN_PROGRESS_STATUS ) -from openpype.lib.applications import ( - ApplicationManager, - get_app_environments_for_context, -) -from openpype.lib.plugin_tools import get_batch_asset_task_info -from openpype.pipeline import install_host -from openpype.hosts.webpublisher.api import WebpublisherHost def cli_publish(project_name, batch_path, user_email, targets): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py index 9ff779636a..eb2737b276 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_batch_data.py @@ -13,12 +13,13 @@ import os import pyblish.api -from openpype.lib.plugin_tools import ( - parse_json, - get_batch_asset_task_info -) -from openpype.lib.remote_publish import get_webpublish_conn, IN_PROGRESS_STATUS from openpype.pipeline import legacy_io +from openpype_modules.webpublisher.lib import ( + parse_json, + get_batch_asset_task_info, + get_webpublish_conn, + IN_PROGRESS_STATUS +) class CollectBatchData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py index 20e277d794..454f78ce9d 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_published_files.py @@ -23,10 +23,8 @@ from openpype.lib import ( get_ffprobe_streams, convert_ffprobe_fps_value, ) -from openpype.lib.plugin_tools import ( - parse_json, - get_subset_name_with_asset_doc -) +from openpype.lib.plugin_tools import get_subset_name_with_asset_doc +from openpype_modules.webpublisher.lib import parse_json class CollectPublishedFiles(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py index f0f29260a2..b5f8ed9c8f 100644 --- a/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py +++ b/openpype/hosts/webpublisher/plugins/publish/collect_tvpaint_workfile_data.py @@ -16,11 +16,11 @@ import uuid import json import shutil import pyblish.api -from openpype.lib.plugin_tools import parse_json from openpype.hosts.tvpaint.worker import ( SenderTVPaintCommands, CollectSceneData ) +from openpype_modules.webpublisher.lib import parse_json class CollectTVPaintWorkfileData(pyblish.api.ContextPlugin): diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 2e9d460a98..e3de555ace 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -10,16 +10,16 @@ from aiohttp.web_response import Response from openpype.client import ( get_projects, get_assets, - OpenPypeMongoConnection, ) from openpype.lib import Logger -from openpype.lib.remote_publish import ( +from openpype.settings import get_project_settings +from openpype_modules.webserver.base_routes import RestApiEndpoint +from openpype_modules.webpublisher.lib import ( + get_webpublish_conn, get_task_data, ERROR_STATUS, REPROCESS_STATUS ) -from openpype.settings import get_project_settings -from openpype_modules.webserver.base_routes import RestApiEndpoint log = Logger.get_logger("WebpublishRoutes") @@ -77,9 +77,7 @@ class WebpublishRestApiResource(JsonApiResource): """Resource carrying OP DB connection for storing batch info into DB.""" def __init__(self): - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - self.dbcon = mongo_client[database_name]["webpublishes"] + self.dbcon = get_webpublish_conn() class ProjectsEndpoint(ResourceRestApiEndpoint): diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 936bd9735f..47c792a575 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -7,8 +7,15 @@ import json import subprocess from openpype.client import OpenPypeMongoConnection +from openpype.modules import ModulesManager from openpype.lib import Logger +from openpype_modules.webpublisher.lib import ( + ERROR_STATUS, + REPROCESS_STATUS, + SENT_REPROCESSING_STATUS +) + from .webpublish_routes import ( RestApiResource, WebpublishRestApiResource, @@ -21,19 +28,12 @@ from .webpublish_routes import ( TaskPublishEndpoint, UserReportEndpoint ) -from openpype.lib.remote_publish import ( - ERROR_STATUS, - REPROCESS_STATUS, - SENT_REPROCESSING_STATUS -) - log = Logger.get_logger("webserver_gui") def run_webserver(*args, **kwargs): """Runs webserver in command line, adds routes.""" - from openpype.modules import ModulesManager manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] From 233d70bdd8d21062842aa88b15841ac1fb61f0a4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:54:05 +0200 Subject: [PATCH 67/80] headless_publish is a method on webpublisher addon --- openpype/hosts/aftereffects/api/lib.py | 19 ++++++++++++++----- openpype/hosts/photoshop/api/lib.py | 9 +++++---- openpype/hosts/webpublisher/addon.py | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/aftereffects/api/lib.py b/openpype/hosts/aftereffects/api/lib.py index ce4cbf09af..d5583ee862 100644 --- a/openpype/hosts/aftereffects/api/lib.py +++ b/openpype/hosts/aftereffects/api/lib.py @@ -3,11 +3,12 @@ import sys import contextlib import traceback import logging +from functools import partial from Qt import QtWidgets from openpype.pipeline import install_host -from openpype.lib.remote_publish import headless_publish +from openpype.modules import ModulesManager from openpype.tools.utils import host_tools from .launch_logic import ProcessLauncher, get_stub @@ -35,10 +36,18 @@ def main(*subprocess_args): launcher.start() if os.environ.get("HEADLESS_PUBLISH"): - launcher.execute_in_main_thread(lambda: headless_publish( - log, - "CloseAE", - os.environ.get("IS_TEST"))) + manager = ModulesManager() + webpublisher_addon = manager["webpublisher"] + + launcher.execute_in_main_thread( + partial( + webpublisher_addon.headless_publish, + log, + "CloseAE", + os.environ.get("IS_TEST") + ) + ) + elif os.environ.get("AVALON_PHOTOSHOP_WORKFILES_ON_LAUNCH", True): save = False if os.getenv("WORKFILES_SAVE_AS"): diff --git a/openpype/hosts/photoshop/api/lib.py b/openpype/hosts/photoshop/api/lib.py index 2f57d64464..73a546604f 100644 --- a/openpype/hosts/photoshop/api/lib.py +++ b/openpype/hosts/photoshop/api/lib.py @@ -5,11 +5,10 @@ import traceback from Qt import QtWidgets -from openpype.api import Logger +from openpype.lib import env_value_to_bool, Logger +from openpype.modules import ModulesManager from openpype.pipeline import install_host from openpype.tools.utils import host_tools -from openpype.lib.remote_publish import headless_publish -from openpype.lib import env_value_to_bool from .launch_logic import ProcessLauncher, stub @@ -35,8 +34,10 @@ def main(*subprocess_args): launcher.start() if env_value_to_bool("HEADLESS_PUBLISH"): + manager = ModulesManager() + webpublisher_addon = manager["webpublisher"] launcher.execute_in_main_thread( - headless_publish, + webpublisher_addon.headless_publish, log, "ClosePS", os.environ.get("IS_TEST") diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 9e63030fe2..0bba8adc4b 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -15,6 +15,30 @@ class WebpublisherAddon(OpenPypeModule, IHostModule): def initialize(self, module_settings): self.enabled = True + def headless_publish(self, log, close_plugin_name=None, is_test=False): + """Runs publish in a opened host with a context. + + Close Python process at the end. + """ + + from openpype.pipeline.publish.lib import remote_publish + from .lib import get_webpublish_conn, publish_and_log + + if is_test: + remote_publish(log, close_plugin_name) + return + + dbcon = get_webpublish_conn() + _id = os.environ.get("BATCH_LOG_ID") + if not _id: + log.warning("Unable to store log records, " + "batch will be unfinished!") + return + + publish_and_log( + dbcon, _id, log, close_plugin_name=close_plugin_name + ) + def cli(self, click_group): click_group.add_command(cli_main) From a98c7953aaf4fecf465b7e9b95357de2056e3018 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:54:30 +0200 Subject: [PATCH 68/80] use 'find_close_plugin' --- openpype/hosts/webpublisher/lib.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/webpublisher/lib.py b/openpype/hosts/webpublisher/lib.py index dde875c934..4bc3f1db80 100644 --- a/openpype/hosts/webpublisher/lib.py +++ b/openpype/hosts/webpublisher/lib.py @@ -12,6 +12,7 @@ from openpype.client.mongo import OpenPypeMongoConnection from openpype.settings import get_project_settings from openpype.lib import Logger from openpype.lib.profiles_filtering import filter_profiles +from openpype.pipeline.publish.lib import find_close_plugin ERROR_STATUS = "error" IN_PROGRESS_STATUS = "in_progress" @@ -108,7 +109,7 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" error_format += "-" * 80 + "\n" - close_plugin = _get_close_plugin(close_plugin_name, log) + close_plugin = find_close_plugin(close_plugin_name, log) if isinstance(_id, str): _id = ObjectId(_id) @@ -227,16 +228,6 @@ def find_variant_key(application_manager, host): return found_variant_key -def _get_close_plugin(close_plugin_name, log): - if close_plugin_name: - plugins = pyblish.api.discover() - for plugin in plugins: - if plugin.__name__ == close_plugin_name: - return plugin - - log.debug("Close plugin not found, app might not close.") - - def get_task_data(batch_dir): """Return parsed data from first task manifest.json From 9c3e37e3f4ab83e465b71d908bcec439df011385 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:55:04 +0200 Subject: [PATCH 69/80] removed unused functions from openpype lib --- openpype/lib/plugin_tools.py | 45 ------ openpype/lib/remote_publish.py | 277 --------------------------------- 2 files changed, 322 deletions(-) delete mode 100644 openpype/lib/remote_publish.py diff --git a/openpype/lib/plugin_tools.py b/openpype/lib/plugin_tools.py index 060db94ae0..659210e6e3 100644 --- a/openpype/lib/plugin_tools.py +++ b/openpype/lib/plugin_tools.py @@ -427,48 +427,3 @@ def get_background_layers(file_url): layer.get("filename")). replace("\\", "/")) return layers - - -def parse_json(path): - """Parses json file at 'path' location - - Returns: - (dict) or None if unparsable - Raises: - AsssertionError if 'path' doesn't exist - """ - path = path.strip('\"') - assert os.path.isfile(path), ( - "Path to json file doesn't exist. \"{}\"".format(path) - ) - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data - - -def get_batch_asset_task_info(ctx): - """Parses context data from webpublisher's batch metadata - - Returns: - (tuple): asset, task_name (Optional), task_type - """ - task_type = "default_task_type" - task_name = None - asset = None - - if ctx["type"] == "task": - items = ctx["path"].split('/') - asset = items[-2] - task_name = ctx["name"] - task_type = ctx["attributes"]["type"] - else: - asset = ctx["name"] - - return asset, task_name, task_type diff --git a/openpype/lib/remote_publish.py b/openpype/lib/remote_publish.py deleted file mode 100644 index 2a901544cc..0000000000 --- a/openpype/lib/remote_publish.py +++ /dev/null @@ -1,277 +0,0 @@ -import os -from datetime import datetime -import collections - -from bson.objectid import ObjectId - -import pyblish.util -import pyblish.api - -from openpype.client.mongo import OpenPypeMongoConnection -from openpype.lib.plugin_tools import parse_json -from openpype.lib.profiles_filtering import filter_profiles -from openpype.api import get_project_settings - -ERROR_STATUS = "error" -IN_PROGRESS_STATUS = "in_progress" -REPROCESS_STATUS = "reprocess" -SENT_REPROCESSING_STATUS = "sent_for_reprocessing" -FINISHED_REPROCESS_STATUS = "republishing_finished" -FINISHED_OK_STATUS = "finished_ok" - - -def headless_publish(log, close_plugin_name=None, is_test=False): - """Runs publish in a opened host with a context and closes Python process. - """ - if not is_test: - dbcon = get_webpublish_conn() - _id = os.environ.get("BATCH_LOG_ID") - if not _id: - log.warning("Unable to store log records, " - "batch will be unfinished!") - return - - publish_and_log(dbcon, _id, log, close_plugin_name=close_plugin_name) - else: - publish(log, close_plugin_name) - - -def get_webpublish_conn(): - """Get connection to OP 'webpublishes' collection.""" - mongo_client = OpenPypeMongoConnection.get_mongo_client() - database_name = os.environ["OPENPYPE_DATABASE_NAME"] - return mongo_client[database_name]["webpublishes"] - - -def start_webpublish_log(dbcon, batch_id, user): - """Start new log record for 'batch_id' - - Args: - dbcon (OpenPypeMongoConnection) - batch_id (str) - user (str) - Returns - (ObjectId) from DB - """ - return dbcon.insert_one({ - "batch_id": batch_id, - "start_date": datetime.now(), - "user": user, - "status": IN_PROGRESS_STATUS, - "progress": 0 # integer 0-100, percentage - }).inserted_id - - -def publish(log, close_plugin_name=None, raise_error=False): - """Loops through all plugins, logs to console. Used for tests. - - Args: - log (openpype.lib.Logger) - close_plugin_name (str): name of plugin with responsibility to - close host app - """ - # Error exit as soon as any error occurs. - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" - - close_plugin = _get_close_plugin(close_plugin_name, log) - - for result in pyblish.util.publish_iter(): - for record in result["records"]: - log.info("{}: {}".format( - result["plugin"].label, record.msg)) - - if result["error"]: - error_message = error_format.format(**result) - log.error(error_message) - if close_plugin: # close host app explicitly after error - context = pyblish.api.Context() - close_plugin().process(context) - if raise_error: - # Fatal Error is because of Deadline - error_message = "Fatal Error: " + error_format.format(**result) - raise RuntimeError(error_message) - - -def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None): - """Loops through all plugins, logs ok and fails into OP DB. - - Args: - dbcon (OpenPypeMongoConnection) - _id (str) - id of current job in DB - log (openpype.lib.Logger) - batch_id (str) - id sent from frontend - close_plugin_name (str): name of plugin with responsibility to - close host app - """ - # Error exit as soon as any error occurs. - error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}\n" - error_format += "-" * 80 + "\n" - - close_plugin = _get_close_plugin(close_plugin_name, log) - - if isinstance(_id, str): - _id = ObjectId(_id) - - log_lines = [] - processed = 0 - log_every = 5 - for result in pyblish.util.publish_iter(): - for record in result["records"]: - log_lines.append("{}: {}".format( - result["plugin"].label, record.msg)) - processed += 1 - - if result["error"]: - log.error(error_format.format(**result)) - log_lines = [error_format.format(**result)] + log_lines - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": ERROR_STATUS, - "log": os.linesep.join(log_lines) - - }} - ) - if close_plugin: # close host app explicitly after error - context = pyblish.api.Context() - close_plugin().process(context) - return - elif processed % log_every == 0: - # pyblish returns progress in 0.0 - 2.0 - progress = min(round(result["progress"] / 2 * 100), 99) - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "progress": progress, - "log": os.linesep.join(log_lines) - }} - ) - - # final update - if batch_id: - dbcon.update_many( - {"batch_id": batch_id, "status": SENT_REPROCESSING_STATUS}, - { - "$set": - { - "finish_date": datetime.now(), - "status": FINISHED_REPROCESS_STATUS, - } - } - ) - - dbcon.update_one( - {"_id": _id}, - { - "$set": - { - "finish_date": datetime.now(), - "status": FINISHED_OK_STATUS, - "progress": 100, - "log": os.linesep.join(log_lines) - } - } - ) - - -def fail_batch(_id, dbcon, msg): - """Set current batch as failed as there is some problem. - - Raises: - ValueError - """ - dbcon.update_one( - {"_id": _id}, - {"$set": - { - "finish_date": datetime.now(), - "status": ERROR_STATUS, - "log": msg - - }} - ) - raise ValueError(msg) - - -def find_variant_key(application_manager, host): - """Searches for latest installed variant for 'host' - - Args: - application_manager (ApplicationManager) - host (str) - Returns - (string) (optional) - Raises: - (ValueError) if no variant found - """ - app_group = application_manager.app_groups.get(host) - if not app_group or not app_group.enabled: - raise ValueError("No application {} configured".format(host)) - - found_variant_key = None - # finds most up-to-date variant if any installed - sorted_variants = collections.OrderedDict( - sorted(app_group.variants.items())) - for variant_key, variant in sorted_variants.items(): - for executable in variant.executables: - if executable.exists(): - found_variant_key = variant_key - - if not found_variant_key: - raise ValueError("No executable for {} found".format(host)) - - return found_variant_key - - -def _get_close_plugin(close_plugin_name, log): - if close_plugin_name: - plugins = pyblish.api.discover() - for plugin in plugins: - if plugin.__name__ == close_plugin_name: - return plugin - - log.debug("Close plugin not found, app might not close.") - - -def get_task_data(batch_dir): - """Return parsed data from first task manifest.json - - Used for `remotepublishfromapp` command where batch contains only - single task with publishable workfile. - - Returns: - (dict) - Throws: - (ValueError) if batch or task manifest not found or broken - """ - batch_data = parse_json(os.path.join(batch_dir, "manifest.json")) - if not batch_data: - raise ValueError( - "Cannot parse batch meta in {} folder".format(batch_dir)) - task_dir_name = batch_data["tasks"][0] - task_data = parse_json(os.path.join(batch_dir, task_dir_name, - "manifest.json")) - if not task_data: - raise ValueError( - "Cannot parse batch meta in {} folder".format(task_data)) - - return task_data - - -def get_timeout(project_name, host_name, task_type): - """Returns timeout(seconds) from Setting profile.""" - filter_data = { - "task_types": task_type, - "hosts": host_name - } - timeout_profiles = (get_project_settings(project_name)["webpublisher"] - ["timeout_profiles"]) - matching_item = filter_profiles(timeout_profiles, filter_data) - timeout = 3600 - if matching_item: - timeout = matching_item["timeout"] - - return timeout From d5f6ad9fdc1727cd3c631698ae258ed8485ca479 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 17:58:09 +0200 Subject: [PATCH 70/80] renamed 'cli_functions' to 'publish_functions' --- openpype/hosts/webpublisher/addon.py | 4 ++-- .../webpublisher/{cli_functions.py => publish_functions.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename openpype/hosts/webpublisher/{cli_functions.py => publish_functions.py} (100%) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 0bba8adc4b..cb639db3fa 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -63,7 +63,7 @@ def publish(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import cli_publish + from .publish_functions import cli_publish cli_publish(project, path, user, targets) @@ -82,6 +82,6 @@ def publishfromapp(project, path, user=None, targets=None): More than one path is allowed. """ - from .cli_functions import cli_publish_from_app + from .publish_functions import cli_publish_from_app cli_publish_from_app(project, path, user, targets) diff --git a/openpype/hosts/webpublisher/cli_functions.py b/openpype/hosts/webpublisher/publish_functions.py similarity index 100% rename from openpype/hosts/webpublisher/cli_functions.py rename to openpype/hosts/webpublisher/publish_functions.py From 338d12e60cd7f2fe9a15efecac07ce7ae8449d57 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:03:09 +0200 Subject: [PATCH 71/80] added cli command for webserver --- openpype/hosts/webpublisher/addon.py | 19 +++++++++++++++++++ .../webserver_service/__init__.py | 6 ++++++ .../webserver_service/webserver_cli.py | 16 ++++++++++------ 3 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 openpype/hosts/webpublisher/webserver_service/__init__.py diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index cb639db3fa..85e16de4a6 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -85,3 +85,22 @@ def publishfromapp(project, path, user=None, targets=None): from .publish_functions import cli_publish_from_app cli_publish_from_app(project, path, user, targets) + + +@cli_main.command() +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) +@click.option("-e", "--executable", help="Executable") +@click.option("-u", "--upload_dir", help="Upload dir") +def webserver(executable, upload_dir, host=None, port=None): + """Starts webserver for communication with Webpublish FR via command line + + OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND + FTRACK_BOT_API_KEY provided with api key from Ftrack. + + Expect "pype.club" user created on Ftrack. + """ + + from .webserver_service import run_webserver + + run_webserver(executable, upload_dir, host, port) diff --git a/openpype/hosts/webpublisher/webserver_service/__init__.py b/openpype/hosts/webpublisher/webserver_service/__init__.py new file mode 100644 index 0000000000..e43f3f063a --- /dev/null +++ b/openpype/hosts/webpublisher/webserver_service/__init__.py @@ -0,0 +1,6 @@ +from .webserver_cli import run_webserver + + +__all__ = ( + "run_webserver", +) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index 47c792a575..093b53d9d3 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -32,21 +32,25 @@ from .webpublish_routes import ( log = Logger.get_logger("webserver_gui") -def run_webserver(*args, **kwargs): +def run_webserver(executable, upload_dir, host=None, port=None): """Runs webserver in command line, adds routes.""" + if not host: + host = "localhost" + if not port: + port = 8079 + manager = ModulesManager() webserver_module = manager.modules_by_name["webserver"] - host = kwargs.get("host") or "localhost" - port = kwargs.get("port") or 8079 + server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url # queue for remotepublishfromapp tasks studio_task_queue = collections.deque() resource = RestApiResource(server_manager, - upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"], + upload_dir=upload_dir, + executable=executable, studio_task_queue=studio_task_queue) projects_endpoint = ProjectsEndpoint(resource) server_manager.add_route( @@ -111,7 +115,7 @@ def run_webserver(*args, **kwargs): last_reprocessed = time.time() while True: if time.time() - last_reprocessed > 20: - reprocess_failed(kwargs["upload_dir"], webserver_url) + reprocess_failed(upload_dir, webserver_url) last_reprocessed = time.time() if studio_task_queue: args = studio_task_queue.popleft() From e2c83c142684ccf0f400ffd345ddcf530bf49ed7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:03:40 +0200 Subject: [PATCH 72/80] renamed webserver_cli.py into webserver.py --- openpype/hosts/webpublisher/webserver_service/__init__.py | 2 +- .../webserver_service/{webserver_cli.py => webserver.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename openpype/hosts/webpublisher/webserver_service/{webserver_cli.py => webserver.py} (100%) diff --git a/openpype/hosts/webpublisher/webserver_service/__init__.py b/openpype/hosts/webpublisher/webserver_service/__init__.py index e43f3f063a..73111d286e 100644 --- a/openpype/hosts/webpublisher/webserver_service/__init__.py +++ b/openpype/hosts/webpublisher/webserver_service/__init__.py @@ -1,4 +1,4 @@ -from .webserver_cli import run_webserver +from .webserver import run_webserver __all__ = ( diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver.py similarity index 100% rename from openpype/hosts/webpublisher/webserver_service/webserver_cli.py rename to openpype/hosts/webpublisher/webserver_service/webserver.py From 971ae6d1ed0ad07713524dd4e7066e100fd22b34 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:04:37 +0200 Subject: [PATCH 73/80] fix import in global commands --- openpype/pype_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index b6c1228ade..fe46a4bc54 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -61,8 +61,8 @@ class PypeCommands: @staticmethod def launch_webpublisher_webservercli(*args, **kwargs): - from openpype.hosts.webpublisher.webserver_service.webserver_cli \ - import (run_webserver) + from openpype.hosts.webpublisher.webserver_service import run_webserver + return run_webserver(*args, **kwargs) @staticmethod From ae11ae16d5fd0b9f1cdb86263a25ace48bbf9b04 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:12:20 +0200 Subject: [PATCH 74/80] modify launch arguments --- .../webserver_service/webpublish_routes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index e3de555ace..4039d2c8ec 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -14,6 +14,7 @@ from openpype.client import ( from openpype.lib import Logger from openpype.settings import get_project_settings from openpype_modules.webserver.base_routes import RestApiEndpoint +from openpype_modules.webpublisher import WebpublisherAddon from openpype_modules.webpublisher.lib import ( get_webpublish_conn, get_task_data, @@ -213,7 +214,7 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): # TVPaint filter { "extensions": [".tvpp"], - "command": "remotepublish", + "command": "publish", "arguments": { "targets": ["tvpaint_worker"] }, @@ -222,13 +223,13 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): # Photoshop filter { "extensions": [".psd", ".psb"], - "command": "remotepublishfromapp", + "command": "publishfromapp", "arguments": { - # Command 'remotepublishfromapp' requires --host argument + # Command 'publishfromapp' requires --host argument "host": "photoshop", # Make sure targets are set to None for cases that default # would change - # - targets argument is not used in 'remotepublishfromapp' + # - targets argument is not used in 'publishfromapp' "targets": ["remotepublish"] }, # does publish need to be handled by a queue, eg. only @@ -240,7 +241,7 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): batch_dir = os.path.join(self.resource.upload_dir, content["batch"]) # Default command and arguments - command = "remotepublish" + command = "publish" add_args = { # All commands need 'project' and 'user' "project": content["project_name"], @@ -271,6 +272,8 @@ class BatchPublishEndpoint(WebpublishApiEndpoint): args = [ openpype_app, + "module", + WebpublisherAddon.name, command, batch_dir ] From 808d1a5dd121d2f771a75d8ea4d061522ca42306 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:36:11 +0200 Subject: [PATCH 75/80] abstrac provides has log attribute --- .../sync_server/providers/abstract_provider.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/providers/abstract_provider.py b/openpype/modules/sync_server/providers/abstract_provider.py index 9c808dc80e..e11a8ba71e 100644 --- a/openpype/modules/sync_server/providers/abstract_provider.py +++ b/openpype/modules/sync_server/providers/abstract_provider.py @@ -10,6 +10,8 @@ class AbstractProvider: CODE = '' LABEL = '' + _log = None + def __init__(self, project_name, site_name, tree=None, presets=None): self.presets = None self.active = False @@ -19,6 +21,12 @@ class AbstractProvider: super(AbstractProvider, self).__init__() + @property + def log(self): + if self._log is None: + self._log = Logger.get_logger(self.__class__.__name__) + return self._log + @abc.abstractmethod def is_active(self): """ @@ -199,11 +207,11 @@ class AbstractProvider: path = anatomy.fill_root(path) except KeyError: msg = "Error in resolving local root from anatomy" - log.error(msg) + self.log.error(msg) raise ValueError(msg) except IndexError: msg = "Path {} contains unfillable placeholder" - log.error(msg) + self.log.error(msg) raise ValueError(msg) return path From 5631fb66a79fe64c38073217aebe32b1a0fa5c60 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:36:31 +0200 Subject: [PATCH 76/80] use log attribute in provides --- .../modules/sync_server/providers/dropbox.py | 17 +++--- .../modules/sync_server/providers/gdrive.py | 53 +++++++++++-------- .../modules/sync_server/providers/sftp.py | 15 +++--- 3 files changed, 45 insertions(+), 40 deletions(-) diff --git a/openpype/modules/sync_server/providers/dropbox.py b/openpype/modules/sync_server/providers/dropbox.py index 89d6990841..e026ae7ef6 100644 --- a/openpype/modules/sync_server/providers/dropbox.py +++ b/openpype/modules/sync_server/providers/dropbox.py @@ -2,12 +2,9 @@ import os import dropbox -from openpype.api import Logger from .abstract_provider import AbstractProvider from ..utils import EditableScopes -log = Logger().get_logger("SyncServer") - class DropboxHandler(AbstractProvider): CODE = 'dropbox' @@ -20,26 +17,26 @@ class DropboxHandler(AbstractProvider): self.dbx = None if not self.presets: - log.info( + self.log.info( "Sync Server: There are no presets for {}.".format(site_name) ) return if not self.presets["enabled"]: - log.debug("Sync Server: Site {} not enabled for {}.". + self.log.debug("Sync Server: Site {} not enabled for {}.". format(site_name, project_name)) return token = self.presets.get("token", "") if not token: msg = "Sync Server: No access token for dropbox provider" - log.info(msg) + self.log.info(msg) return team_folder_name = self.presets.get("team_folder_name", "") if not team_folder_name: msg = "Sync Server: No team folder name for dropbox provider" - log.info(msg) + self.log.info(msg) return acting_as_member = self.presets.get("acting_as_member", "") @@ -47,7 +44,7 @@ class DropboxHandler(AbstractProvider): msg = ( "Sync Server: No acting member for dropbox provider" ) - log.info(msg) + self.log.info(msg) return try: @@ -55,7 +52,7 @@ class DropboxHandler(AbstractProvider): token, acting_as_member, team_folder_name ) except Exception as e: - log.info("Could not establish dropbox object: {}".format(e)) + self.log.info("Could not establish dropbox object: {}".format(e)) return super(AbstractProvider, self).__init__() @@ -448,7 +445,7 @@ class DropboxHandler(AbstractProvider): path = anatomy.fill_root(path) except KeyError: msg = "Error in resolving local root from anatomy" - log.error(msg) + self.log.error(msg) raise ValueError(msg) return path diff --git a/openpype/modules/sync_server/providers/gdrive.py b/openpype/modules/sync_server/providers/gdrive.py index bef707788b..9a3ce89cf5 100644 --- a/openpype/modules/sync_server/providers/gdrive.py +++ b/openpype/modules/sync_server/providers/gdrive.py @@ -5,12 +5,12 @@ import sys import six import platform -from openpype.api import Logger -from openpype.api import get_system_settings +from openpype.lib import Logger +from openpype.settings import get_system_settings from .abstract_provider import AbstractProvider from ..utils import time_function, ResumableError -log = Logger().get_logger("SyncServer") +log = Logger.get_logger("GDriveHandler") try: from googleapiclient.discovery import build @@ -69,13 +69,17 @@ class GDriveHandler(AbstractProvider): self.presets = presets if not self.presets: - log.info("Sync Server: There are no presets for {}.". - format(site_name)) + self.log.info( + "Sync Server: There are no presets for {}.".format(site_name) + ) return if not self.presets["enabled"]: - log.debug("Sync Server: Site {} not enabled for {}.". - format(site_name, project_name)) + self.log.debug( + "Sync Server: Site {} not enabled for {}.".format( + site_name, project_name + ) + ) return current_platform = platform.system().lower() @@ -85,20 +89,22 @@ class GDriveHandler(AbstractProvider): if not cred_path: msg = "Sync Server: Please, fill the credentials for gdrive "\ "provider for platform '{}' !".format(current_platform) - log.info(msg) + self.log.info(msg) return try: cred_path = cred_path.format(**os.environ) except KeyError as e: - log.info("Sync Server: The key(s) {} does not exist in the " - "environment variables".format(" ".join(e.args))) + self.log.info(( + "Sync Server: The key(s) {} does not exist in the " + "environment variables" + ).format(" ".join(e.args))) return if not os.path.exists(cred_path): msg = "Sync Server: No credentials for gdrive provider " + \ "for '{}' on path '{}'!".format(site_name, cred_path) - log.info(msg) + self.log.info(msg) return self.service = None @@ -318,7 +324,7 @@ class GDriveHandler(AbstractProvider): fields='id') media.stream() - log.debug("Start Upload! {}".format(source_path)) + self.log.debug("Start Upload! {}".format(source_path)) last_tick = status = response = None status_val = 0 while response is None: @@ -331,7 +337,7 @@ class GDriveHandler(AbstractProvider): if not last_tick or \ time.time() - last_tick >= server.LOG_PROGRESS_SEC: last_tick = time.time() - log.debug("Uploaded %d%%." % + self.log.debug("Uploaded %d%%." % int(status_val * 100)) server.update_db(project_name=project_name, new_file_id=None, @@ -350,8 +356,9 @@ class GDriveHandler(AbstractProvider): if 'has not granted' in ex._get_reason().strip(): raise PermissionError(ex._get_reason().strip()) - log.warning("Forbidden received, hit quota. " - "Injecting 60s delay.") + self.log.warning( + "Forbidden received, hit quota. Injecting 60s delay." + ) time.sleep(60) return False raise @@ -417,7 +424,7 @@ class GDriveHandler(AbstractProvider): if not last_tick or \ time.time() - last_tick >= server.LOG_PROGRESS_SEC: last_tick = time.time() - log.debug("Downloaded %d%%." % + self.log.debug("Downloaded %d%%." % int(status_val * 100)) server.update_db(project_name=project_name, new_file_id=None, @@ -629,9 +636,9 @@ class GDriveHandler(AbstractProvider): ["gdrive"] ) except KeyError: - log.info(("Sync Server: There are no presets for Gdrive " + - "provider."). - format(str(provider_presets))) + log.info(( + "Sync Server: There are no presets for Gdrive provider." + ).format(str(provider_presets))) return return provider_presets @@ -704,7 +711,7 @@ class GDriveHandler(AbstractProvider): roots[self.MY_DRIVE_STR] = self.service.files() \ .get(fileId='root').execute() except errors.HttpError: - log.warning("HttpError in sync loop, " + self.log.warning("HttpError in sync loop, " "trying next loop", exc_info=True) raise ResumableError @@ -727,7 +734,7 @@ class GDriveHandler(AbstractProvider): Returns: (dictionary) path as a key, folder id as a value """ - log.debug("build_tree len {}".format(len(folders))) + self.log.debug("build_tree len {}".format(len(folders))) if not self.root: # build only when necessary, could be expensive self.root = self._prepare_root_info() @@ -779,9 +786,9 @@ class GDriveHandler(AbstractProvider): loop_cnt += 1 if len(no_parents_yet) > 0: - log.debug("Some folders path are not resolved {}". + self.log.debug("Some folders path are not resolved {}". format(no_parents_yet)) - log.debug("Remove deleted folders from trash.") + self.log.debug("Remove deleted folders from trash.") return tree diff --git a/openpype/modules/sync_server/providers/sftp.py b/openpype/modules/sync_server/providers/sftp.py index 302ffae3e6..40f11cb9dd 100644 --- a/openpype/modules/sync_server/providers/sftp.py +++ b/openpype/modules/sync_server/providers/sftp.py @@ -4,10 +4,10 @@ import time import threading import platform -from openpype.api import Logger -from openpype.api import get_system_settings +from openpype.lib import Logger +from openpype.settings import get_system_settings from .abstract_provider import AbstractProvider -log = Logger().get_logger("SyncServer") +log = Logger.get_logger("SyncServer-SFTPHandler") pysftp = None try: @@ -43,8 +43,9 @@ class SFTPHandler(AbstractProvider): self.presets = presets if not self.presets: - log.warning("Sync Server: There are no presets for {}.". - format(site_name)) + self.log.warning( + "Sync Server: There are no presets for {}.".format(site_name) + ) return # store to instance for reconnect @@ -423,7 +424,7 @@ class SFTPHandler(AbstractProvider): return pysftp.Connection(**conn_params) except (paramiko.ssh_exception.SSHException, pysftp.exceptions.ConnectionException): - log.warning("Couldn't connect", exc_info=True) + self.log.warning("Couldn't connect", exc_info=True) def _mark_progress(self, project_name, file, representation, server, site, source_path, target_path, direction): @@ -445,7 +446,7 @@ class SFTPHandler(AbstractProvider): time.time() - last_tick >= server.LOG_PROGRESS_SEC: status_val = target_file_size / source_file_size last_tick = time.time() - log.debug(direction + "ed %d%%." % int(status_val * 100)) + self.log.debug(direction + "ed %d%%." % int(status_val * 100)) server.update_db(project_name=project_name, new_file_id=None, file=file, From 54b8719b76c98b30d30e81b828e2dfb9ce13d0a8 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:49:04 +0200 Subject: [PATCH 77/80] fix attr initialization --- openpype/modules/timers_manager/rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/timers_manager/rest_api.py b/openpype/modules/timers_manager/rest_api.py index 6686407350..4a2e9e6575 100644 --- a/openpype/modules/timers_manager/rest_api.py +++ b/openpype/modules/timers_manager/rest_api.py @@ -10,7 +10,7 @@ class TimersManagerModuleRestApi: happens in Workfile app. """ def __init__(self, user_module, server_manager): - self.log = None + self._log = None self.module = user_module self.server_manager = server_manager From 59f36cc7c8ef54e3ac54d547e5f772bc726f3f1b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 25 Aug 2022 18:49:20 +0200 Subject: [PATCH 78/80] log traceback when webserver connection is not possible --- openpype/modules/webserver/webserver_module.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openpype/modules/webserver/webserver_module.py b/openpype/modules/webserver/webserver_module.py index 686bd27bfd..16861abd29 100644 --- a/openpype/modules/webserver/webserver_module.py +++ b/openpype/modules/webserver/webserver_module.py @@ -53,9 +53,12 @@ class WebServerModule(OpenPypeModule, ITrayService): try: module.webserver_initialization(self.server_manager) except Exception: - self.log.warning(( - "Failed to connect module \"{}\" to webserver." - ).format(module.name)) + self.log.warning( + ( + "Failed to connect module \"{}\" to webserver." + ).format(module.name), + exc_info=True + ) def tray_init(self): self.create_server_manager() From bad5b9b194f498903900ee283ed5a4b14e25a198 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 26 Aug 2022 10:04:17 +0200 Subject: [PATCH 79/80] fix import --- openpype/hosts/photoshop/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/photoshop/__init__.py b/openpype/hosts/photoshop/__init__.py index b3f66ea35c..773f73d624 100644 --- a/openpype/hosts/photoshop/__init__.py +++ b/openpype/hosts/photoshop/__init__.py @@ -1,4 +1,4 @@ -from .module import ( +from .addon import ( PhotoshopAddon, PHOTOSHOP_HOST_DIR, ) From 45c112eb84ae741d4b102ea89ac5c64c01f591f5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 26 Aug 2022 10:12:35 +0200 Subject: [PATCH 80/80] fixed arguments --- openpype/hosts/webpublisher/addon.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/webpublisher/addon.py b/openpype/hosts/webpublisher/addon.py index 85e16de4a6..7d26d5a7ff 100644 --- a/openpype/hosts/webpublisher/addon.py +++ b/openpype/hosts/webpublisher/addon.py @@ -57,7 +57,7 @@ def cli_main(): @click.option("-t", "--targets", help="Targets", default=None, multiple=True) def publish(project, path, user=None, targets=None): - """Start CLI publishing. + """Start publishing (Inner command). Publish collects json from paths provided as an argument. More than one path is allowed. @@ -70,13 +70,13 @@ def publish(project, path, user=None, targets=None): @cli_main.command() @click.argument("path") +@click.option("-p", "--project", help="Project") @click.option("-h", "--host", help="Host") @click.option("-u", "--user", help="User email address") -@click.option("-p", "--project", help="Project") @click.option("-t", "--targets", help="Targets", default=None, multiple=True) -def publishfromapp(project, path, user=None, targets=None): - """Start CLI publishing. +def publishfromapp(project, path, host, user=None, targets=None): + """Start publishing through application (Inner command). Publish collects json from paths provided as an argument. More than one path is allowed. @@ -84,16 +84,16 @@ def publishfromapp(project, path, user=None, targets=None): from .publish_functions import cli_publish_from_app - cli_publish_from_app(project, path, user, targets) + cli_publish_from_app(project, path, host, user, targets) @cli_main.command() -@click.option("-h", "--host", help="Host", default=None) -@click.option("-p", "--port", help="Port", default=None) @click.option("-e", "--executable", help="Executable") @click.option("-u", "--upload_dir", help="Upload dir") +@click.option("-h", "--host", help="Host", default=None) +@click.option("-p", "--port", help="Port", default=None) def webserver(executable, upload_dir, host=None, port=None): - """Starts webserver for communication with Webpublish FR via command line + """Start service for communication with Webpublish Front end. OP must be congigured on a machine, eg. OPENPYPE_MONGO filled AND FTRACK_BOT_API_KEY provided with api key from Ftrack.