From 3770eb69b653f31292faee388ed416f7bf8a171f Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 29 Mar 2022 18:34:34 +0300 Subject: [PATCH 01/81] add rstex function --- .../maya/plugins/publish/extract_look.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index a8893072d0..1516495278 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -6,6 +6,7 @@ import json import tempfile import contextlib import subprocess +from openpype.lib.vendor_bin_utils import find_executable from collections import OrderedDict from maya import cmds # noqa @@ -42,6 +43,58 @@ def find_paths_by_hash(texture_hash): return io.distinct(key, {"type": "version"}) +def rstex(source, *args): + """Make `.rstexbin` using `redshiftTextureProcessor` + with some default settings. + + This function requires the `REDSHIFT_COREDATAPATH` + to be in `PATH`. + + Args: + source (str): Path to source file. + *args: Additional arguments for `redshiftTextureProcessor`. + + Returns: + str: Output of `redshiftTextureProcessor` command. + + """ + if "REDSHIFT_COREDATAPATH" not in os.environ: + raise RuntimeError("Must have Redshift available.") + + redshift_bin_path = os.path.join( + os.environ["REDSHIFT_COREDATAPATH"], + "bin", + "redshiftTextureProcessor" + ) + + texture_processor_path = find_executable(redshift_bin_path) + + cmd = [ + texture_processor_path, + escape_space(source), + + ] + + cmd.extend(args) + + cmd = " ".join(cmd) + + CREATE_NO_WINDOW = 0x08000000 + kwargs = dict(args=cmd, stderr=subprocess.STDOUT) + + if sys.platform == "win32": + kwargs["creationflags"] = CREATE_NO_WINDOW + try: + out = subprocess.check_output(**kwargs) + except subprocess.CalledProcessError as exc: + print(exc) + import traceback + + traceback.print_exc() + raise + return out + + def maketx(source, destination, *args): """Make `.tx` using `maketx` with some default settings. From 72b45229e943c419083e243ae27a540373edd6e2 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 29 Mar 2022 19:23:25 +0300 Subject: [PATCH 02/81] fix style warnings --- openpype/hosts/maya/plugins/publish/extract_look.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 1516495278..6102b311a3 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -65,10 +65,10 @@ def rstex(source, *args): os.environ["REDSHIFT_COREDATAPATH"], "bin", "redshiftTextureProcessor" - ) + ) texture_processor_path = find_executable(redshift_bin_path) - + cmd = [ texture_processor_path, escape_space(source), @@ -83,7 +83,7 @@ def rstex(source, *args): kwargs = dict(args=cmd, stderr=subprocess.STDOUT) if sys.platform == "win32": - kwargs["creationflags"] = CREATE_NO_WINDOW + kwargs["creationflags"] = CREATE_NO_WINDOW try: out = subprocess.check_output(**kwargs) except subprocess.CalledProcessError as exc: From e65a1ea7d144abdcd0681641b45c50d11e903405 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 29 Mar 2022 19:45:00 +0300 Subject: [PATCH 03/81] remove extra line --- openpype/hosts/maya/plugins/publish/extract_look.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 6102b311a3..cd647a6733 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -71,8 +71,7 @@ def rstex(source, *args): cmd = [ texture_processor_path, - escape_space(source), - + escape_space(source), ] cmd.extend(args) From 2406f78f4717451c2408a21b0f36bd66bd4ec95b Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 29 Mar 2022 19:45:49 +0300 Subject: [PATCH 04/81] fix trailing whitespace --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index cd647a6733..fb90d7538b 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -71,7 +71,7 @@ def rstex(source, *args): cmd = [ texture_processor_path, - escape_space(source), + escape_space(source), ] cmd.extend(args) From be840d3a8b8b786c3fdfb56ab8ef7577c04747f3 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 4 Apr 2022 15:50:33 +0300 Subject: [PATCH 05/81] add exectuable path finder function --- openpype/lib/vendor_bin_utils.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py index 23e28ea304..c30a1ee709 100644 --- a/openpype/lib/vendor_bin_utils.py +++ b/openpype/lib/vendor_bin_utils.py @@ -120,6 +120,26 @@ def get_oiio_tools_path(tool="oiiotool"): return find_executable(os.path.join(oiio_dir, tool)) +def get_redshift_tool(tool_name): + """Path to redshift texture processor. + + On Windows it adds .exe extension if missing from tool argument. + + Args: + tool (string): Tool name. + + Returns: + str: Full path to redshift texture processor executable. + """ + redshift_tool_path = os.path.join( + os.environ["REDSHIFT_COREDATAPATH"], + "bin", + tool_name + ) + + return find_executable(redshift_tool_path) + + def get_ffmpeg_tool_path(tool="ffmpeg"): """Path to vendorized FFmpeg executable. From d4b8d47b18ed1605cdffa9a635586b40e29a6adf Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 4 Apr 2022 15:51:31 +0300 Subject: [PATCH 06/81] use redshift tool finder in extractor --- openpype/hosts/maya/plugins/publish/extract_look.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index fb90d7538b..6ce3a981f4 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -6,7 +6,7 @@ import json import tempfile import contextlib import subprocess -from openpype.lib.vendor_bin_utils import find_executable +from openpype.lib.vendor_bin_utils import get_redshift_tool from collections import OrderedDict from maya import cmds # noqa @@ -61,13 +61,7 @@ def rstex(source, *args): if "REDSHIFT_COREDATAPATH" not in os.environ: raise RuntimeError("Must have Redshift available.") - redshift_bin_path = os.path.join( - os.environ["REDSHIFT_COREDATAPATH"], - "bin", - "redshiftTextureProcessor" - ) - - texture_processor_path = find_executable(redshift_bin_path) + texture_processor_path = get_redshift_tool("TextureProcessor") cmd = [ texture_processor_path, From abc299e86eea14ec425c07e46c5af51551cf1a3e Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 5 Apr 2022 20:00:54 +0300 Subject: [PATCH 07/81] Add redshift texture processing option to schema --- .../schemas/projects_schema/schemas/schema_maya_create.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json index 0544b4bab7..b0bd46d20f 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json @@ -21,6 +21,11 @@ "key": "make_tx", "label": "Make tx files" }, + { + "type": "boolean", + "key": "rstex", + "label": "Make Redshift texture files" + }, { "type": "list", "key": "defaults", From 640415a0959bbc67f0e516b2363c4998b5d74508 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 5 Apr 2022 20:18:42 +0300 Subject: [PATCH 08/81] adjust key name --- .../schemas/projects_schema/schemas/schema_maya_create.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json index b0bd46d20f..bf3c9b3fe8 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json @@ -23,7 +23,7 @@ }, { "type": "boolean", - "key": "rstex", + "key": "rs_tex", "label": "Make Redshift texture files" }, { From 0bcf353c82df569dc4bab517f3f2713f05f29b71 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 5 Apr 2022 20:26:47 +0300 Subject: [PATCH 09/81] add redshift texture create option to look creator --- openpype/hosts/maya/plugins/create/create_look.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/create/create_look.py b/openpype/hosts/maya/plugins/create/create_look.py index 56e2640919..c190a73ade 100644 --- a/openpype/hosts/maya/plugins/create/create_look.py +++ b/openpype/hosts/maya/plugins/create/create_look.py @@ -12,6 +12,7 @@ class CreateLook(plugin.Creator): family = "look" icon = "paint-brush" make_tx = True + rs_tex = False def __init__(self, *args, **kwargs): super(CreateLook, self).__init__(*args, **kwargs) @@ -20,6 +21,7 @@ class CreateLook(plugin.Creator): # Whether to automatically convert the textures to .tx upon publish. self.data["maketx"] = self.make_tx - + # Whether to automatically convert the textures to .rstex upon publish. + self.data["rstex"] = self.rs_tex # Enable users to force a copy. self.data["forceCopy"] = False From 7bbf381c6e8fbd6a12a9ecf58d891f15d0e5ad83 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 5 Apr 2022 20:29:54 +0300 Subject: [PATCH 10/81] add rs_tex option to schema defaults --- openpype/settings/defaults/project_settings/maya.json | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 19d9a95595..7c09fa7891 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -34,6 +34,7 @@ "CreateLook": { "enabled": true, "make_tx": true, + "rs_tex": false, "defaults": [ "Main" ] From 078775e9b6bb5e814d1f284202419760853e631a Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 5 Apr 2022 21:16:02 +0300 Subject: [PATCH 11/81] add rstex variable to process_resources --- openpype/hosts/maya/plugins/publish/extract_look.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 6ce3a981f4..11f62ab80b 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -363,7 +363,8 @@ class ExtractLook(openpype.api.Extractor): # be the input file to multiple nodes. resources = instance.data["resources"] do_maketx = instance.data.get("maketx", False) - + # Option to convert textures to native redshift textures + do_rstex = instance.data.get("rstex", False) # Collect all unique files used in the resources files_metadata = {} for resource in resources: From 6ac5cd4a4a5efad533512b9deea45fe0bba4532a Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Fri, 8 Apr 2022 07:22:49 +0300 Subject: [PATCH 12/81] Add redshift texture processing flag --- openpype/hosts/maya/plugins/publish/extract_look.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 11f62ab80b..87e6625653 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -396,6 +396,7 @@ class ExtractLook(openpype.api.Extractor): source, mode, texture_hash = self._process_texture( filepath, + do_rstex, do_maketx, staging=staging_dir, linearize=linearize, @@ -487,7 +488,7 @@ class ExtractLook(openpype.api.Extractor): resources_dir, basename + ext ) - def _process_texture(self, filepath, do_maketx, staging, linearize, force): + def _process_texture(self, filepath, do_rstex, do_maketx, staging, linearize, force): """Process a single texture file on disk for publishing. This will: 1. Check whether it's already published, if so it will do hardlink From c43ec04003c7763d82d62944cc36c62512f11fb4 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Fri, 8 Apr 2022 07:50:49 +0300 Subject: [PATCH 13/81] add redshift processor call to generate .rstexbin --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 87e6625653..6ce8ff6052 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -548,6 +548,10 @@ class ExtractLook(openpype.api.Extractor): return converted, COPY, texture_hash + self.log.info("Generating .rstexbin file for %s .." % filepath) + # Generates Redshift optimized textures using Redshift processor + if do_rstex: + rstex(filepath) return filepath, COPY, texture_hash From 92c1ac7342d94889abb79a71a9be8b8750d7fba7 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 19 Apr 2022 21:33:11 +0300 Subject: [PATCH 14/81] refactor convertor function to abstract class and inherited class --- .../maya/plugins/publish/extract_look.py | 75 +++++++++++-------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index e0a5bff56c..7f23663721 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" +from abc import ABC, abstractmethod import os import sys import json @@ -43,50 +44,60 @@ def find_paths_by_hash(texture_hash): key = "data.sourceHashes.{0}".format(texture_hash) return io.distinct(key, {"type": "version"}) +class TextureProcessor(metaclass=ABC.ABCMeta): + def __init__(self): + #TODO: Figure out design for predetermined objects to be initialized. + + @abstractmethod + def process(self, filepath): -def rstex(source, *args): - """Make `.rstexbin` using `redshiftTextureProcessor` - with some default settings. + - This function requires the `REDSHIFT_COREDATAPATH` - to be in `PATH`. +class MakeRSTexBin(TextureProcessor): + + def process(source, *args): + """Make `.rstexbin` using `redshiftTextureProcessor` + with some default settings. - Args: - source (str): Path to source file. - *args: Additional arguments for `redshiftTextureProcessor`. + This function requires the `REDSHIFT_COREDATAPATH` + to be in `PATH`. - Returns: - str: Output of `redshiftTextureProcessor` command. + Args: + source (str): Path to source file. + *args: Additional arguments for `redshiftTextureProcessor`. - """ - if "REDSHIFT_COREDATAPATH" not in os.environ: - raise RuntimeError("Must have Redshift available.") + Returns: + str: Output of `redshiftTextureProcessor` command. - texture_processor_path = get_redshift_tool("TextureProcessor") + """ + if "REDSHIFT_COREDATAPATH" not in os.environ: + raise RuntimeError("Must have Redshift available.") - cmd = [ - texture_processor_path, - escape_space(source), - ] + texture_processor_path = get_redshift_tool("TextureProcessor") - cmd.extend(args) + cmd = [ + texture_processor_path, + escape_space(source), + ] - cmd = " ".join(cmd) + cmd.extend(args) - CREATE_NO_WINDOW = 0x08000000 - kwargs = dict(args=cmd, stderr=subprocess.STDOUT) + cmd = " ".join(cmd) - if sys.platform == "win32": - kwargs["creationflags"] = CREATE_NO_WINDOW - try: - out = subprocess.check_output(**kwargs) - except subprocess.CalledProcessError as exc: - print(exc) - import traceback + CREATE_NO_WINDOW = 0x08000000 + kwargs = dict(args=cmd, stderr=subprocess.STDOUT) - traceback.print_exc() - raise - return out + if sys.platform == "win32": + kwargs["creationflags"] = CREATE_NO_WINDOW + try: + processed_filepath = subprocess.check_output(**kwargs) + except subprocess.CalledProcessError as exc: + print(exc) + import traceback + + traceback.print_exc() + raise + return processed_filepath def maketx(source, destination, *args): From 74f2c78415b091cdbc9bd447549d20dc0b796235 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 19 Apr 2022 21:34:26 +0300 Subject: [PATCH 15/81] refactor tx conversion into class --- .../maya/plugins/publish/extract_look.py | 92 +++++++++---------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 7f23663721..54ab7b7877 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -99,65 +99,65 @@ class MakeRSTexBin(TextureProcessor): raise return processed_filepath +class MakeTX(TextureProcessor): + def process(source, destination, *args): + """Make `.tx` using `maketx` with some default settings. -def maketx(source, destination, *args): - """Make `.tx` using `maketx` with some default settings. + The settings are based on default as used in Arnold's + txManager in the scene. + This function requires the `maketx` executable to be + on the `PATH`. - The settings are based on default as used in Arnold's - txManager in the scene. - This function requires the `maketx` executable to be - on the `PATH`. + Args: + source (str): Path to source file. + destination (str): Writing destination path. + *args: Additional arguments for `maketx`. - Args: - source (str): Path to source file. - destination (str): Writing destination path. - *args: Additional arguments for `maketx`. + Returns: + str: Output of `maketx` command. - Returns: - str: Output of `maketx` command. + """ + from openpype.lib import get_oiio_tools_path - """ - from openpype.lib import get_oiio_tools_path + maketx_path = get_oiio_tools_path("maketx") - maketx_path = get_oiio_tools_path("maketx") + if not os.path.exists(maketx_path): + print( + "OIIO tool not found in {}".format(maketx_path)) + raise AssertionError("OIIO tool not found") - if not os.path.exists(maketx_path): - print( - "OIIO tool not found in {}".format(maketx_path)) - raise AssertionError("OIIO tool not found") + cmd = [ + maketx_path, + "-v", # verbose + "-u", # update mode + # unpremultiply before conversion (recommended when alpha present) + "--unpremult", + "--checknan", + # use oiio-optimized settings for tile-size, planarconfig, metadata + "--oiio", + "--filter lanczos3", + ] - cmd = [ - maketx_path, - "-v", # verbose - "-u", # update mode - # unpremultiply before conversion (recommended when alpha present) - "--unpremult", - "--checknan", - # use oiio-optimized settings for tile-size, planarconfig, metadata - "--oiio", - "--filter lanczos3", - ] + cmd.extend(args) + cmd.extend(["-o", escape_space(destination), escape_space(source)]) - cmd.extend(args) - cmd.extend(["-o", escape_space(destination), escape_space(source)]) + cmd = " ".join(cmd) - cmd = " ".join(cmd) + CREATE_NO_WINDOW = 0x08000000 # noqa + kwargs = dict(args=cmd, stderr=subprocess.STDOUT) - CREATE_NO_WINDOW = 0x08000000 # noqa - kwargs = dict(args=cmd, stderr=subprocess.STDOUT) + if sys.platform == "win32": + kwargs["creationflags"] = CREATE_NO_WINDOW + try: + processed_filepath = subprocess.check_output(**kwargs) + except subprocess.CalledProcessError as exc: + print(exc) + import traceback - if sys.platform == "win32": - kwargs["creationflags"] = CREATE_NO_WINDOW - try: - out = subprocess.check_output(**kwargs) - except subprocess.CalledProcessError as exc: - print(exc) - import traceback + traceback.print_exc() + raise - traceback.print_exc() - raise - - return out + return processed_filepath @contextlib.contextmanager From 7b1346e300c047654d6378c300916a4ac76acf56 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 19 Apr 2022 21:41:48 +0300 Subject: [PATCH 16/81] add processor list and adjust logic for more options later --- openpype/hosts/maya/plugins/publish/extract_look.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 54ab7b7877..4b77a47729 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -374,9 +374,15 @@ class ExtractLook(openpype.api.Extractor): # might be included more than once amongst the resources as they could # be the input file to multiple nodes. resources = instance.data["resources"] + # Specify texture processing executables to activate + processors = [] do_maketx = instance.data.get("maketx", False) + if do_maketx: + processors.append(MakeTX) # Option to convert textures to native redshift textures do_rstex = instance.data.get("rstex", False) + if do_rstex: + processors.append(MakeRSTexBin) # Collect all unique files used in the resources files_metadata = {} for resource in resources: From 0100ea5bf8496915875ad0f7ca90ec1bd93e88de Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 7 Jun 2022 15:46:30 +0300 Subject: [PATCH 17/81] Move redshift tool finder function to extractor. --- .../maya/plugins/publish/extract_look.py | 29 +++++++++++++++---- openpype/lib/vendor_bin_utils.py | 20 ------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 92d8f5ab17..357f7a4430 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -8,7 +8,7 @@ import tempfile import platform import contextlib import subprocess -from openpype.lib.vendor_bin_utils import get_redshift_tool +from openpype.lib.vendor_bin_utils import find_executable from collections import OrderedDict from maya import cmds # noqa @@ -46,15 +46,13 @@ def find_paths_by_hash(texture_hash): class TextureProcessor(metaclass=ABC.ABCMeta): def __init__(self): - #TODO: Figure out design for predetermined objects to be initialized. - + #TODO: Figure out design for predetermined objects to be initialized. + @abstractmethod def process(self, filepath): - class MakeRSTexBin(TextureProcessor): - def process(source, *args): """Make `.rstexbin` using `redshiftTextureProcessor` with some default settings. @@ -99,6 +97,7 @@ class MakeRSTexBin(TextureProcessor): raise return processed_filepath + class MakeTX(TextureProcessor): def process(source, destination, *args): """Make `.tx` using `maketx` with some default settings. @@ -603,3 +602,23 @@ class ExtractModelRenderSets(ExtractLook): self.scene_type = self.scene_type_prefix + self.scene_type return typ + + +def get_redshift_tool(tool_name): + """Path to redshift texture processor. + + On Windows it adds .exe extension if missing from tool argument. + + Args: + tool (string): Tool name. + + Returns: + str: Full path to redshift texture processor executable. + """ + redshift_tool_path = os.path.join( + os.environ["REDSHIFT_COREDATAPATH"], + "bin", + tool_name + ) + + return find_executable(redshift_tool_path) diff --git a/openpype/lib/vendor_bin_utils.py b/openpype/lib/vendor_bin_utils.py index cdc1290400..e5ab2872a0 100644 --- a/openpype/lib/vendor_bin_utils.py +++ b/openpype/lib/vendor_bin_utils.py @@ -123,26 +123,6 @@ def get_oiio_tools_path(tool="oiiotool"): return find_executable(os.path.join(oiio_dir, tool)) -def get_redshift_tool(tool_name): - """Path to redshift texture processor. - - On Windows it adds .exe extension if missing from tool argument. - - Args: - tool (string): Tool name. - - Returns: - str: Full path to redshift texture processor executable. - """ - redshift_tool_path = os.path.join( - os.environ["REDSHIFT_COREDATAPATH"], - "bin", - tool_name - ) - - return find_executable(redshift_tool_path) - - def get_ffmpeg_tool_path(tool="ffmpeg"): """Path to vendorized FFmpeg executable. From 2102e4f4fa86778542d96e9758dbb12967b6b762 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 7 Jun 2022 15:46:50 +0300 Subject: [PATCH 18/81] Add variable for redshift os path. --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 357f7a4430..69d7eb78af 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -615,8 +615,10 @@ def get_redshift_tool(tool_name): Returns: str: Full path to redshift texture processor executable. """ + redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] + redshift_tool_path = os.path.join( - os.environ["REDSHIFT_COREDATAPATH"], + redshift_os_path, "bin", tool_name ) From 406ac826e018ffcac07aa9145f5c532b86ddfd27 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 7 Jun 2022 16:09:53 +0300 Subject: [PATCH 19/81] Style fix --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 69d7eb78af..2e83af1437 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -46,7 +46,7 @@ def find_paths_by_hash(texture_hash): class TextureProcessor(metaclass=ABC.ABCMeta): def __init__(self): - #TODO: Figure out design for predetermined objects to be initialized. + # TODO: Figure out design for predetermined objects to be initialized. @abstractmethod def process(self, filepath): From 710ed3a889ee3d7043501d514d0cc3be3e8d571e Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Wed, 8 Jun 2022 10:37:46 +0300 Subject: [PATCH 20/81] Start moving processors logic. --- .../maya/plugins/publish/extract_look.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 2e83af1437..cce2643dba 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -44,6 +44,7 @@ def find_paths_by_hash(texture_hash): key = "data.sourceHashes.{0}".format(texture_hash) return legacy_io.distinct(key, {"type": "version"}) + class TextureProcessor(metaclass=ABC.ABCMeta): def __init__(self): # TODO: Figure out design for predetermined objects to be initialized. @@ -51,6 +52,8 @@ class TextureProcessor(metaclass=ABC.ABCMeta): @abstractmethod def process(self, filepath): + return processed_texture_path + class MakeRSTexBin(TextureProcessor): def process(source, *args): @@ -373,15 +376,6 @@ class ExtractLook(openpype.api.Extractor): # might be included more than once amongst the resources as they could # be the input file to multiple nodes. resources = instance.data["resources"] - # Specify texture processing executables to activate - processors = [] - do_maketx = instance.data.get("maketx", False) - if do_maketx: - processors.append(MakeTX) - # Option to convert textures to native redshift textures - do_rstex = instance.data.get("rstex", False) - if do_rstex: - processors.append(MakeRSTexBin) # Collect all unique files used in the resources files_metadata = {} for resource in resources: @@ -420,8 +414,7 @@ class ExtractLook(openpype.api.Extractor): source, mode, texture_hash = self._process_texture( filepath, - do_rstex, - do_maketx, + processors, staging=staging_dir, linearize=linearize, force=force_copy @@ -514,7 +507,7 @@ class ExtractLook(openpype.api.Extractor): resources_dir, basename + ext ) - def _process_texture(self, filepath, do_rstex, do_maketx, staging, linearize, force): + def _process_texture(self, filepath, processors, staging, linearize, force): """Process a single texture file on disk for publishing. This will: 1. Check whether it's already published, if so it will do hardlink @@ -546,7 +539,17 @@ class ExtractLook(openpype.api.Extractor): ("Paths not found on disk, " "skipping hardlink: %s") % (existing,) ) - + texture_files = self.collect_text + # Specify texture processing executables to activate + processors = [] + do_maketx = instance.data.get("maketx", False) + if do_maketx: + processors.append(MakeTX) + # Option to convert textures to native redshift textures + do_rstex = instance.data.get("rstex", False) + if do_rstex: + processors.append(MakeRSTexBin) + if do_maketx and ext != ".tx": # Produce .tx file in staging if source file is not .tx converted = os.path.join(staging, "resources", fname + ".tx") From 68caaa7528882654077a208b835a712f3e319850 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 10:56:26 +0300 Subject: [PATCH 21/81] move function --- .../maya/plugins/publish/extract_look.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index cce2643dba..d6c3588280 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -24,6 +24,28 @@ COPY = 1 HARDLINK = 2 +def get_redshift_tool(tool_name): + """Path to redshift texture processor. + + On Windows it adds .exe extension if missing from tool argument. + + Args: + tool (string): Tool name. + + Returns: + str: Full path to redshift texture processor executable. + """ + redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] + + redshift_tool_path = os.path.join( + redshift_os_path, + "bin", + tool_name + ) + + return find_executable(redshift_tool_path) + + def escape_space(path): """Ensure path is enclosed by quotes to allow paths with spaces""" return '"{}"'.format(path) if " " in path else path @@ -605,25 +627,3 @@ class ExtractModelRenderSets(ExtractLook): self.scene_type = self.scene_type_prefix + self.scene_type return typ - - -def get_redshift_tool(tool_name): - """Path to redshift texture processor. - - On Windows it adds .exe extension if missing from tool argument. - - Args: - tool (string): Tool name. - - Returns: - str: Full path to redshift texture processor executable. - """ - redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] - - redshift_tool_path = os.path.join( - redshift_os_path, - "bin", - tool_name - ) - - return find_executable(redshift_tool_path) From 29b69bcbb805702080fe54e52505fe09a3741f99 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 20:47:26 +0300 Subject: [PATCH 22/81] Class cleanup --- openpype/hosts/maya/plugins/publish/extract_look.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index d7b179daf2..be31deeb6e 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -69,12 +69,12 @@ def find_paths_by_hash(texture_hash): class TextureProcessor(metaclass=ABC.ABCMeta): def __init__(self): - # TODO: Figure out design for predetermined objects to be initialized. + pass @abstractmethod def process(self, filepath): - return processed_texture_path + pass class MakeRSTexBin(TextureProcessor): @@ -544,8 +544,7 @@ class ExtractLook(openpype.api.Extractor): fname, ext = os.path.splitext(os.path.basename(filepath)) args = [] - if do_maketx: - args.append("maketx") + texture_hash = openpype.api.source_hash(filepath, *args) # If source has been published before with the same settings, @@ -571,7 +570,7 @@ class ExtractLook(openpype.api.Extractor): do_rstex = instance.data.get("rstex", False) if do_rstex: processors.append(MakeRSTexBin) - + if do_maketx and ext != ".tx": # Produce .tx file in staging if source file is not .tx converted = os.path.join(staging, "resources", fname + ".tx") From e78314ca92283b38fd05d8995aeb9c06b460277e Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 20:49:07 +0300 Subject: [PATCH 23/81] Remove unused maketx code. --- .../maya/plugins/publish/extract_look.py | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index be31deeb6e..3e1f91f5b7 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -544,7 +544,6 @@ class ExtractLook(openpype.api.Extractor): fname, ext = os.path.splitext(os.path.basename(filepath)) args = [] - texture_hash = openpype.api.source_hash(filepath, *args) # If source has been published before with the same settings, @@ -571,32 +570,6 @@ class ExtractLook(openpype.api.Extractor): if do_rstex: processors.append(MakeRSTexBin) - if do_maketx and ext != ".tx": - # Produce .tx file in staging if source file is not .tx - converted = os.path.join(staging, "resources", fname + ".tx") - - if linearize: - self.log.info("tx: converting sRGB -> linear") - colorconvert = "--colorconvert sRGB linear" - else: - colorconvert = "" - - # Ensure folder exists - if not os.path.exists(os.path.dirname(converted)): - os.makedirs(os.path.dirname(converted)) - - self.log.info("Generating .tx file for %s .." % filepath) - maketx( - filepath, - converted, - # Include `source-hash` as string metadata - "-sattrib", - "sourceHash", - escape_space(texture_hash), - colorconvert, - ) - - return converted, COPY, texture_hash self.log.info("Generating .rstexbin file for %s .." % filepath) # Generates Redshift optimized textures using Redshift processor From 00d877e2dfca6b1e89a4b4d289b2621a6e8b2037 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 20:50:51 +0300 Subject: [PATCH 24/81] Move processors list --- .../maya/plugins/publish/extract_look.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 3e1f91f5b7..88c93a8e3b 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -433,7 +433,15 @@ class ExtractLook(openpype.api.Extractor): # if do_maketx: # color_space = "Raw" - + # Specify texture processing executables to activate + processors = [] + do_maketx = instance.data.get("maketx", False) + if do_maketx: + processors.append(MakeTX) + # Option to convert textures to native redshift textures + do_rstex = instance.data.get("rstex", False) + if do_rstex: + processors.append(MakeRSTexBin) source, mode, texture_hash = self._process_texture( filepath, processors, @@ -559,16 +567,6 @@ class ExtractLook(openpype.api.Extractor): ("Paths not found on disk, " "skipping hardlink: %s") % (existing,) ) - texture_files = self.collect_text - # Specify texture processing executables to activate - processors = [] - do_maketx = instance.data.get("maketx", False) - if do_maketx: - processors.append(MakeTX) - # Option to convert textures to native redshift textures - do_rstex = instance.data.get("rstex", False) - if do_rstex: - processors.append(MakeRSTexBin) self.log.info("Generating .rstexbin file for %s .." % filepath) From 2933e409c769f13afd0e1df778fdacd06cf7b848 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 20:59:19 +0300 Subject: [PATCH 25/81] Handle texture processing through processors separately --- openpype/hosts/maya/plugins/publish/extract_look.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 88c93a8e3b..73d661168a 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -442,6 +442,7 @@ class ExtractLook(openpype.api.Extractor): do_rstex = instance.data.get("rstex", False) if do_rstex: processors.append(MakeRSTexBin) + source, mode, texture_hash = self._process_texture( filepath, processors, @@ -567,12 +568,9 @@ class ExtractLook(openpype.api.Extractor): ("Paths not found on disk, " "skipping hardlink: %s") % (existing,) ) - - - self.log.info("Generating .rstexbin file for %s .." % filepath) - # Generates Redshift optimized textures using Redshift processor - if do_rstex: - rstex(filepath) + for processor in processors: + processor().process(filepath) + self.log.info("Generating .rstexbin file for %s .." % filepath) return filepath, COPY, texture_hash From b054175d6e834dabd7b3d8719eed901ab139062f Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 21:06:02 +0300 Subject: [PATCH 26/81] Reorganize functionality for do_maketx to linearize properly. --- .../maya/plugins/publish/extract_look.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 73d661168a..e90f9759f9 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -426,23 +426,20 @@ class ExtractLook(openpype.api.Extractor): for filepath in files_metadata: linearize = False + + # Specify texture processing executables to activate + processors = [] + do_maketx = instance.data.get("maketx", False) + do_rstex = instance.data.get("rstex", False) + if do_maketx: + processors.append(MakeTX) + # Option to convert textures to native redshift textures + if do_rstex: + processors.append(MakeRSTexBin) if do_maketx and files_metadata[filepath]["color_space"].lower() == "srgb": # noqa: E501 linearize = True # set its file node to 'raw' as tx will be linearized files_metadata[filepath]["color_space"] = "Raw" - - # if do_maketx: - # color_space = "Raw" - # Specify texture processing executables to activate - processors = [] - do_maketx = instance.data.get("maketx", False) - if do_maketx: - processors.append(MakeTX) - # Option to convert textures to native redshift textures - do_rstex = instance.data.get("rstex", False) - if do_rstex: - processors.append(MakeRSTexBin) - source, mode, texture_hash = self._process_texture( filepath, processors, From fd3125deda242f2676f1262fffb4e93212ffa300 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 21:07:26 +0300 Subject: [PATCH 27/81] Style fixes --- openpype/hosts/maya/plugins/publish/extract_look.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index e90f9759f9..018e45a01f 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -436,6 +436,7 @@ class ExtractLook(openpype.api.Extractor): # Option to convert textures to native redshift textures if do_rstex: processors.append(MakeRSTexBin) + if do_maketx and files_metadata[filepath]["color_space"].lower() == "srgb": # noqa: E501 linearize = True # set its file node to 'raw' as tx will be linearized From 41e7ac78aa340d4af45c48b036fb3ccf39c56f79 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Thu, 9 Jun 2022 22:11:12 +0300 Subject: [PATCH 28/81] adjust comment --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 018e45a01f..922e347561 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -568,7 +568,7 @@ class ExtractLook(openpype.api.Extractor): ) for processor in processors: processor().process(filepath) - self.log.info("Generating .rstexbin file for %s .." % filepath) + self.log.info("Generating texture file for %s .." % filepath) return filepath, COPY, texture_hash From 35be81615ee986b0b71482a8f13097fa469ab477 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Fri, 10 Jun 2022 10:51:19 +0300 Subject: [PATCH 29/81] Fix returned filepath --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 922e347561..7693e91765 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -567,9 +567,9 @@ class ExtractLook(openpype.api.Extractor): "skipping hardlink: %s") % (existing,) ) for processor in processors: - processor().process(filepath) + processed_path = processor().process(filepath) self.log.info("Generating texture file for %s .." % filepath) - return filepath, COPY, texture_hash + return processed_path, COPY, texture_hash class ExtractModelRenderSets(ExtractLook): From cfb90934289a742572d93323d48df6ae061acdb7 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Fri, 10 Jun 2022 20:32:58 +0300 Subject: [PATCH 30/81] Append processors check, append path return. --- openpype/hosts/maya/plugins/publish/extract_look.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 7693e91765..c72aede0d4 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -566,9 +566,13 @@ class ExtractLook(openpype.api.Extractor): ("Paths not found on disk, " "skipping hardlink: %s") % (existing,) ) - for processor in processors: - processed_path = processor().process(filepath) - self.log.info("Generating texture file for %s .." % filepath) + + if bool(processors): + for processor in processors: + processed_path = processor().process(filepath) + self.log.info("Generating texture file for %s .." % filepath) + return processed_path + return processed_path, COPY, texture_hash From eb9994484e94f9e595d95fac302da49cef986cc4 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Fri, 10 Jun 2022 20:35:53 +0300 Subject: [PATCH 31/81] Style fix --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index c72aede0d4..6d8b6f1d8e 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -536,7 +536,7 @@ class ExtractLook(openpype.api.Extractor): resources_dir, basename + ext ) - def _process_texture(self, filepath, processors, staging, linearize, force): + def _process_texture(self, filepath, processors, staging, linearize, force): # noqa """Process a single texture file on disk for publishing. This will: 1. Check whether it's already published, if so it will do hardlink From e5fdd9e9302fe82620b5fb0c94c4ce1188c4a731 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 26 Jul 2022 20:45:54 +0300 Subject: [PATCH 32/81] Syntax fix. --- openpype/hosts/maya/plugins/publish/extract_look.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 6d8b6f1d8e..3fe8139dd6 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" -from abc import ABC, abstractmethod +import abc import os import sys import json @@ -67,11 +67,11 @@ def find_paths_by_hash(texture_hash): return legacy_io.distinct(key, {"type": "version"}) -class TextureProcessor(metaclass=ABC.ABCMeta): +class TextureProcessor(abc.ABCMeta): def __init__(self): pass - @abstractmethod + @abc.abstractmethod def process(self, filepath): pass From ab14895753c72177ebdfc7ff4f6e8b0bb1eb68ec Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 26 Jul 2022 23:27:11 +0300 Subject: [PATCH 33/81] Change metaclass inheritance formatting to 2.7 --- openpype/hosts/maya/plugins/publish/extract_look.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 3fe8139dd6..be6d863878 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" -import abc +from abc import ABCMeta, abstractmethod import os import sys import json @@ -67,11 +67,13 @@ def find_paths_by_hash(texture_hash): return legacy_io.distinct(key, {"type": "version"}) -class TextureProcessor(abc.ABCMeta): +class TextureProcessor(object): + __metaclass__ = ABCMeta + def __init__(self): pass - @abc.abstractmethod + @abstractmethod def process(self, filepath): pass From c471bbe71e20f09df30c1ea734a958f3a308e3f6 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Wed, 27 Jul 2022 02:26:18 +0300 Subject: [PATCH 34/81] Fix inheritance with `six`, adjust processing code --- .../maya/plugins/publish/extract_look.py | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index be6d863878..19e0bd9568 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" from abc import ABCMeta, abstractmethod +import six import os import sys import json @@ -67,9 +68,9 @@ def find_paths_by_hash(texture_hash): return legacy_io.distinct(key, {"type": "version"}) -class TextureProcessor(object): - __metaclass__ = ABCMeta - +@six.add_metaclass(ABCMeta) +class TextureProcessor: + @abstractmethod def __init__(self): pass @@ -80,7 +81,10 @@ class TextureProcessor(object): class MakeRSTexBin(TextureProcessor): - def process(source, *args): + def __init__(self): + super(TextureProcessor, self).__init__() + + def process(self, source, *args): """Make `.rstexbin` using `redshiftTextureProcessor` with some default settings. @@ -126,7 +130,10 @@ class MakeRSTexBin(TextureProcessor): class MakeTX(TextureProcessor): - def process(source, destination, *args): + def __init__(self): + super(TextureProcessor, self).__init__() + + def process(self, source, destination, *args): """Make `.tx` using `maketx` with some default settings. The settings are based on default as used in Arnold's @@ -558,6 +565,10 @@ class ExtractLook(openpype.api.Extractor): # If source has been published before with the same settings, # then don't reprocess but hardlink from the original existing = find_paths_by_hash(texture_hash) + # if processors["do_maketx"]: + # Produce .tx file in staging if source file is not .tx + + if existing and not force: self.log.info("Found hash in database, preparing hardlink..") source = next((p for p in existing if os.path.exists(p)), None) @@ -571,9 +582,15 @@ class ExtractLook(openpype.api.Extractor): if bool(processors): for processor in processors: - processed_path = processor().process(filepath) - self.log.info("Generating texture file for %s .." % filepath) - return processed_path + if processor == MakeTX: + converted = os.path.join(staging, "resources", fname + ".tx") + processed_path = processor().process(converted, filepath) + self.log.info("Generating texture file for %s .." % filepath) # noqa + return processed_path + elif processor == MakeRSTexBin: + processed_path = processor().process(filepath) + self.log.info("Generating texture file for %s .." % filepath) # noqa + return processed_path return processed_path, COPY, texture_hash From 5f4d06baecce44fd735e1a26770f0a617c0284fb Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Wed, 27 Jul 2022 02:48:43 +0300 Subject: [PATCH 35/81] Remove unnecessary comment, style fixes. --- openpype/hosts/maya/plugins/publish/extract_look.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 19e0bd9568..bb4335a3d5 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -172,7 +172,7 @@ class MakeTX(TextureProcessor): ] cmd.extend(args) - cmd.extend(["-o", escape_space(destination), escape_space(source)]) + cmd.extend([escape_space(source), "-o", escape_space(destination)]) cmd = " ".join(cmd) @@ -188,6 +188,8 @@ class MakeTX(TextureProcessor): import traceback traceback.print_exc() + print(exc.returncode) + print(exc.output) raise return processed_filepath @@ -565,9 +567,6 @@ class ExtractLook(openpype.api.Extractor): # If source has been published before with the same settings, # then don't reprocess but hardlink from the original existing = find_paths_by_hash(texture_hash) - # if processors["do_maketx"]: - # Produce .tx file in staging if source file is not .tx - if existing and not force: self.log.info("Found hash in database, preparing hardlink..") @@ -583,8 +582,8 @@ class ExtractLook(openpype.api.Extractor): if bool(processors): for processor in processors: if processor == MakeTX: - converted = os.path.join(staging, "resources", fname + ".tx") - processed_path = processor().process(converted, filepath) + converted = os.path.join(staging, "resources", fname + ".tx") # noqa + processed_path = processor().process(filepath, converted) self.log.info("Generating texture file for %s .." % filepath) # noqa return processed_path elif processor == MakeRSTexBin: From 9b0cc4dfac23100fa3979b53418734c39399c5ca Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 1 Aug 2022 03:28:26 +0300 Subject: [PATCH 36/81] Continue refactor --- .../maya/plugins/publish/extract_look.py | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 136b64a547..8d30268619 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -615,12 +615,29 @@ class ExtractLook(openpype.api.Extractor): ("Paths not found on disk, " "skipping hardlink: %s") % (existing,) ) + config_path = get_ocio_config_path("nuke-default") + color_config = "--colorconfig {0}".format(config_path) + # Ensure folder exists + if linearize: + self.log.info("tx: converting sRGB -> linear") + colorconvert = "--colorconvert sRGB linear" + else: + colorconvert = "" + + converted = os.path.join(staging, "resources", fname + ".tx") + if not os.path.exists(os.path.dirname(converted)): + os.makedirs(os.path.dirname(converted)) if bool(processors): for processor in processors: if processor == MakeTX: - converted = os.path.join(staging, "resources", fname + ".tx") # noqa - processed_path = processor().process(filepath, converted) + processed_path = processor().process(filepath, + converted, + "--sattrib", + "sourceHash %", + escape_space(texture_hash), # noqa + colorconvert, + color_config) self.log.info("Generating texture file for %s .." % filepath) # noqa return processed_path elif processor == MakeRSTexBin: @@ -628,27 +645,18 @@ class ExtractLook(openpype.api.Extractor): self.log.info("Generating texture file for %s .." % filepath) # noqa return processed_path - return processed_path, COPY, texture_hash - config_path = get_ocio_config_path("nuke-default") - color_config = "--colorconfig {0}".format(config_path) - # Ensure folder exists - if not os.path.exists(os.path.dirname(converted)): - os.makedirs(os.path.dirname(converted)) - - self.log.info("Generating .tx file for %s .." % filepath) - maketx( - filepath, - converted, - # Include `source-hash` as string metadata - "--sattrib", - "sourceHash", - escape_space(texture_hash), - colorconvert, - color_config - ) - - return converted, COPY, texture_hash + # self.log.info("Generating .tx file for %s .." % filepath) + # maketx( + # filepath, + # converted, + # # Include `source-hash` as string metadata + # "--sattrib", + # "sourceHash", + # escape_space(texture_hash), + # colorconvert, + # color_config + # ) return filepath, COPY, texture_hash From c35f1cfe3c295d7915e4a2b856f48ecae85cab38 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 1 Aug 2022 04:09:09 +0300 Subject: [PATCH 37/81] Remove leftover code --- .../maya/plugins/publish/extract_look.py | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 8d30268619..ecbb070916 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -114,27 +114,10 @@ class MakeRSTexBin(TextureProcessor): This function requires the `REDSHIFT_COREDATAPATH` to be in `PATH`. - cmd = [ - maketx_path, - "-v", # verbose - "-u", # update mode - # unpremultiply before conversion (recommended when alpha present) - "--unpremult", - "--checknan", - # use oiio-optimized settings for tile-size, planarconfig, metadata - "--oiio", - "--filter lanczos3", - escape_space(source) - ] Args: source (str): Path to source file. *args: Additional arguments for `redshiftTextureProcessor`. - cmd.extend(args) - cmd.extend(["-o", escape_space(destination)]) - Returns: - str: Output of `redshiftTextureProcessor` command. - """ if "REDSHIFT_COREDATAPATH" not in os.environ: raise RuntimeError("Must have Redshift available.") @@ -634,7 +617,7 @@ class ExtractLook(openpype.api.Extractor): processed_path = processor().process(filepath, converted, "--sattrib", - "sourceHash %", + "sourceHash", escape_space(texture_hash), # noqa colorconvert, color_config) @@ -645,19 +628,6 @@ class ExtractLook(openpype.api.Extractor): self.log.info("Generating texture file for %s .." % filepath) # noqa return processed_path - - # self.log.info("Generating .tx file for %s .." % filepath) - # maketx( - # filepath, - # converted, - # # Include `source-hash` as string metadata - # "--sattrib", - # "sourceHash", - # escape_space(texture_hash), - # colorconvert, - # color_config - # ) - return filepath, COPY, texture_hash From 099dfba8fa19dca4bc1cc9a30632f659854c9300 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 1 Aug 2022 05:06:58 +0300 Subject: [PATCH 38/81] Fix return bug --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index ecbb070916..53b6dcbf35 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -622,11 +622,11 @@ class ExtractLook(openpype.api.Extractor): colorconvert, color_config) self.log.info("Generating texture file for %s .." % filepath) # noqa - return processed_path + return processed_path, COPY, texture_hash elif processor == MakeRSTexBin: processed_path = processor().process(filepath) self.log.info("Generating texture file for %s .." % filepath) # noqa - return processed_path + return processed_path, COPY, texture_hash return filepath, COPY, texture_hash From 8995dcdff4a9a6c5c951958fb71b99ee93b44029 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 1 Aug 2022 12:26:44 +0300 Subject: [PATCH 39/81] Check for return value, adjust argument --- .../hosts/maya/plugins/publish/extract_look.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 53b6dcbf35..48af644326 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" from abc import ABCMeta, abstractmethod + import six import os import sys @@ -189,10 +190,11 @@ class MakeTX(TextureProcessor): # use oiio-optimized settings for tile-size, planarconfig, metadata "--oiio", "--filter lanczos3", + escape_space(source) ] cmd.extend(args) - cmd.extend([escape_space(source), "-o", escape_space(destination)]) + cmd.extend(["-o", escape_space(destination)]) cmd = " ".join(cmd) @@ -620,13 +622,21 @@ class ExtractLook(openpype.api.Extractor): "sourceHash", escape_space(texture_hash), # noqa colorconvert, - color_config) + color_config, + ) self.log.info("Generating texture file for %s .." % filepath) # noqa - return processed_path, COPY, texture_hash + self.log.info(converted) + if processed_path: + return processed_path + else: + self.log.info("maketx has returned nothing") elif processor == MakeRSTexBin: processed_path = processor().process(filepath) self.log.info("Generating texture file for %s .." % filepath) # noqa - return processed_path, COPY, texture_hash + if processed_path: + return processed_path + else: + self.log.info("redshift texture converter has returned nothing") # noqa return filepath, COPY, texture_hash From 3cd7fd503c0cb94ef3f4da86a3c465ff6bc26cb1 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Mon, 15 Aug 2022 04:23:24 +0300 Subject: [PATCH 40/81] Fix assignment bug, remove unnecessary class sugar --- openpype/hosts/maya/plugins/publish/extract_look.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 48af644326..c092e2ac25 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -94,7 +94,7 @@ def find_paths_by_hash(texture_hash): @six.add_metaclass(ABCMeta) class TextureProcessor: - @abstractmethod + def __init__(self): pass @@ -106,7 +106,7 @@ class TextureProcessor: class MakeRSTexBin(TextureProcessor): def __init__(self): - super(TextureProcessor, self).__init__() + super(MakeRSTexBin, self).__init__() def process(self, source, *args): """Make `.rstexbin` using `redshiftTextureProcessor` @@ -152,7 +152,7 @@ class MakeRSTexBin(TextureProcessor): class MakeTX(TextureProcessor): def __init__(self): - super(TextureProcessor, self).__init__() + super(MakeTX, self).__init__() def process(self, source, destination, *args): """Make `.tx` using `maketx` with some default settings. @@ -627,18 +627,18 @@ class ExtractLook(openpype.api.Extractor): self.log.info("Generating texture file for %s .." % filepath) # noqa self.log.info(converted) if processed_path: - return processed_path + return processed_path, COPY, texture_hash else: self.log.info("maketx has returned nothing") elif processor == MakeRSTexBin: processed_path = processor().process(filepath) self.log.info("Generating texture file for %s .." % filepath) # noqa if processed_path: - return processed_path + return processed_path, COPY, texture_hash else: self.log.info("redshift texture converter has returned nothing") # noqa - return filepath, COPY, texture_hash + return processed_path, COPY, texture_hash class ExtractModelRenderSets(ExtractLook): From 9ec42cb4376e0e14eb990cfd22c6a9f1c38e0575 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 23 Aug 2022 13:05:37 +0300 Subject: [PATCH 41/81] Fix bugs --- .../hosts/maya/plugins/publish/extract_look.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 52751eea81..19765d4396 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -125,7 +125,7 @@ class MakeRSTexBin(TextureProcessor): if "REDSHIFT_COREDATAPATH" not in os.environ: raise RuntimeError("Must have Redshift available.") - texture_processor_path = get_redshift_tool("TextureProcessor") + texture_processor_path = get_redshift_tool("redshiftTextureProcessor") cmd = [ texture_processor_path, @@ -142,14 +142,14 @@ class MakeRSTexBin(TextureProcessor): if sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW try: - processed_filepath = subprocess.check_output(**kwargs) + subprocess.check_output(**kwargs) except subprocess.CalledProcessError as exc: print(exc) import traceback traceback.print_exc() raise - return processed_filepath + return source class MakeTX(TextureProcessor): @@ -206,7 +206,7 @@ class MakeTX(TextureProcessor): if sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW try: - processed_filepath = subprocess.check_output(**kwargs) + subprocess.check_output(**kwargs) except subprocess.CalledProcessError as exc: print(exc) import traceback @@ -216,7 +216,7 @@ class MakeTX(TextureProcessor): print(exc.output) raise - return processed_filepath + return destination @contextlib.contextmanager @@ -629,7 +629,7 @@ class ExtractLook(openpype.api.Extractor): if bool(processors): for processor in processors: - if processor == MakeTX: + if processor is MakeTX: processed_path = processor().process(filepath, converted, "--sattrib", @@ -644,7 +644,7 @@ class ExtractLook(openpype.api.Extractor): return processed_path, COPY, texture_hash else: self.log.info("maketx has returned nothing") - elif processor == MakeRSTexBin: + elif processor is MakeRSTexBin: processed_path = processor().process(filepath) self.log.info("Generating texture file for %s .." % filepath) # noqa if processed_path: From cc62035e1be2e02d38c3e1972bd37f458f70eb83 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 23 Aug 2022 13:22:59 +0300 Subject: [PATCH 42/81] Fix destination finding for copying resrouce. --- .../maya/plugins/publish/extract_look.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 19765d4396..7f79867cea 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -483,9 +483,15 @@ class ExtractLook(openpype.api.Extractor): linearize=linearize, force=force_copy ) - destination = self.resource_destination(instance, - source, - do_maketx) + for processor in processors: + if processor is MakeTX: + destination = self.resource_destination( + instance, source, MakeTX + ) + elif processor is MakeRSTexBin: + destination = self.resource_destination( + instance, source, MakeRSTexBin + ) # Force copy is specified. if force_copy: @@ -512,9 +518,15 @@ class ExtractLook(openpype.api.Extractor): if source not in destinations: # Cache destination as source resource might be included # multiple times - destinations[source] = self.resource_destination( - instance, source, do_maketx - ) + for processor in processors: + if processor is MakeTX: + destinations[source] = self.resource_destination( + instance, source, MakeTX + ) + elif processor is MakeRSTexBin: + destinations[source] = self.resource_destination( + instance, source, MakeRSTexBin + ) # Preserve color space values (force value after filepath change) # This will also trigger in the same order at end of context to @@ -555,7 +567,7 @@ class ExtractLook(openpype.api.Extractor): "attrRemap": remap, } - def resource_destination(self, instance, filepath, do_maketx): + def resource_destination(self, instance, filepath, processor): """Get resource destination path. This is utility function to change path if resource file name is @@ -564,7 +576,7 @@ class ExtractLook(openpype.api.Extractor): Args: instance: Current Instance. filepath (str): Resource path - do_maketx (bool): Flag if resource is processed by `maketx`. + processor: Texture processor converting resource. Returns: str: Path to resource file @@ -576,8 +588,10 @@ class ExtractLook(openpype.api.Extractor): basename, ext = os.path.splitext(os.path.basename(filepath)) # If `maketx` then the texture will always end with .tx - if do_maketx: + if processor == MakeTX: ext = ".tx" + elif processor == MakeRSTexBin: + ext = ".rstexbin" return os.path.join( resources_dir, basename + ext From ad8177cee3c7ee6241b995b781b9dc93fead19b1 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 23 Aug 2022 16:10:44 +0300 Subject: [PATCH 43/81] Removed submodule vendor/configs/OpenColorIO-Configs --- vendor/configs/OpenColorIO-Configs | 1 - 1 file changed, 1 deletion(-) delete mode 160000 vendor/configs/OpenColorIO-Configs diff --git a/vendor/configs/OpenColorIO-Configs b/vendor/configs/OpenColorIO-Configs deleted file mode 160000 index 0bb079c08b..0000000000 --- a/vendor/configs/OpenColorIO-Configs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0bb079c08be410030669cbf5f19ff869b88af953 From 590538eb128dc8e7e8e6eee0b8d16ab593530e5e Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 4 Oct 2022 11:07:09 +0300 Subject: [PATCH 44/81] Fix import bug --- openpype/hosts/maya/plugins/publish/extract_look.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 7e640c375d..f3128306ee 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -11,13 +11,14 @@ import platform import contextlib import subprocess from openpype.lib.vendor_bin_utils import find_executable +from openpype.lib import source_hash from collections import OrderedDict from maya import cmds # noqa import pyblish.api -from openpype.lib import source_hash, run_subprocess + from openpype.pipeline import legacy_io, publish from openpype.hosts.maya.api import lib @@ -93,7 +94,6 @@ def find_paths_by_hash(texture_hash): key = "data.sourceHashes.{0}".format(texture_hash) return legacy_io.distinct(key, {"type": "version"}) - @six.add_metaclass(ABCMeta) class TextureProcessor: @@ -612,7 +612,7 @@ class ExtractLook(publish.Extractor): fname, ext = os.path.splitext(os.path.basename(filepath)) args = [] - texture_hash = openpype.api.source_hash(filepath, *args) + texture_hash = source_hash(filepath, *args) # If source has been published before with the same settings, # then don't reprocess but hardlink from the original From 035281fa4c939e6e56d356a97535cf837cc933a2 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 4 Oct 2022 11:08:27 +0300 Subject: [PATCH 45/81] Style fix --- openpype/hosts/maya/plugins/publish/extract_look.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index f3128306ee..6b0fe36d8f 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -94,6 +94,7 @@ def find_paths_by_hash(texture_hash): key = "data.sourceHashes.{0}".format(texture_hash) return legacy_io.distinct(key, {"type": "version"}) + @six.add_metaclass(ABCMeta) class TextureProcessor: From 18435fa7065065d8446c3b0448f664f9ef4a8123 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 4 Oct 2022 14:11:30 +0300 Subject: [PATCH 46/81] Add validation to check for texture --- .../plugins/publish/validate_look_contents.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/validate_look_contents.py b/openpype/hosts/maya/plugins/publish/validate_look_contents.py index 53501d11e5..837e8ea3a2 100644 --- a/openpype/hosts/maya/plugins/publish/validate_look_contents.py +++ b/openpype/hosts/maya/plugins/publish/validate_look_contents.py @@ -1,6 +1,7 @@ import pyblish.api import openpype.hosts.maya.api.action from openpype.pipeline.publish import ValidateContentsOrder +from maya import cmds # noqa class ValidateLookContents(pyblish.api.InstancePlugin): @@ -85,6 +86,7 @@ class ValidateLookContents(pyblish.api.InstancePlugin): invalid.add(instance.name) return list(invalid) + @classmethod def validate_looks(cls, instance): @@ -112,3 +114,23 @@ class ValidateLookContents(pyblish.api.InstancePlugin): invalid.append(node) return invalid + + @classmethod + def validate_renderer(cls, instance): + + renderer = cmds.getAttr( + 'defaultRenderGlobals.currentRenderer').lower() + do_maketx = instance.data.get("maketx", False) + do_rstex = instance.data.get("rstex", False) + processors = [] + + if do_maketx: + processors.append('arnold') + if do_rstex: + processors.append('redshift') + + for processor in processors: + if processor == renderer: + continue + else: + cls.log.error("Converted texture does not match current renderer.") # noqa From 3728ce0a586820bd0e7619dd7acc5cff82e0f918 Mon Sep 17 00:00:00 2001 From: Allan Ihsan Date: Tue, 4 Oct 2022 14:16:54 +0300 Subject: [PATCH 47/81] Style fix --- openpype/hosts/maya/plugins/publish/validate_look_contents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/validate_look_contents.py b/openpype/hosts/maya/plugins/publish/validate_look_contents.py index 837e8ea3a2..5e242ed3d2 100644 --- a/openpype/hosts/maya/plugins/publish/validate_look_contents.py +++ b/openpype/hosts/maya/plugins/publish/validate_look_contents.py @@ -119,7 +119,7 @@ class ValidateLookContents(pyblish.api.InstancePlugin): def validate_renderer(cls, instance): renderer = cmds.getAttr( - 'defaultRenderGlobals.currentRenderer').lower() + 'defaultRenderGlobals.currentRenderer').lower() do_maketx = instance.data.get("maketx", False) do_rstex = instance.data.get("rstex", False) processors = [] From 2baabed6be7eb6977cb7fcb0d69b7c79e1b18327 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 24 Mar 2023 18:50:09 +0100 Subject: [PATCH 48/81] Work in progress extract look cleanup - Fix file hashing so it includes arguments to maketx - Fix maketx destination colorspace when OCIO is enabled - Use pre-collected colorspaces of the resources instead of trying to retrieve again - Fix colorspace attributes being reinterpreted by maya on export (fix remapping) - Fix support for checking config path of maya default OCIO config (due to using `lib.get_color_management_preferences` which remaps that path) --- .../maya/plugins/publish/extract_look.py | 413 ++++++++++-------- 1 file changed, 220 insertions(+), 193 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 447c9a615c..5c03aa5a5a 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -149,22 +149,6 @@ class ExtractLook(publish.Extractor): scene_type = "ma" look_data_type = "json" - @staticmethod - def get_renderer_name(): - """Get renderer name from Maya. - - Returns: - str: Renderer name. - - """ - renderer = cmds.getAttr( - "defaultRenderGlobals.currentRenderer" - ).lower() - # handle various renderman names - if renderer.startswith("renderman"): - renderer = "renderman" - return renderer - def get_maya_scene_type(self, instance): """Get Maya scene type from settings. @@ -209,11 +193,9 @@ class ExtractLook(publish.Extractor): maya_path = os.path.join(dir_path, maya_fname) json_path = os.path.join(dir_path, json_fname) - self.log.info("Performing extraction..") - # Remove all members of the sets so they are not included in the # exported file by accident - self.log.info("Extract sets (%s) ..." % _scene_type) + self.log.info("Processing sets..") lookdata = instance.data["lookData"] relationships = lookdata["relationships"] sets = list(relationships.keys()) @@ -221,6 +203,7 @@ class ExtractLook(publish.Extractor): self.log.info("No sets found") return + self.log.debug("Processing resources..") results = self.process_resources(instance, staging_dir=dir_path) transfers = results["fileTransfers"] hardlinks = results["fileHardlinks"] @@ -228,6 +211,7 @@ class ExtractLook(publish.Extractor): remap = results["attrRemap"] # Extract in correct render layer + self.log.info("Extracting look maya scene file: {}".format(maya_path)) layer = instance.data.get("renderlayer", "defaultRenderLayer") with lib.renderlayer(layer): # TODO: Ensure membership edits don't become renderlayer overrides @@ -299,40 +283,39 @@ class ExtractLook(publish.Extractor): # Source hash for the textures instance.data["sourceHashes"] = hashes - """ - self.log.info("Returning colorspaces to their original values ...") - for attr, value in remap.items(): - self.log.info(" - {}: {}".format(attr, value)) - cmds.setAttr(attr, value, type="string") - """ self.log.info("Extracted instance '%s' to: %s" % (instance.name, maya_path)) - def process_resources(self, instance, staging_dir): + def _set_resource_result_colorspace(self, resource, colorspace): + """Update resource resulting colorspace after texture processing""" + if "result_colorspace" in resource: + if resource["result_colorspace"] == colorspace: + return + + self.log.warning( + "Resource already has a resulting colorspace but is now " + "being overridden to a new one: {} -> {}".format( + resource["result_colorspace"], colorspace + ) + ) + resource["result_colorspace"] = colorspace + + def process_resources(self, instance, staging_dir): + """Process all resources in the instance. + + It is assumed that all resources are nodes using file textures. + + Extract the textures to transfer, possibly convert with maketx and + remap the node paths to the destination path. Note that a source + might be included more than once amongst the resources as they could + be the input file to multiple nodes. + + """ - # Extract the textures to transfer, possibly convert with maketx and - # remap the node paths to the destination path. Note that a source - # might be included more than once amongst the resources as they could - # be the input file to multiple nodes. resources = instance.data["resources"] do_maketx = instance.data.get("maketx", False) + color_management = lib.get_color_management_preferences() - # Collect all unique files used in the resources - files_metadata = {} - for resource in resources: - # Preserve color space values (force value after filepath change) - # This will also trigger in the same order at end of context to - # ensure after context it's still the original value. - color_space = resource.get("color_space") - - for f in resource["files"]: - files_metadata[os.path.normpath(f)] = { - "color_space": color_space} - - # Process the resource files - transfers = [] - hardlinks = [] - hashes = {} # Temporary fix to NOT create hardlinks on windows machines if platform.system().lower() == "windows": self.log.info( @@ -342,58 +325,83 @@ class ExtractLook(publish.Extractor): else: force_copy = instance.data.get("forceCopy", False) - for filepath in files_metadata: - - linearize = False - # if OCIO color management enabled - # it won't take the condition of the files_metadata - - ocio_maya = cmds.colorManagementPrefs(q=True, - cmConfigFileEnabled=True, - cmEnabled=True) - - if do_maketx and not ocio_maya: - if files_metadata[filepath]["color_space"].lower() == "srgb": # noqa: E501 - linearize = True - # set its file node to 'raw' as tx will be linearized - files_metadata[filepath]["color_space"] = "Raw" - - # if do_maketx: - # color_space = "Raw" - - source, mode, texture_hash = self._process_texture( - filepath, - resource, - do_maketx, - staging=staging_dir, - linearize=linearize, - force=force_copy - ) - destination = self.resource_destination(instance, - source, - do_maketx) - - # Force copy is specified. - if force_copy: - mode = COPY - - if mode == COPY: - transfers.append((source, destination)) - self.log.info('file will be copied {} -> {}'.format( - source, destination)) - elif mode == HARDLINK: - hardlinks.append((source, destination)) - self.log.info('file will be hardlinked {} -> {}'.format( - source, destination)) - - # Store the hashes from hash to destination to include in the - # database - hashes[texture_hash] = destination - - # Remap the resources to the destination path (change node attributes) + # Process all resource's individual files + processed_files = {} + transfers = [] + hardlinks = [] + hashes = {} destinations = {} - remap = OrderedDict() # needs to be ordered, see color space values + remap = OrderedDict() for resource in resources: + colorspace = resource["color_space"] + + for filepath in resource["files"]: + filepath = os.path.normpath(filepath) + + if filepath in processed_files: + # The file was already processed, likely due to usage by + # another resource in the scene. We confirm here it + # didn't do color spaces different than the current + # resource. + processed_file = processed_files[filepath] + processed_colorspace = processed_file["color_space"] + processed_result_colorspace = processed_file["result_color_space"] + self.log.debug( + "File was already processed. Likely used by another " + "resource too: {}".format(filepath) + ) + + if colorspace != processed_file["color_space"]: + self.log.warning( + "File was already processed but using another" + "colorspace: {} <-> {}" + "".format(colorspace, processed_colorspace)) + + self._set_resource_result_colorspace( + resource, colorspace=processed_result_colorspace + ) + continue + + texture_result = self._process_texture( + filepath, + do_maketx=do_maketx, + staging_dir=staging_dir, + force_copy=force_copy, + color_management=color_management, + colorspace=colorspace + ) + source, mode, texture_hash, result_colorspace = texture_result + destination = self.resource_destination(instance, + source, + do_maketx) + + # Set the resulting color space on the resource + self._set_resource_result_colorspace( + resource, colorspace=result_colorspace + ) + + processed_files[filepath] = { + "color_space": colorspace, + "result_color_space": result_colorspace, + } + + # Force copy is specified. + if force_copy: + mode = COPY + + if mode == COPY: + transfers.append((source, destination)) + self.log.info('file will be copied {} -> {}'.format( + source, destination)) + elif mode == HARDLINK: + hardlinks.append((source, destination)) + self.log.info('file will be hardlinked {} -> {}'.format( + source, destination)) + + # Store the hashes from hash to destination to include in the + # database + hashes[texture_hash] = destination + source = os.path.normpath(resource["source"]) if source not in destinations: # Cache destination as source resource might be included @@ -402,35 +410,23 @@ class ExtractLook(publish.Extractor): instance, source, do_maketx ) + # Set up remapping attributes for the node during the publish + # The order of these can be important if one attribute directly + # affects another, e.g. we set colorspace after filepath because + # maya sometimes tries to guess the colorspace when changing + # filepaths (which is avoidable, but we don't want to have those + # attributes changed in the resulting publish) + # Remap filepath to publish destination + filepath_attr = resource["attribute"] + remap[filepath_attr] = destinations[source] + # Preserve color space values (force value after filepath change) # This will also trigger in the same order at end of context to # ensure after context it's still the original value. - color_space_attr = resource["node"] + ".colorSpace" - try: - color_space = cmds.getAttr(color_space_attr) - except ValueError: - # node doesn't have color space attribute - color_space = "Raw" - else: - # get the resolved files - metadata = files_metadata.get(source) - # if the files are unresolved from `source` - # assume color space from the first file of - # the resource - if not metadata: - first_file = next(iter(resource.get( - "files", [])), None) - if not first_file: - continue - first_filepath = os.path.normpath(first_file) - metadata = files_metadata[first_filepath] - if metadata["color_space"] == "Raw": - # set color space to raw if we linearized it - color_space = "Raw" - # Remap file node filename to destination - remap[color_space_attr] = color_space - attr = resource["attribute"] - remap[attr] = destinations[source] + node = resource["node"] + if cmds.attributeQuery("colorSpace", node=node, exists=True): + color_space_attr = "{}.colorSpace".format(node) + remap[color_space_attr] = resource["result_color_space"] self.log.info("Finished remapping destinations ...") @@ -469,91 +465,115 @@ class ExtractLook(publish.Extractor): resources_dir, basename + ext ) - def _process_texture(self, filepath, resource, - do_maketx, staging, linearize, force): - """Process a single texture file on disk for publishing. - This will: - 1. Check whether it's already published, if so it will do hardlink - 2. If not published and maketx is enabled, generate a new .tx file. - 3. Compute the destination path for the source file. - Args: - filepath (str): The source file path to process. - do_maketx (bool): Whether to produce a .tx file - Returns: - """ - - fname, ext = os.path.splitext(os.path.basename(filepath)) - - args = [] - if do_maketx: - args.append("maketx") - texture_hash = source_hash(filepath, *args) + def _get_existing_hashed_texture(self, texture_hash): + """Return the first found filepath from a texture hash""" # If source has been published before with the same settings, # then don't reprocess but hardlink from the original existing = find_paths_by_hash(texture_hash) - if existing and not force: + if existing: self.log.info("Found hash in database, preparing hardlink..") source = next((p for p in existing if os.path.exists(p)), None) if source: return source, HARDLINK, texture_hash else: self.log.warning( - ("Paths not found on disk, " - "skipping hardlink: %s") % (existing,) + "Paths not found on disk, " + "skipping hardlink: {}".format(existing) ) + def _process_texture(self, + filepath, + do_maketx, + staging_dir, + force_copy, + color_management, + colorspace): + """Process a single texture file on disk for publishing. + This will: + 1. Check whether it's already published, if so it will do hardlink + 2. If not published and maketx is enabled, generate a new .tx file. + 3. Compute the destination path for the source file. + + Args: + filepath (str): The source file path to process. + do_maketx (bool): Whether to produce a .tx file + staging_dir (str): The staging directory to write to. + force_copy (bool): Whether to force a copy even if a file hash + might have existed already in the project, otherwise + hardlinking the existing file is allowed. + color_management (dict): Maya's Color Management settings from + `lib.get_color_management_preferences` + colorspace (str): The source colorspace of the resources this + texture belongs to. + + Returns: + tuple: (filepath, copy_mode, texture_hash, result_colorspace) + """ + + fname, ext = os.path.splitext(os.path.basename(filepath)) + + # Note: The texture hash is only reliable if we include any potential + # conversion arguments provide to e.g. `maketx` + args = [] + hash_args = [] + if do_maketx and ext != ".tx": - # Produce .tx file in staging if source file is not .tx - converted = os.path.join(staging, "resources", fname + ".tx") - additional_args = [ + # Define .tx filepath in staging if source file is not .tx + converted = os.path.join(staging_dir, "resources", fname + ".tx") + + if color_management["enabled"]: + config_path = color_management["config"] + if not os.path.exists(config_path): + raise RuntimeError("OCIO config not found at: " + "{}".format(config_path)) + + render_colorspace = color_management["rendering_space"] + + self.log.info("tx: converting colorspace {0} " + "-> {1}".format(colorspace, render_colorspace)) + args.extend(["--colorconvert", colorspace, render_colorspace]) + args.extend(["--colorconfig", config_path]) + + else: + # We can't rely on the colorspace attribute when not + # in color managed mode because the collected color space + # is the color space attribute of the file node which can be + # any string whatsoever but only appears disabled in Attribute + # Editor. We assume we're always converting to linear/Raw if + # the source file is assumed to be sRGB. + render_colorspace = "linear" + if _has_arnold(): + img_info = image_info(filepath) + color_space = guess_colorspace(img_info) + if color_space.lower() == "sRGB": + self.log.info("tx: converting sRGB -> linear") + args.extend(["--colorconvert", "sRGB", "Raw"]) + else: + self.log.info("tx: texture's colorspace " + "is already linear") + else: + self.log.warning("tx: cannot guess the colorspace, " + "color conversion won't be available!") + + hash_args.append("maketx") + hash_args.extend(args) + + texture_hash = source_hash(filepath, *args) + + if not force_copy: + existing = self._get_existing_hashed_texture(filepath) + if existing: + return existing + + # Exclude these additional arguments from the hashing because + # it is the hash itself + args.extend([ "--sattrib", "sourceHash", texture_hash - ] - if linearize: - if cmds.colorManagementPrefs(query=True, cmEnabled=True): - render_colorspace = cmds.colorManagementPrefs(query=True, - renderingSpaceName=True) # noqa - config_path = cmds.colorManagementPrefs(query=True, - configFilePath=True) # noqa - if not os.path.exists(config_path): - raise RuntimeError("No OCIO config path found!") + ]) - color_space_attr = resource["node"] + ".colorSpace" - try: - color_space = cmds.getAttr(color_space_attr) - except ValueError: - # node doesn't have color space attribute - if _has_arnold(): - img_info = image_info(filepath) - color_space = guess_colorspace(img_info) - else: - color_space = "Raw" - self.log.info("tx: converting {0} -> {1}".format(color_space, render_colorspace)) # noqa - - additional_args.extend(["--colorconvert", - color_space, - render_colorspace]) - else: - - if _has_arnold(): - img_info = image_info(filepath) - color_space = guess_colorspace(img_info) - if color_space == "sRGB": - self.log.info("tx: converting sRGB -> linear") - additional_args.extend(["--colorconvert", - "sRGB", - "Raw"]) - else: - self.log.info("tx: texture's colorspace " - "is already linear") - else: - self.log.warning("cannot guess the colorspace" - "color conversion won't be available!") # noqa - - - additional_args.extend(["--colorconfig", config_path]) # Ensure folder exists if not os.path.exists(os.path.dirname(converted)): os.makedirs(os.path.dirname(converted)) @@ -562,13 +582,20 @@ class ExtractLook(publish.Extractor): maketx( filepath, converted, - additional_args, + args, self.log ) - return converted, COPY, texture_hash + return converted, COPY, texture_hash, render_colorspace - return filepath, COPY, texture_hash + # No special treatment for this file + texture_hash = source_hash(filepath) + if not force_copy: + existing = self._get_existing_hashed_texture(filepath) + if existing: + return existing + + return filepath, COPY, texture_hash, colorspace class ExtractModelRenderSets(ExtractLook): From c7e12b5184b39738d9225504de89054938754720 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 24 Mar 2023 19:39:51 +0100 Subject: [PATCH 49/81] Cosmetics --- openpype/hosts/maya/plugins/publish/extract_look.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 5c03aa5a5a..a067e63339 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -344,8 +344,6 @@ class ExtractLook(publish.Extractor): # didn't do color spaces different than the current # resource. processed_file = processed_files[filepath] - processed_colorspace = processed_file["color_space"] - processed_result_colorspace = processed_file["result_color_space"] self.log.debug( "File was already processed. Likely used by another " "resource too: {}".format(filepath) @@ -355,10 +353,12 @@ class ExtractLook(publish.Extractor): self.log.warning( "File was already processed but using another" "colorspace: {} <-> {}" - "".format(colorspace, processed_colorspace)) + "".format(colorspace, + processed_file["color_space"])) self._set_resource_result_colorspace( - resource, colorspace=processed_result_colorspace + resource, + colorspace=processed_file["result_color_space"] ) continue From b37c15f58215bbdb51227e690cf3dde0cd0fdd96 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 10:52:44 +0100 Subject: [PATCH 50/81] More WIP refactoring for TextureProcessors --- .../maya/plugins/publish/extract_look.py | 295 +++++++++--------- 1 file changed, 148 insertions(+), 147 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index b21ce72296..7515fdeb8c 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -26,37 +26,6 @@ COPY = 1 HARDLINK = 2 -def _has_arnold(): - """Return whether the arnold package is available and can be imported.""" - try: - import arnold # noqa: F401 - return True - except (ImportError, ModuleNotFoundError): - return False - - -def get_redshift_tool(tool_name): - """Path to redshift texture processor. - - On Windows it adds .exe extension if missing from tool argument. - - Args: - tool (string): Tool name. - - Returns: - str: Full path to redshift texture processor executable. - """ - redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] - - redshift_tool_path = os.path.join( - redshift_os_path, - "bin", - tool_name - ) - - return find_executable(redshift_tool_path) - - def find_paths_by_hash(texture_hash): """Find the texture hash key in the dictionary. @@ -75,17 +44,26 @@ def find_paths_by_hash(texture_hash): @six.add_metaclass(ABCMeta) class TextureProcessor: - def __init__(self, log=None): if log is None: log = logging.getLogger(self.__class___.__name__) + self.log = log @abstractmethod - def process(self, filepath): - + def apply_settings(self, system_settings, project_settings): pass + @abstractmethod + def process(self, + source, + colorspace, + color_management, + staging_dir): + pass + + # TODO: Warning this only supports Py3.3+ @staticmethod + @abstractmethod def get_extension(): pass @@ -93,10 +71,11 @@ class TextureProcessor: class MakeRSTexBin(TextureProcessor): """Make `.rstexbin` using `redshiftTextureProcessor`""" - def __init__(self): - super(MakeRSTexBin, self).__init__() - - def process(self, source, *args): + def process(self, + source, + colorspace, + color_management, + staging_dir): """ with some default settings. @@ -105,24 +84,22 @@ class MakeRSTexBin(TextureProcessor): Args: source (str): Path to source file. - *args: Additional arguments for `redshiftTextureProcessor`. """ if "REDSHIFT_COREDATAPATH" not in os.environ: raise RuntimeError("Must have Redshift available.") - texture_processor_path = get_redshift_tool("redshiftTextureProcessor") + texture_processor_path = self.get_redshift_tool( + "redshiftTextureProcessor" + ) if not texture_processor_path: - raise KnownPublishError("Must have Redshift available.", - title="Make RSTexBin texture") + raise KnownPublishError("Must have Redshift available.") subprocess_args = [ texture_processor_path, source ] - subprocess_args.extend(args) - self.log.debug(" ".join(subprocess_args)) try: out = run_subprocess(subprocess_args) @@ -131,18 +108,43 @@ class MakeRSTexBin(TextureProcessor): exc_info=True) raise + # TODO: Implement correct return values return out @staticmethod def get_extension(): return ".rstexbin" + @staticmethod + def get_redshift_tool(tool_name): + """Path to redshift texture processor. + + On Windows it adds .exe extension if missing from tool argument. + + Args: + tool (string): Tool name. + + Returns: + str: Full path to redshift texture processor executable. + """ + redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] + + redshift_tool_path = os.path.join( + redshift_os_path, + "bin", + tool_name + ) + + return find_executable(redshift_tool_path) + class MakeTX(TextureProcessor): - def __init__(self): - super(MakeTX, self).__init__() - def process(self, source, destination, *args): + def process(self, + source, + colorspace, + color_management, + staging_dir): """Make `.tx` using `maketx` with some default settings. The settings are based on default as used in Arnold's @@ -152,8 +154,6 @@ class MakeTX(TextureProcessor): Args: source (str): Path to source file. - destination (str): Writing destination path. - *args: Additional arguments for `maketx`. Returns: str: Output of `maketx` command. @@ -164,9 +164,78 @@ class MakeTX(TextureProcessor): maketx_path = get_oiio_tools_path("maketx") if not maketx_path: - print( - "OIIO tool not found in {}".format(maketx_path)) - raise AssertionError("OIIO tool not found") + raise AssertionError( + "OIIO 'maketx' tool not found. Result: {}".format(maketx_path) + ) + + # Define .tx filepath in staging if source file is not .tx + fname, ext = os.path.splitext(os.path.basename(source)) + if ext == ".tx": + # TODO: Implement this fallback + # Do nothing if the source file is already a .tx file. + # return source, COPY, texture_hash, render_colorspace + pass + + args = [] + if color_management["enabled"]: + config_path = color_management["config"] + if not os.path.exists(config_path): + raise RuntimeError("OCIO config not found at: " + "{}".format(config_path)) + + render_colorspace = color_management["rendering_space"] + + self.log.info("tx: converting colorspace {0} " + "-> {1}".format(colorspace, + render_colorspace)) + args.extend(["--colorconvert", colorspace, render_colorspace]) + args.extend(["--colorconfig", config_path]) + + else: + # We can't rely on the colorspace attribute when not in color + # managed mode because the collected color space is the color space + # attribute of the file node which can be any string whatsoever + # but only appears disabled in Attribute Editor. We assume we're + # always converting to linear/Raw if the source file is assumed to + # be sRGB. + # TODO Without color management do we even know we can do + # "colorconvert" and what config does that end up using since + # colorconvert is a OCIO command line flag for maketx. + # Also, Raw != linear? + render_colorspace = "linear" + if self._has_arnold(): + img_info = image_info(source) + color_space = guess_colorspace(img_info) + if color_space.lower() == "sRGB": + self.log.info("tx: converting sRGB -> linear") + args.extend(["--colorconvert", "sRGB", "Raw"]) + else: + self.log.info("tx: texture's colorspace " + "is already linear") + else: + self.log.warning("tx: cannot guess the colorspace, " + "color conversion won't be " + "available!") + + args.append("maketx") + args.extend(args) + + texture_hash = source_hash(source, *args) + + # Exclude these additional arguments from the hashing because + # it is the hash itself + args.extend([ + "--sattrib", + "sourceHash", + texture_hash + ]) + + # Ensure folder exists + converted = os.path.join(staging_dir, "resources", fname + ".tx") + if not os.path.exists(os.path.dirname(converted)): + os.makedirs(os.path.dirname(converted)) + + self.log.info("Generating .tx file for %s .." % source) subprocess_args = [ maketx_path, @@ -186,18 +255,27 @@ class MakeTX(TextureProcessor): self.log.debug(" ".join(subprocess_args)) try: - out = run_subprocess(subprocess_args) + run_subprocess(subprocess_args) except Exception: self.log.error("Texture maketx conversion failed", exc_info=True) raise - return out + return converted, COPY, texture_hash, render_colorspace @staticmethod def get_extension(): return ".tx" + @staticmethod + def _has_arnold(): + """Return whether the arnold package is available and importable.""" + try: + import arnold # noqa: F401 + return True + except (ImportError, ModuleNotFoundError): + return False + @contextlib.contextmanager def no_workspace_dir(): @@ -565,7 +643,7 @@ class ExtractLook(publish.Extractor): basename, ext = os.path.splitext(os.path.basename(filepath)) # Get extension from the last processor - for processors in reversed(processors): + for processor in reversed(processors): ext = processor.get_extension() self.log.debug("Processor {} defined extension: " "{}".format(processor, ext)) @@ -620,7 +698,6 @@ class ExtractLook(publish.Extractor): Returns: tuple: (filepath, copy_mode, texture_hash, result_colorspace) """ - fname, ext = os.path.splitext(os.path.basename(filepath)) # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` @@ -629,104 +706,28 @@ class ExtractLook(publish.Extractor): if len(processors) > 1: raise KnownPublishError( - "More than one texture processor not supported" + "More than one texture processor not supported. " + "Current processors enabled: {}".format(processors) ) # TODO: Make all processors take the same arguments for processor in processors: - if processor is MakeTX: - processed_path = processor().process(filepath, - converted, - "--sattrib", - "sourceHash", - escape_space(texture_hash), # noqa - colorconvert, - color_config, - ) - self.log.info("Generating texture file for %s .." % filepath) # noqa - self.log.info(converted) - if processed_path: - return processed_path, COPY, texture_hash - else: - self.log.info("maketx has returned nothing") - elif processor is MakeRSTexBin: - processed_path = processor().process(filepath) - self.log.info("Generating texture file for %s .." % filepath) # noqa - if processed_path: - return processed_path, COPY, texture_hash - else: - self.log.info("redshift texture converter has returned nothing") # noqa + self.log.debug("Processing texture {} with processor {}".format( + filepath, processor + )) - # TODO: continue this refactoring to processor - if do_maketx and ext != ".tx": - # Define .tx filepath in staging if source file is not .tx - converted = os.path.join(staging_dir, "resources", fname + ".tx") + processed_result = processor.process(filepath, + colorspace, + color_management) + if not processed_result: + raise RuntimeError("Texture Processor {} returned " + "no result.".format(processor)) - if color_management["enabled"]: - config_path = color_management["config"] - if not os.path.exists(config_path): - raise RuntimeError("OCIO config not found at: " - "{}".format(config_path)) + processed_path, processed_texture_hash = processed_result + self.log.info("Generated processed " + "texture: {}".format(processed_path)) - render_colorspace = color_management["rendering_space"] - - self.log.info("tx: converting colorspace {0} " - "-> {1}".format(colorspace, render_colorspace)) - args.extend(["--colorconvert", colorspace, render_colorspace]) - args.extend(["--colorconfig", config_path]) - - else: - # We can't rely on the colorspace attribute when not - # in color managed mode because the collected color space - # is the color space attribute of the file node which can be - # any string whatsoever but only appears disabled in Attribute - # Editor. We assume we're always converting to linear/Raw if - # the source file is assumed to be sRGB. - render_colorspace = "linear" - if _has_arnold(): - img_info = image_info(filepath) - color_space = guess_colorspace(img_info) - if color_space.lower() == "sRGB": - self.log.info("tx: converting sRGB -> linear") - args.extend(["--colorconvert", "sRGB", "Raw"]) - else: - self.log.info("tx: texture's colorspace " - "is already linear") - else: - self.log.warning("tx: cannot guess the colorspace, " - "color conversion won't be available!") - - hash_args.append("maketx") - hash_args.extend(args) - - texture_hash = source_hash(filepath, *args) - - if not force_copy: - existing = self._get_existing_hashed_texture(filepath) - if existing: - return existing - - # Exclude these additional arguments from the hashing because - # it is the hash itself - args.extend([ - "--sattrib", - "sourceHash", - texture_hash - ]) - - # Ensure folder exists - if not os.path.exists(os.path.dirname(converted)): - os.makedirs(os.path.dirname(converted)) - - self.log.info("Generating .tx file for %s .." % filepath) - maketx( - filepath, - converted, - args, - self.log - ) - - return converted, COPY, texture_hash, render_colorspace + return processed_path, COPY, processed_texture_hash # No special treatment for this file texture_hash = source_hash(filepath) From a6a392e9640b1028fe5f1e199df5bb0d96c4f570 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 12:32:35 +0100 Subject: [PATCH 51/81] Allow maketx with color management enabled --- .../publish/validate_look_color_space.py | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 openpype/hosts/maya/plugins/publish/validate_look_color_space.py diff --git a/openpype/hosts/maya/plugins/publish/validate_look_color_space.py b/openpype/hosts/maya/plugins/publish/validate_look_color_space.py deleted file mode 100644 index b1bdeb7541..0000000000 --- a/openpype/hosts/maya/plugins/publish/validate_look_color_space.py +++ /dev/null @@ -1,26 +0,0 @@ -from maya import cmds - -import pyblish.api -from openpype.pipeline.publish import ValidateContentsOrder -from openpype.pipeline import PublishValidationError - - -class ValidateMayaColorSpace(pyblish.api.InstancePlugin): - """ - Check if the OCIO Color Management and maketx options - enabled at the same time - """ - - order = ValidateContentsOrder - families = ['look'] - hosts = ['maya'] - label = 'Color Management with maketx' - - def process(self, instance): - ocio_maya = cmds.colorManagementPrefs(q=True, - cmConfigFileEnabled=True, - cmEnabled=True) - maketx = instance.data["maketx"] - - if ocio_maya and maketx: - raise PublishValidationError("Maya is color managed and maketx option is on. OpenPype doesn't support this combination yet.") # noqa From 6b841253d7e33d194db4e4cd3804952a819631b9 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 14:26:22 +0100 Subject: [PATCH 52/81] Fix hash args --- openpype/hosts/maya/plugins/publish/extract_look.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 7515fdeb8c..c8339a1335 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -217,10 +217,9 @@ class MakeTX(TextureProcessor): "color conversion won't be " "available!") - args.append("maketx") - args.extend(args) - - texture_hash = source_hash(source, *args) + hash_args = ["maketx"] + hash_args.extend(args) + texture_hash = source_hash(source, *hash_args) # Exclude these additional arguments from the hashing because # it is the hash itself From 44c0009e728921076bbfdad59fca0b834261b01c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 14:37:33 +0100 Subject: [PATCH 53/81] Allow to configure extra arguments in OP settings for `maketx` --- .../maya/plugins/publish/extract_look.py | 34 ++++++++++++++++--- .../defaults/project_settings/maya.json | 3 ++ .../schemas/schema_maya_publish.json | 16 +++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index c8339a1335..2951261cd6 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -139,13 +139,31 @@ class MakeRSTexBin(TextureProcessor): class MakeTX(TextureProcessor): + """Make `.tx` using `maketx` with some default settings.""" + + def __init__(self, log=None): + super(MakeTX, self).__init__(log=log) + self.extra_args = [] + + def apply_settings(self, system_settings, project_settings): + # Allow extra maketx arguments from project settings + extra_args_dict = ( + project_settings["maya"]["publish"] + .get("ExtractLook", {}) + .get("maketx_arguments", {}) + ) + extra_args = [] + for flag, value in extra_args_dict.items(): + extra_args.append(flag) + extra_args.append(value) + self.extra_args = extra_args def process(self, source, colorspace, color_management, staging_dir): - """Make `.tx` using `maketx` with some default settings. + """ The settings are based on default as used in Arnold's txManager in the scene. @@ -154,6 +172,10 @@ class MakeTX(TextureProcessor): Args: source (str): Path to source file. + colorspace (str): Colorspace of the source file. + color_management (dict): Maya Color management data from + `lib.get_color_management_preferences` + staging_dir (str): Output directory to write to. Returns: str: Output of `maketx` command. @@ -230,9 +252,9 @@ class MakeTX(TextureProcessor): ]) # Ensure folder exists - converted = os.path.join(staging_dir, "resources", fname + ".tx") - if not os.path.exists(os.path.dirname(converted)): - os.makedirs(os.path.dirname(converted)) + destination = os.path.join(staging_dir, "resources", fname + ".tx") + if not os.path.exists(os.path.dirname(destination)): + os.makedirs(os.path.dirname(destination)) self.log.info("Generating .tx file for %s .." % source) @@ -250,6 +272,8 @@ class MakeTX(TextureProcessor): ] subprocess_args.extend(args) + if self.extra_args: + subprocess_args.extend(self.extra_args) subprocess_args.extend(["-o", destination]) self.log.debug(" ".join(subprocess_args)) @@ -260,7 +284,7 @@ class MakeTX(TextureProcessor): exc_info=True) raise - return converted, COPY, texture_hash, render_colorspace + return destination, COPY, texture_hash, render_colorspace @staticmethod def get_extension(): diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index ef327fbd6b..502dbda870 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -925,6 +925,9 @@ "enabled": true, "active": true, "ogsfx_path": "/maya2glTF/PBR/shaders/glTF_PBR.ogsfx" + }, + "ExtractLook": { + "maketx_arguments": {} } }, "load": { diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json index 5a66f8a513..da12dde6b2 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json @@ -996,6 +996,22 @@ "label": "GLSL Shader Directory" } ] + }, + { + "type": "dict", + "collapsible": true, + "key": "ExtractLook", + "label": "Extract Look", + "children": [ + { + "key": "maketx_arguments", + "label": "Extra arguments for maketx command line", + "type": "dict-modifiable", + "object_type": { + "type": "text" + } + } + ] } ] } From c181d3ff2e14a3797a274ce9e90c1c08f21cac5f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 14:54:19 +0100 Subject: [PATCH 54/81] Intialize texture processors with applied settings --- .../maya/plugins/publish/extract_look.py | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 2951261cd6..9777f14f11 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -402,6 +402,25 @@ class ExtractLook(publish.Extractor): self.log.info("No sets found") return + # Specify texture processing executables to activate + # TODO: Load these more dynamically once we support more processors + processors = [] + context = instance.context + for key, Processor in { + # Instance data key to texture processor mapping + "maketx": MakeTX, + "rstex": MakeRSTexBin + }.items(): + if instance.data.get(key, False): + processor = Processor() + processor.apply_settings(context.data["system_settings"], + context.data["project_settings"]) + processors.append(processor) + + if processors: + self.log.debug("Collected texture processors: " + "{}".format(processors)) + self.log.debug("Processing resources..") results = self.process_resources(instance, staging_dir=dir_path) transfers = results["fileTransfers"] @@ -499,7 +518,7 @@ class ExtractLook(publish.Extractor): ) resource["result_colorspace"] = colorspace - def process_resources(self, instance, staging_dir): + def process_resources(self, instance, staging_dir, processors): """Process all resources in the instance. It is assumed that all resources are nodes using file textures. @@ -512,7 +531,6 @@ class ExtractLook(publish.Extractor): """ resources = instance.data["resources"] - do_maketx = instance.data.get("maketx", False) color_management = lib.get_color_management_preferences() # Temporary fix to NOT create hardlinks on windows machines @@ -532,14 +550,6 @@ class ExtractLook(publish.Extractor): destinations = {} remap = OrderedDict() - # Specify texture processing executables to activate - processors = [] - if instance.data.get("maketx", False): - processors.append(MakeTX) - # Option to convert textures to native redshift textures - if instance.data.get("rstex", False): - processors.append(MakeRSTexBin) - for resource in resources: colorspace = resource["color_space"] @@ -724,9 +734,6 @@ class ExtractLook(publish.Extractor): # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` - args = [] - hash_args = [] - if len(processors) > 1: raise KnownPublishError( "More than one texture processor not supported. " @@ -741,7 +748,8 @@ class ExtractLook(publish.Extractor): processed_result = processor.process(filepath, colorspace, - color_management) + color_management, + staging_dir) if not processed_result: raise RuntimeError("Texture Processor {} returned " "no result.".format(processor)) From e83708a3fb5da5710142578a4d433643ef529c89 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 14:54:52 +0100 Subject: [PATCH 55/81] Cosmetics --- openpype/hosts/maya/plugins/publish/extract_look.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 9777f14f11..632c64014f 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -149,8 +149,7 @@ class MakeTX(TextureProcessor): # Allow extra maketx arguments from project settings extra_args_dict = ( project_settings["maya"]["publish"] - .get("ExtractLook", {}) - .get("maketx_arguments", {}) + .get("ExtractLook", {}).get("maketx_arguments", {}) ) extra_args = [] for flag, value in extra_args_dict.items(): From d8c763c191f46448cfdbbc17c8939f1c6bf7f9fd Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 14:56:15 +0100 Subject: [PATCH 56/81] Include extra args in maketx texture hash --- openpype/hosts/maya/plugins/publish/extract_look.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 632c64014f..41377bb8f6 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -238,8 +238,11 @@ class MakeTX(TextureProcessor): "color conversion won't be " "available!") + # Note: The texture hash is only reliable if we include any potential + # conversion arguments provide to e.g. `maketx` hash_args = ["maketx"] hash_args.extend(args) + hash_args.extend(self.extra_args) texture_hash = source_hash(source, *hash_args) # Exclude these additional arguments from the hashing because @@ -731,8 +734,6 @@ class ExtractLook(publish.Extractor): tuple: (filepath, copy_mode, texture_hash, result_colorspace) """ - # Note: The texture hash is only reliable if we include any potential - # conversion arguments provide to e.g. `maketx` if len(processors) > 1: raise KnownPublishError( "More than one texture processor not supported. " From ba1d0c570ab4a20417a450dd2c5c030ac3f4901d Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 15:01:03 +0100 Subject: [PATCH 57/81] Fix arguments --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 41377bb8f6..77cf0d8a83 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -424,7 +424,9 @@ class ExtractLook(publish.Extractor): "{}".format(processors)) self.log.debug("Processing resources..") - results = self.process_resources(instance, staging_dir=dir_path) + results = self.process_resources(instance, + staging_dir=dir_path, + processors=processors) transfers = results["fileTransfers"] hardlinks = results["fileHardlinks"] hashes = results["fileHashes"] From 0fa5eab7c904ab497274c37c6d0a8dfc38015f36 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 20:29:49 +0100 Subject: [PATCH 58/81] Implement TextureResult dataclass + fix publishing with and without `maketx` --- .../maya/plugins/publish/extract_look.py | 106 ++++++++++++------ 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 77cf0d8a83..b166e85e7a 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -10,6 +10,7 @@ import tempfile import platform import contextlib from collections import OrderedDict +import attr from maya import cmds # noqa @@ -26,6 +27,19 @@ COPY = 1 HARDLINK = 2 +@attr.s +class TextureResult: + # Path to the file + path = attr.ib() + # Colorspace of the resulting texture. This might not be the input + # colorspace of the texture if a TextureProcessor has processed the file. + colorspace = attr.ib() + # Hash generated for the texture using openpype.lib.source_hash + file_hash = attr.ib() + # The transfer mode, e.g. COPY or HARDLINK + transfer_mode = attr.ib() + + def find_paths_by_hash(texture_hash): """Find the texture hash key in the dictionary. @@ -46,7 +60,7 @@ def find_paths_by_hash(texture_hash): class TextureProcessor: def __init__(self, log=None): if log is None: - log = logging.getLogger(self.__class___.__name__) + log = logging.getLogger(self.__class__.__name__) self.log = log @abstractmethod @@ -67,6 +81,10 @@ class TextureProcessor: def get_extension(): pass + def __repr__(self): + # Log instance as class name + return self.__class__.__name__ + class MakeRSTexBin(TextureProcessor): """Make `.rstexbin` using `redshiftTextureProcessor`""" @@ -100,6 +118,9 @@ class MakeRSTexBin(TextureProcessor): source ] + hash_args = ["rstex"] + texture_hash = source_hash(source, *hash_args) + self.log.debug(" ".join(subprocess_args)) try: out = run_subprocess(subprocess_args) @@ -108,8 +129,12 @@ class MakeRSTexBin(TextureProcessor): exc_info=True) raise - # TODO: Implement correct return values - return out + return TextureResult( + path=out, + file_hash=texture_hash, + colorspace=colorspace, + transfer_mode=COPY + ) @staticmethod def get_extension(): @@ -192,10 +217,13 @@ class MakeTX(TextureProcessor): # Define .tx filepath in staging if source file is not .tx fname, ext = os.path.splitext(os.path.basename(source)) if ext == ".tx": - # TODO: Implement this fallback # Do nothing if the source file is already a .tx file. - # return source, COPY, texture_hash, render_colorspace - pass + return TextureResult( + path=source, + file_hash=None, # todo: unknown texture hash? + colorspace=colorspace, + transfer_mode=COPY + ) args = [] if color_management["enabled"]: @@ -286,7 +314,12 @@ class MakeTX(TextureProcessor): exc_info=True) raise - return destination, COPY, texture_hash, render_colorspace + return TextureResult( + path=destination, + file_hash=texture_hash, + colorspace=render_colorspace, + transfer_mode=COPY + ) @staticmethod def get_extension(): @@ -510,17 +543,17 @@ class ExtractLook(publish.Extractor): def _set_resource_result_colorspace(self, resource, colorspace): """Update resource resulting colorspace after texture processing""" - if "result_colorspace" in resource: - if resource["result_colorspace"] == colorspace: + if "result_color_space" in resource: + if resource["result_color_space"] == colorspace: return self.log.warning( "Resource already has a resulting colorspace but is now " "being overridden to a new one: {} -> {}".format( - resource["result_colorspace"], colorspace + resource["result_color_space"], colorspace ) ) - resource["result_colorspace"] = colorspace + resource["result_color_space"] = colorspace def process_resources(self, instance, staging_dir, processors): """Process all resources in the instance. @@ -592,37 +625,33 @@ class ExtractLook(publish.Extractor): color_management=color_management, colorspace=colorspace ) - source, mode, texture_hash, result_colorspace = texture_result + source = texture_result.path destination = self.resource_destination(instance, - source, + texture_result.path, processors) # Set the resulting color space on the resource self._set_resource_result_colorspace( - resource, colorspace=result_colorspace + resource, colorspace=texture_result.colorspace ) processed_files[filepath] = { "color_space": colorspace, - "result_color_space": result_colorspace, + "result_color_space": texture_result.colorspace, } - # Force copy is specified. - if force_copy: - mode = COPY - - if mode == COPY: + if force_copy or texture_result.transfer_mode == COPY: transfers.append((source, destination)) self.log.info('file will be copied {} -> {}'.format( source, destination)) - elif mode == HARDLINK: + elif texture_result.transfer_mode == HARDLINK: hardlinks.append((source, destination)) self.log.info('file will be hardlinked {} -> {}'.format( source, destination)) # Store the hashes from hash to destination to include in the # database - hashes[texture_hash] = destination + hashes[texture_result.file_hash] = destination source = os.path.normpath(resource["source"]) if source not in destinations: @@ -697,10 +726,9 @@ class ExtractLook(publish.Extractor): # then don't reprocess but hardlink from the original existing = find_paths_by_hash(texture_hash) if existing: - self.log.info("Found hash in database, preparing hardlink..") source = next((p for p in existing if os.path.exists(p)), None) if source: - return source, HARDLINK, texture_hash + return source else: self.log.warning( "Paths not found on disk, " @@ -722,7 +750,7 @@ class ExtractLook(publish.Extractor): Args: filepath (str): The source file path to process. - processors (list): List of TexProcessor processing the texture + processors (list): List of TextureProcessor processing the texture staging_dir (str): The staging directory to write to. force_copy (bool): Whether to force a copy even if a file hash might have existed already in the project, otherwise @@ -733,7 +761,7 @@ class ExtractLook(publish.Extractor): texture belongs to. Returns: - tuple: (filepath, copy_mode, texture_hash, result_colorspace) + TextureResult: The texture result information. """ if len(processors) > 1: @@ -742,7 +770,6 @@ class ExtractLook(publish.Extractor): "Current processors enabled: {}".format(processors) ) - # TODO: Make all processors take the same arguments for processor in processors: self.log.debug("Processing texture {} with processor {}".format( filepath, processor @@ -755,21 +782,32 @@ class ExtractLook(publish.Extractor): if not processed_result: raise RuntimeError("Texture Processor {} returned " "no result.".format(processor)) - - processed_path, processed_texture_hash = processed_result self.log.info("Generated processed " - "texture: {}".format(processed_path)) + "texture: {}".format(processed_result.path)) - return processed_path, COPY, processed_texture_hash + # TODO: Currently all processors force copy instead of allowing + # hardlinks using source hashes. This should be refactored + return processed_result - # No special treatment for this file + # No texture processing for this file texture_hash = source_hash(filepath) if not force_copy: existing = self._get_existing_hashed_texture(filepath) if existing: - return existing + self.log.info("Found hash in database, preparing hardlink..") + return TextureResult( + path=filepath, + file_hash=texture_hash, + colorspace=colorspace, + transfer_mode=HARDLINK + ) - return filepath, COPY, texture_hash, colorspace + return TextureResult( + path=filepath, + file_hash=texture_hash, + colorspace=colorspace, + transfer_mode=COPY + ) class ExtractModelRenderSets(ExtractLook): From 81b5d771270b27849ee322d09c06e9ee75cd4f73 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sat, 25 Mar 2023 20:42:34 +0100 Subject: [PATCH 59/81] Cleanup, fix Py2 compatibility --- .../maya/plugins/publish/extract_look.py | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index b166e85e7a..dd59cd7dcc 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -63,7 +63,6 @@ class TextureProcessor: log = logging.getLogger(self.__class__.__name__) self.log = log - @abstractmethod def apply_settings(self, system_settings, project_settings): pass @@ -73,11 +72,28 @@ class TextureProcessor: colorspace, color_management, staging_dir): + """Process the `source` texture. + + Must be implemented on inherited class. + + This must always return a TextureResult even when it does not generate + a texture. If it doesn't generate a texture then it should return a + TextureResult using the input path and colorspace. + + Args: + source (str): Path to source file. + colorspace (str): Colorspace of the source file. + color_management (dict): Maya Color management data from + `lib.get_color_management_preferences` + staging_dir (str): Output directory to write to. + + Returns: + TextureResult: The resulting texture information. + + """ pass - # TODO: Warning this only supports Py3.3+ @staticmethod - @abstractmethod def get_extension(): pass @@ -164,7 +180,12 @@ class MakeRSTexBin(TextureProcessor): class MakeTX(TextureProcessor): - """Make `.tx` using `maketx` with some default settings.""" + """Make `.tx` using `maketx` with some default settings. + + Some hardcoded arguments passed to `maketx` are based on the defaults used + in Arnold's txManager tool. + + """ def __init__(self, log=None): super(MakeTX, self).__init__(log=log) @@ -187,12 +208,10 @@ class MakeTX(TextureProcessor): colorspace, color_management, staging_dir): - """ + """Process the texture. - The settings are based on default as used in Arnold's - txManager in the scene. This function requires the `maketx` executable to be - on the `PATH`. + available in the OIIO tool. Args: source (str): Path to source file. @@ -202,7 +221,7 @@ class MakeTX(TextureProcessor): staging_dir (str): Output directory to write to. Returns: - str: Output of `maketx` command. + TextureResult: The resulting texture information. """ from openpype.lib import get_oiio_tools_path @@ -474,7 +493,7 @@ class ExtractLook(publish.Extractor): # To avoid Maya trying to automatically remap the file # textures relative to the `workspace -directory` we force # it to a fake temporary workspace. This fixes textures - # getting incorrectly remapped. (LKD-17, PLN-101) + # getting incorrectly remapped. with no_workspace_dir(): with lib.attribute_values(remap): with lib.maintained_selection(): @@ -710,9 +729,11 @@ class ExtractLook(publish.Extractor): # Get extension from the last processor for processor in reversed(processors): - ext = processor.get_extension() - self.log.debug("Processor {} defined extension: " - "{}".format(processor, ext)) + processor_ext = processor.get_extension() + if processor_ext: + self.log.debug("Processor {} defined extension: " + "{}".format(processor, ext)) + ext = processor_ext break return os.path.join( From 16e5bb630f015f7deaa9a86e592cfe8f34fb94ac Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 20:30:35 +0200 Subject: [PATCH 60/81] Cleanup --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index dd59cd7dcc..2368295bf4 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -441,8 +441,6 @@ class ExtractLook(publish.Extractor): dir_path = self.staging_dir(instance) maya_fname = "{0}.{1}".format(instance.name, self.scene_type) json_fname = "{0}.{1}".format(instance.name, self.look_data_type) - - # Make texture dump folder maya_path = os.path.join(dir_path, maya_fname) json_path = os.path.join(dir_path, json_fname) From b340f6a57ca0ae9c22c29e0e2efebe23e8685df8 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 20:51:10 +0200 Subject: [PATCH 61/81] Clarify more about the issue in logging message + minor cleanup --- .../hosts/maya/plugins/publish/extract_look.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 2368295bf4..b68d8ae545 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -623,10 +623,14 @@ class ExtractLook(publish.Extractor): if colorspace != processed_file["color_space"]: self.log.warning( - "File was already processed but using another" - "colorspace: {} <-> {}" - "".format(colorspace, - processed_file["color_space"])) + "File '{}' was already processed using colorspace " + "'{}' instead of the current resource's " + "colorspace '{}'. The already processed texture " + "result's colorspace '{}' will be used." + "".format(filepath, + colorspace, + processed_file["color_space"], + processed_file["result_color_space"])) self._set_resource_result_colorspace( resource, @@ -642,7 +646,6 @@ class ExtractLook(publish.Extractor): color_management=color_management, colorspace=colorspace ) - source = texture_result.path destination = self.resource_destination(instance, texture_result.path, processors) @@ -657,6 +660,7 @@ class ExtractLook(publish.Extractor): "result_color_space": texture_result.colorspace, } + source = texture_result.path if force_copy or texture_result.transfer_mode == COPY: transfers.append((source, destination)) self.log.info('file will be copied {} -> {}'.format( From 81e25eb3a30a4ad801f09ff528458f5ba309e4fb Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 21:06:50 +0200 Subject: [PATCH 62/81] Move caching into one place --- .../maya/plugins/publish/extract_look.py | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index b68d8ae545..2e268749a0 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -596,14 +596,23 @@ class ExtractLook(publish.Extractor): else: force_copy = instance.data.get("forceCopy", False) + destinations_cache = {} + + def get_resource_destination_cached(path): + """Get resource destination with cached result per filepath""" + if path not in destinations_cache: + self.get_resource_destination(path, + instance.data["resourcesDir"], + processors) + destinations_cache[path] = destination + return destinations_cache[path] + # Process all resource's individual files processed_files = {} transfers = [] hardlinks = [] hashes = {} - destinations = {} remap = OrderedDict() - for resource in resources: colorspace = resource["color_space"] @@ -646,9 +655,6 @@ class ExtractLook(publish.Extractor): color_management=color_management, colorspace=colorspace ) - destination = self.resource_destination(instance, - texture_result.path, - processors) # Set the resulting color space on the resource self._set_resource_result_colorspace( @@ -661,6 +667,7 @@ class ExtractLook(publish.Extractor): } source = texture_result.path + destination = get_resource_destination_cached(source) if force_copy or texture_result.transfer_mode == COPY: transfers.append((source, destination)) self.log.info('file will be copied {} -> {}'.format( @@ -674,14 +681,6 @@ class ExtractLook(publish.Extractor): # database hashes[texture_result.file_hash] = destination - source = os.path.normpath(resource["source"]) - if source not in destinations: - # Cache destination as source resource might be included - # multiple times - destinations[source] = self.resource_destination( - instance, source, processors - ) - # Set up remapping attributes for the node during the publish # The order of these can be important if one attribute directly # affects another, e.g. we set colorspace after filepath because @@ -690,7 +689,9 @@ class ExtractLook(publish.Extractor): # attributes changed in the resulting publish) # Remap filepath to publish destination filepath_attr = resource["attribute"] - remap[filepath_attr] = destinations[source] + remap[filepath_attr] = get_resource_destination_cached( + resource["source"] + ) # Preserve color space values (force value after filepath change) # This will also trigger in the same order at end of context to @@ -709,23 +710,21 @@ class ExtractLook(publish.Extractor): "attrRemap": remap, } - def resource_destination(self, instance, filepath, processors): + def get_resource_destination(self, filepath, resources_dir, processors): """Get resource destination path. This is utility function to change path if resource file name is changed by some external tool like `maketx`. Args: - instance: Current Instance. - filepath (str): Resource path - processor: Texture processors converting resource. + filepath (str): Resource source path + resources_dir (str): Destination dir for resources in publish. + processors (list): Texture processors converting resource. Returns: str: Path to resource file """ - resources_dir = instance.data["resourcesDir"] - # Compute destination location basename, ext = os.path.splitext(os.path.basename(filepath)) From 03e5ff92ea3d6d229d7b91640852eead7628a356 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 21:07:38 +0200 Subject: [PATCH 63/81] Fix typo --- openpype/hosts/maya/plugins/publish/extract_look.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 2e268749a0..f846b4ee3d 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -601,9 +601,8 @@ class ExtractLook(publish.Extractor): def get_resource_destination_cached(path): """Get resource destination with cached result per filepath""" if path not in destinations_cache: - self.get_resource_destination(path, - instance.data["resourcesDir"], - processors) + destination = self.get_resource_destination( + path, instance.data["resourcesDir"], processors) destinations_cache[path] = destination return destinations_cache[path] From 2917ee27750384a3c952ced17e257fa99d48aeb8 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 21:10:57 +0200 Subject: [PATCH 64/81] Fix logging --- openpype/hosts/maya/plugins/publish/extract_look.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index f846b4ee3d..d5d8da04e7 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -730,9 +730,11 @@ class ExtractLook(publish.Extractor): # Get extension from the last processor for processor in reversed(processors): processor_ext = processor.get_extension() - if processor_ext: - self.log.debug("Processor {} defined extension: " - "{}".format(processor, ext)) + if processor_ext and ext != processor_ext: + self.log.debug("Processor {} overrides extension to '{}' " + "for path: {}".format(processor, + processor_ext, + filepath)) ext = processor_ext break From 00e5f220f4fd664ecd00cc7cae4fefab6984f89c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Sun, 26 Mar 2023 21:12:37 +0200 Subject: [PATCH 65/81] Add todo --- openpype/hosts/maya/plugins/publish/extract_look.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index d5d8da04e7..33ba4cc7a3 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -687,6 +687,11 @@ class ExtractLook(publish.Extractor): # filepaths (which is avoidable, but we don't want to have those # attributes changed in the resulting publish) # Remap filepath to publish destination + # TODO It would be much better if we could use the destination path + # from the actual processed texture results, but since the + # attribute will need to preserve tokens like , etc for + # now we will define the output path from the attribute value + # including the tokens to persist them. filepath_attr = resource["attribute"] remap[filepath_attr] = get_resource_destination_cached( resource["source"] From 35f76a2365d76aa97ca32a5591e831662dcd4029 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 10:20:02 +0200 Subject: [PATCH 66/81] Fix Redshift .rstexbin support --- openpype/hosts/maya/plugins/publish/extract_look.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 33ba4cc7a3..46eb940879 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -137,16 +137,21 @@ class MakeRSTexBin(TextureProcessor): hash_args = ["rstex"] texture_hash = source_hash(source, *hash_args) + # Redshift stores the output texture next to the input but with + # the extension replaced to `.rstexbin + basename, ext = os.path.splitext(source) + destination = "{}{}".format(basename, self.get_extension()) + self.log.debug(" ".join(subprocess_args)) try: - out = run_subprocess(subprocess_args) + run_subprocess(subprocess_args) except Exception: self.log.error("Texture .rstexbin conversion failed", exc_info=True) raise return TextureResult( - path=out, + path=destination, file_hash=texture_hash, colorspace=colorspace, transfer_mode=COPY From 95fcce48bd3a2476a89bf2823fb1abcaa8b1d056 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 10:22:33 +0200 Subject: [PATCH 67/81] Move `REDSHIFT_COREDATAPATH` environment check to `get_redshift_tool` --- openpype/hosts/maya/plugins/publish/extract_look.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 46eb940879..ad9ba34224 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -120,9 +120,6 @@ class MakeRSTexBin(TextureProcessor): source (str): Path to source file. """ - if "REDSHIFT_COREDATAPATH" not in os.environ: - raise RuntimeError("Must have Redshift available.") - texture_processor_path = self.get_redshift_tool( "redshiftTextureProcessor" ) @@ -173,10 +170,11 @@ class MakeRSTexBin(TextureProcessor): Returns: str: Full path to redshift texture processor executable. """ - redshift_os_path = os.environ["REDSHIFT_COREDATAPATH"] + if "REDSHIFT_COREDATAPATH" not in os.environ: + raise RuntimeError("Must have Redshift available.") redshift_tool_path = os.path.join( - redshift_os_path, + os.environ["REDSHIFT_COREDATAPATH"], "bin", tool_name ) From 9773cbbedc5224b130badee8a734d090c76aee33 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 10:27:13 +0200 Subject: [PATCH 68/81] Cleanup --- .../maya/plugins/publish/extract_look.py | 92 ++++++++++--------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index ad9ba34224..4e04999e7e 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -29,6 +29,7 @@ HARDLINK = 2 @attr.s class TextureResult: + """The resulting texture of a processed file for a resource""" # Path to the file path = attr.ib() # Colorspace of the resulting texture. This might not be the input @@ -56,6 +57,38 @@ def find_paths_by_hash(texture_hash): return legacy_io.distinct(key, {"type": "version"}) +@contextlib.contextmanager +def no_workspace_dir(): + """Force maya to a fake temporary workspace directory. + + Note: This is not maya.cmds.workspace 'rootDirectory' but the 'directory' + + This helps to avoid Maya automatically remapping image paths to files + relative to the currently set directory. + + """ + + # Store current workspace + original = cmds.workspace(query=True, directory=True) + + # Set a fake workspace + fake_workspace_dir = tempfile.mkdtemp() + cmds.workspace(directory=fake_workspace_dir) + + try: + yield + finally: + try: + cmds.workspace(directory=original) + except RuntimeError: + # If the original workspace directory didn't exist either + # ignore the fact that it fails to reset it to the old path + pass + + # Remove the temporary directory + os.rmdir(fake_workspace_dir) + + @six.add_metaclass(ABCMeta) class TextureProcessor: def __init__(self, log=None): @@ -64,6 +97,16 @@ class TextureProcessor: self.log = log def apply_settings(self, system_settings, project_settings): + """Apply OpenPype system/project settings to the TextureProcessor + + Args: + system_settings (dict): OpenPype system settings + project_settings (dict): OpenPype project settings + + Returns: + None + + """ pass @abstractmethod @@ -110,16 +153,7 @@ class MakeRSTexBin(TextureProcessor): colorspace, color_management, staging_dir): - """ - with some default settings. - This function requires the `REDSHIFT_COREDATAPATH` - to be in `PATH`. - - Args: - source (str): Path to source file. - - """ texture_processor_path = self.get_redshift_tool( "redshiftTextureProcessor" ) @@ -135,7 +169,7 @@ class MakeRSTexBin(TextureProcessor): texture_hash = source_hash(source, *hash_args) # Redshift stores the output texture next to the input but with - # the extension replaced to `.rstexbin + # the extension replaced to `.rstexbin` basename, ext = os.path.splitext(source) destination = "{}{}".format(basename, self.get_extension()) @@ -165,7 +199,7 @@ class MakeRSTexBin(TextureProcessor): On Windows it adds .exe extension if missing from tool argument. Args: - tool (string): Tool name. + tool_name (string): Tool name. Returns: str: Full path to redshift texture processor executable. @@ -213,8 +247,8 @@ class MakeTX(TextureProcessor): staging_dir): """Process the texture. - This function requires the `maketx` executable to be - available in the OIIO tool. + This function requires the `maketx` executable to be available in an + OpenImageIO toolset detectable by OpenPype. Args: source (str): Path to source file. @@ -357,38 +391,6 @@ class MakeTX(TextureProcessor): return False -@contextlib.contextmanager -def no_workspace_dir(): - """Force maya to a fake temporary workspace directory. - - Note: This is not maya.cmds.workspace 'rootDirectory' but the 'directory' - - This helps to avoid Maya automatically remapping image paths to files - relative to the currently set directory. - - """ - - # Store current workspace - original = cmds.workspace(query=True, directory=True) - - # Set a fake workspace - fake_workspace_dir = tempfile.mkdtemp() - cmds.workspace(directory=fake_workspace_dir) - - try: - yield - finally: - try: - cmds.workspace(directory=original) - except RuntimeError: - # If the original workspace directory didn't exist either - # ignore the fact that it fails to reset it to the old path - pass - - # Remove the temporary directory - os.rmdir(fake_workspace_dir) - - class ExtractLook(publish.Extractor): """Extract Look (Maya Scene + JSON) From 7f4fe957bc43d6290b63cdc86d4b4844fcd435c4 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 12:02:44 +0200 Subject: [PATCH 69/81] Move `get_oiio_tools_path` import to top of file --- openpype/hosts/maya/plugins/publish/extract_look.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 4e04999e7e..e0869b73e6 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -17,7 +17,7 @@ from maya import cmds # noqa import pyblish.api from openpype.lib.vendor_bin_utils import find_executable -from openpype.lib import source_hash, run_subprocess +from openpype.lib import source_hash, run_subprocess, get_oiio_tools_path from openpype.pipeline import legacy_io, publish, KnownPublishError from openpype.hosts.maya.api import lib from openpype.hosts.maya.api.lib import image_info, guess_colorspace @@ -261,7 +261,6 @@ class MakeTX(TextureProcessor): TextureResult: The resulting texture information. """ - from openpype.lib import get_oiio_tools_path maketx_path = get_oiio_tools_path("maketx") From 39d68780670c2333c0d39ef083331723b501c28d Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 12:30:31 +0200 Subject: [PATCH 70/81] Support flags without values for extra `maketx` arguments - Add todo for hardcoded maketx arguments - Add sourceHash argument at end to increase readability of the log --- .../maya/plugins/publish/extract_look.py | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index e0869b73e6..41d34a7ead 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -236,8 +236,16 @@ class MakeTX(TextureProcessor): ) extra_args = [] for flag, value in extra_args_dict.items(): + if not flag: + self.log.debug("Ignoring empty flag from `maketx_arguments` " + "setting..") + continue + extra_args.append(flag) - extra_args.append(value) + if value.strip(): + # There might be flags without values like --opaque-detect + extra_args.append(value) + self.extra_args = extra_args def process(self, @@ -328,14 +336,6 @@ class MakeTX(TextureProcessor): hash_args.extend(self.extra_args) texture_hash = source_hash(source, *hash_args) - # Exclude these additional arguments from the hashing because - # it is the hash itself - args.extend([ - "--sattrib", - "sourceHash", - texture_hash - ]) - # Ensure folder exists destination = os.path.join(staging_dir, "resources", fname + ".tx") if not os.path.exists(os.path.dirname(destination)): @@ -347,9 +347,12 @@ class MakeTX(TextureProcessor): maketx_path, "-v", # verbose "-u", # update mode + # --checknan doesn't influence the output file but aborts the + # conversion if it finds any. So we can avoid need + "--checknan", + # todo: --unpremult, --oiio, --filter should be in the file hash # unpremultiply before conversion (recommended when alpha present) "--unpremult", - "--checknan", # use oiio-optimized settings for tile-size, planarconfig, metadata "--oiio", "--filter", "lanczos3", @@ -359,6 +362,15 @@ class MakeTX(TextureProcessor): subprocess_args.extend(args) if self.extra_args: subprocess_args.extend(self.extra_args) + + # Add source hash attribute after other arguments for log readability + # Note: argument is excluding from the hash since it is the hash itself + subprocess_args.extend([ + "--sattrib", + "sourceHash", + texture_hash + ]) + subprocess_args.extend(["-o", destination]) self.log.debug(" ".join(subprocess_args)) From db27765637f0213cab4d4d2b8a89d2fb2f147ecd Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 12:31:01 +0200 Subject: [PATCH 71/81] Grammar --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 41d34a7ead..f3d0790e22 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -364,7 +364,7 @@ class MakeTX(TextureProcessor): subprocess_args.extend(self.extra_args) # Add source hash attribute after other arguments for log readability - # Note: argument is excluding from the hash since it is the hash itself + # Note: argument is excluded from the hash since it is the hash itself subprocess_args.extend([ "--sattrib", "sourceHash", From 7dfaa5b4f4cbf4b8e821d0b4af2e78bc5d7bee97 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 12:36:59 +0200 Subject: [PATCH 72/81] Add maketx hardcoded flags to the file hash --- .../hosts/maya/plugins/publish/extract_look.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index f3d0790e22..be3de13b37 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -288,7 +288,15 @@ class MakeTX(TextureProcessor): transfer_mode=COPY ) - args = [] + # Hardcoded default arguments for maketx conversion based on Arnold's + # txManager in Maya + args = [ + # unpremultiply before conversion (recommended when alpha present) + "--unpremult", + # use oiio-optimized settings for tile-size, planarconfig, metadata + "--oiio", + "--filter", "lanczos3", + ] if color_management["enabled"]: config_path = color_management["config"] if not os.path.exists(config_path): @@ -348,14 +356,8 @@ class MakeTX(TextureProcessor): "-v", # verbose "-u", # update mode # --checknan doesn't influence the output file but aborts the - # conversion if it finds any. So we can avoid need + # conversion if it finds any. So we can avoid it for the file hash "--checknan", - # todo: --unpremult, --oiio, --filter should be in the file hash - # unpremultiply before conversion (recommended when alpha present) - "--unpremult", - # use oiio-optimized settings for tile-size, planarconfig, metadata - "--oiio", - "--filter", "lanczos3", source ] From 3444660a982b13cf798b575e9ebecf8f85f49f05 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 13:20:26 +0200 Subject: [PATCH 73/81] Convert to "linear" because it's always available in OIIO if OCIO is disabled or no valid config is found - If OCIO is not enabled (or cannot find a valid configuration, OIIO will at least be able to convert among linear, sRGB, and Rec709.) --- openpype/hosts/maya/plugins/publish/extract_look.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index be3de13b37..5b9b0777a0 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -316,19 +316,15 @@ class MakeTX(TextureProcessor): # managed mode because the collected color space is the color space # attribute of the file node which can be any string whatsoever # but only appears disabled in Attribute Editor. We assume we're - # always converting to linear/Raw if the source file is assumed to + # always converting to linear if the source file is assumed to # be sRGB. - # TODO Without color management do we even know we can do - # "colorconvert" and what config does that end up using since - # colorconvert is a OCIO command line flag for maketx. - # Also, Raw != linear? render_colorspace = "linear" if self._has_arnold(): img_info = image_info(source) color_space = guess_colorspace(img_info) if color_space.lower() == "sRGB": self.log.info("tx: converting sRGB -> linear") - args.extend(["--colorconvert", "sRGB", "Raw"]) + args.extend(["--colorconvert", "sRGB", render_colorspace]) else: self.log.info("tx: texture's colorspace " "is already linear") From 6f015d6d64278b696f861b0b85284ed9e4ca9417 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 13:31:33 +0200 Subject: [PATCH 74/81] Cleanup/refactor based on @fabiaserra comments --- .../maya/plugins/publish/extract_look.py | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 5b9b0777a0..daf6735660 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -91,6 +91,9 @@ def no_workspace_dir(): @six.add_metaclass(ABCMeta) class TextureProcessor: + + extension = None + def __init__(self, log=None): if log is None: log = logging.getLogger(self.__class__.__name__) @@ -136,10 +139,6 @@ class TextureProcessor: """ pass - @staticmethod - def get_extension(): - pass - def __repr__(self): # Log instance as class name return self.__class__.__name__ @@ -148,6 +147,8 @@ class TextureProcessor: class MakeRSTexBin(TextureProcessor): """Make `.rstexbin` using `redshiftTextureProcessor`""" + extension = ".rstexbin" + def process(self, source, colorspace, @@ -171,7 +172,7 @@ class MakeRSTexBin(TextureProcessor): # Redshift stores the output texture next to the input but with # the extension replaced to `.rstexbin` basename, ext = os.path.splitext(source) - destination = "{}{}".format(basename, self.get_extension()) + destination = "{}{}".format(basename, self.extension) self.log.debug(" ".join(subprocess_args)) try: @@ -188,10 +189,6 @@ class MakeRSTexBin(TextureProcessor): transfer_mode=COPY ) - @staticmethod - def get_extension(): - return ".rstexbin" - @staticmethod def get_redshift_tool(tool_name): """Path to redshift texture processor. @@ -224,6 +221,8 @@ class MakeTX(TextureProcessor): """ + extension = ".tx" + def __init__(self, log=None): super(MakeTX, self).__init__(log=log) self.extra_args = [] @@ -335,15 +334,13 @@ class MakeTX(TextureProcessor): # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` - hash_args = ["maketx"] - hash_args.extend(args) - hash_args.extend(self.extra_args) + hash_args = ["maketx"] + args + self.extra_args texture_hash = source_hash(source, *hash_args) # Ensure folder exists - destination = os.path.join(staging_dir, "resources", fname + ".tx") - if not os.path.exists(os.path.dirname(destination)): - os.makedirs(os.path.dirname(destination)) + resources_dir = os.path.join(staging_dir, "resources") + if not os.path.exists(resources_dir): + os.makedirs(resources_dir) self.log.info("Generating .tx file for %s .." % source) @@ -369,6 +366,7 @@ class MakeTX(TextureProcessor): texture_hash ]) + destination = os.path.join(resources_dir, fname + ".tx") subprocess_args.extend(["-o", destination]) self.log.debug(" ".join(subprocess_args)) @@ -386,10 +384,6 @@ class MakeTX(TextureProcessor): transfer_mode=COPY ) - @staticmethod - def get_extension(): - return ".tx" - @staticmethod def _has_arnold(): """Return whether the arnold package is available and importable.""" @@ -748,7 +742,7 @@ class ExtractLook(publish.Extractor): # Get extension from the last processor for processor in reversed(processors): - processor_ext = processor.get_extension() + processor_ext = processor.extension if processor_ext and ext != processor_ext: self.log.debug("Processor {} overrides extension to '{}' " "for path: {}".format(processor, @@ -785,9 +779,12 @@ class ExtractLook(publish.Extractor): color_management, colorspace): """Process a single texture file on disk for publishing. + This will: 1. Check whether it's already published, if so it will do hardlink - 2. If not published and maketx is enabled, generate a new .tx file. + (if the texture hash is found and force copy is not enabled) + 2. It will process the texture using the supplied texture + processors like MakeTX and MakeRSTexBin if enabled. 3. Compute the destination path for the source file. Args: From e11d1f279ac03da71158075b1793b7b673303cab Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 13:46:33 +0200 Subject: [PATCH 75/81] Add assumption for some file formats to be sRGB: `.png`, `.jpeg` and `.jpg` --- .../maya/plugins/publish/extract_look.py | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index daf6735660..9c360c8dd4 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -26,6 +26,10 @@ from openpype.hosts.maya.api.lib import image_info, guess_colorspace COPY = 1 HARDLINK = 2 +# File formats we will assume are sRGB when Maya Color Management is disabled +# Currently only used for `maketx` color conversion logic +NONLINEAR_FILE_FORMATS = {".png", ".jpeg", ".jpg"} + @attr.s class TextureResult: @@ -311,6 +315,7 @@ class MakeTX(TextureProcessor): args.extend(["--colorconfig", config_path]) else: + self.log.debug("Maya color management is disabled..") # We can't rely on the colorspace attribute when not in color # managed mode because the collected color space is the color space # attribute of the file node which can be any string whatsoever @@ -318,19 +323,28 @@ class MakeTX(TextureProcessor): # always converting to linear if the source file is assumed to # be sRGB. render_colorspace = "linear" - if self._has_arnold(): + assumed_input_colorspace = "linear" + if ext.lower() in NONLINEAR_FILE_FORMATS: + assumed_input_colorspace = "sRGB" + elif self._has_arnold(): + # Assume colorspace based on input image bit-depth img_info = image_info(source) - color_space = guess_colorspace(img_info) - if color_space.lower() == "sRGB": - self.log.info("tx: converting sRGB -> linear") - args.extend(["--colorconvert", "sRGB", render_colorspace]) - else: - self.log.info("tx: texture's colorspace " - "is already linear") + assumed_input_colorspace = guess_colorspace(img_info) else: - self.log.warning("tx: cannot guess the colorspace, " - "color conversion won't be " - "available!") + self.log.warning("tx: cannot guess the colorspace, a linear " + "colorspace will be assumed for file: " + "{}".format(source)) + + if assumed_input_colorspace == "sRGB": + self.log.info("tx: converting sRGB -> linear") + args.extend(["--colorconvert", "sRGB", render_colorspace]) + elif assumed_input_colorspace == "linear": + self.log.info("tx: texture's colorspace " + "is already linear") + else: + self.log.warning("Unexpected texture color space: {} " + "(expected either 'linear' or 'sRGB')" + "".format(assumed_input_colorspace)) # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` From bf60e7370453831d4ee3f89a32fd435091450ef3 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 13:47:26 +0200 Subject: [PATCH 76/81] Remove `.png` from nonlinear formats assumption because apparently they can be 32-bit --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 9c360c8dd4..c68ce56fcc 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -28,7 +28,7 @@ HARDLINK = 2 # File formats we will assume are sRGB when Maya Color Management is disabled # Currently only used for `maketx` color conversion logic -NONLINEAR_FILE_FORMATS = {".png", ".jpeg", ".jpg"} +NONLINEAR_FILE_FORMATS = {".jpeg", ".jpg"} @attr.s From 5ed6c29eee6531eaa5dde107a98a777ce091f232 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 15:23:14 +0200 Subject: [PATCH 77/81] Mimic Arnold tx manager behavior whenever maya color management is disabled - Do no color conversion when color management is disabled --- .../maya/plugins/publish/extract_look.py | 42 +++++-------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index c68ce56fcc..7405eb1a9f 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -315,36 +315,10 @@ class MakeTX(TextureProcessor): args.extend(["--colorconfig", config_path]) else: - self.log.debug("Maya color management is disabled..") - # We can't rely on the colorspace attribute when not in color - # managed mode because the collected color space is the color space - # attribute of the file node which can be any string whatsoever - # but only appears disabled in Attribute Editor. We assume we're - # always converting to linear if the source file is assumed to - # be sRGB. - render_colorspace = "linear" - assumed_input_colorspace = "linear" - if ext.lower() in NONLINEAR_FILE_FORMATS: - assumed_input_colorspace = "sRGB" - elif self._has_arnold(): - # Assume colorspace based on input image bit-depth - img_info = image_info(source) - assumed_input_colorspace = guess_colorspace(img_info) - else: - self.log.warning("tx: cannot guess the colorspace, a linear " - "colorspace will be assumed for file: " - "{}".format(source)) - - if assumed_input_colorspace == "sRGB": - self.log.info("tx: converting sRGB -> linear") - args.extend(["--colorconvert", "sRGB", render_colorspace]) - elif assumed_input_colorspace == "linear": - self.log.info("tx: texture's colorspace " - "is already linear") - else: - self.log.warning("Unexpected texture color space: {} " - "(expected either 'linear' or 'sRGB')" - "".format(assumed_input_colorspace)) + # Maya Color management is disabled. We cannot rely on an OCIO + self.log.debug("tx: Maya color management is disabled. No color " + "conversion will be applied to .tx conversion for: " + "{}".format(source)) # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` @@ -383,9 +357,15 @@ class MakeTX(TextureProcessor): destination = os.path.join(resources_dir, fname + ".tx") subprocess_args.extend(["-o", destination]) + # We want to make sure we are explicit about what OCIO config gets + # used. So when we supply no --colorconfig flag that no fallback to + # an OCIO env var occurs. + env = os.environ.copy() + env.pop("OCIO", None) + self.log.debug(" ".join(subprocess_args)) try: - run_subprocess(subprocess_args) + run_subprocess(subprocess_args, env=env) except Exception: self.log.error("Texture maketx conversion failed", exc_info=True) From 2cb03b75b74ee145dba5f28b90f09861fe7b350a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 15:35:03 +0200 Subject: [PATCH 78/81] Clean up imports a bit. --- .../maya/plugins/publish/extract_look.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 7405eb1a9f..e939992454 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -1,26 +1,24 @@ # -*- coding: utf-8 -*- """Maya look extractor.""" -import logging from abc import ABCMeta, abstractmethod - -import six -import os -import json -import tempfile -import platform -import contextlib from collections import OrderedDict +import contextlib +import json +import logging +import os +import platform +import tempfile +import six import attr -from maya import cmds # noqa - import pyblish.api +from maya import cmds # noqa + from openpype.lib.vendor_bin_utils import find_executable from openpype.lib import source_hash, run_subprocess, get_oiio_tools_path from openpype.pipeline import legacy_io, publish, KnownPublishError from openpype.hosts.maya.api import lib -from openpype.hosts.maya.api.lib import image_info, guess_colorspace # Modes for transfer COPY = 1 From 108bcd8f27acd3d6b632c9df969357f1cee9773b Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 15:37:27 +0200 Subject: [PATCH 79/81] Fix missing variable declaration --- openpype/hosts/maya/plugins/publish/extract_look.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index e939992454..9599f9f809 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -317,6 +317,8 @@ class MakeTX(TextureProcessor): self.log.debug("tx: Maya color management is disabled. No color " "conversion will be applied to .tx conversion for: " "{}".format(source)) + # Assume linear + render_colorspace = "linear" # Note: The texture hash is only reliable if we include any potential # conversion arguments provide to e.g. `maketx` From 22dbc4e4fa5227db80fda932f4b9b10a76ba64bd Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 27 Mar 2023 15:40:33 +0200 Subject: [PATCH 80/81] Remove unused variable `NONLINEAR_FILE_FORMATS` --- openpype/hosts/maya/plugins/publish/extract_look.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 9599f9f809..1e339542d7 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -24,10 +24,6 @@ from openpype.hosts.maya.api import lib COPY = 1 HARDLINK = 2 -# File formats we will assume are sRGB when Maya Color Management is disabled -# Currently only used for `maketx` color conversion logic -NONLINEAR_FILE_FORMATS = {".jpeg", ".jpg"} - @attr.s class TextureResult: From 421048083164e549a69bdc16e248b33d7cc0c71f Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 29 Mar 2023 12:51:07 +0200 Subject: [PATCH 81/81] Refactor ExtractLook maketx argument in settings to more structured arguments/parameters --- .../maya/plugins/publish/extract_look.py | 20 +++++++++---------- .../defaults/project_settings/maya.json | 2 +- .../schemas/schema_maya_publish.json | 17 ++++++++++++++-- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/extract_look.py b/openpype/hosts/maya/plugins/publish/extract_look.py index 1e339542d7..93054e5fbb 100644 --- a/openpype/hosts/maya/plugins/publish/extract_look.py +++ b/openpype/hosts/maya/plugins/publish/extract_look.py @@ -227,21 +227,21 @@ class MakeTX(TextureProcessor): def apply_settings(self, system_settings, project_settings): # Allow extra maketx arguments from project settings - extra_args_dict = ( + args_settings = ( project_settings["maya"]["publish"] - .get("ExtractLook", {}).get("maketx_arguments", {}) + .get("ExtractLook", {}).get("maketx_arguments", []) ) extra_args = [] - for flag, value in extra_args_dict.items(): - if not flag: - self.log.debug("Ignoring empty flag from `maketx_arguments` " - "setting..") + for arg_data in args_settings: + argument = arg_data["argument"] + parameters = arg_data["parameters"] + if not argument: + self.log.debug("Ignoring empty parameter from " + "`maketx_arguments` setting..") continue - extra_args.append(flag) - if value.strip(): - # There might be flags without values like --opaque-detect - extra_args.append(value) + extra_args.append(argument) + extra_args.extend(parameters) self.extra_args = extra_args diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 26dd66770f..8f5a3c75ab 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -928,7 +928,7 @@ "ogsfx_path": "/maya2glTF/PBR/shaders/glTF_PBR.ogsfx" }, "ExtractLook": { - "maketx_arguments": {} + "maketx_arguments": [] } }, "load": { diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json index da12dde6b2..7ced375cb5 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_publish.json @@ -1004,11 +1004,24 @@ "label": "Extract Look", "children": [ { + "type": "list", "key": "maketx_arguments", "label": "Extra arguments for maketx command line", - "type": "dict-modifiable", "object_type": { - "type": "text" + "type": "dict", + "children": [ + { + "key": "argument", + "label": "Argument", + "type": "text" + }, + { + "key": "parameters", + "label": "Parameters", + "type": "list", + "object_type": "text" + } + ] } } ]