From d7a2c57fd836ae6e26bc0f7e02f3b46a81a70dd8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 31 Jul 2024 13:54:41 +0200 Subject: [PATCH 01/45] Refactor OCIO config handling, introduce fallback mechanism - Added function to extract config path from profile data - Updated global config retrieval to use new function - Introduced fallback mechanism for missing product entities --- client/ayon_core/pipeline/colorspace.py | 67 +++++++++++++++++----- server/settings/conversion.py | 38 +++++++++++++ server/settings/main.py | 75 ++++++++++++++++++++++--- 3 files changed, 156 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 099616ff4a..1986e2d407 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -699,6 +699,33 @@ def get_ocio_config_views(config_path): ) +def _get_config_path_from_profile_data( + data, data_type, template_data) -> dict: + """Get config path from profile data. + + Args: + data (dict[str, Any]): Profile data. + data_type (str): Profile type. + template_data (dict[str, Any]): Template data. + + Returns: + dict[str, str]: Config data with path and template. + """ + template = data[data_type] + result = StringTemplate.format_strict_template( + template, template_data + ) + normalized_path = str(result.normalized()) + if not os.path.exists(normalized_path): + log.warning(f"Path was not found '{normalized_path}'.") + return None + + return { + "path": normalized_path, + "template": template + } + + def _get_global_config_data( project_name, host_name, @@ -717,7 +744,7 @@ def _get_global_config_data( 2. Custom path to ocio config. 3. Path to 'ocioconfig' representation on product. Name of product can be defined in settings. Product name can be regex but exact match is - always preferred. + always preferred. Fallback can be defined in case no product is found. None is returned when no profile is found, when path @@ -755,19 +782,8 @@ def _get_global_config_data( profile_type = profile["type"] if profile_type in ("builtin_path", "custom_path"): - template = profile[profile_type] - result = StringTemplate.format_strict_template( - template, template_data - ) - normalized_path = str(result.normalized()) - if not os.path.exists(normalized_path): - log.warning(f"Path was not found '{normalized_path}'.") - return None - - return { - "path": normalized_path, - "template": template - } + return _get_config_path_from_profile_data( + profile, profile_type, template_data) # TODO decide if this is the right name for representation repre_name = "ocioconfig" @@ -778,7 +794,21 @@ def _get_global_config_data( return None folder_path = folder_info["path"] - product_name = profile["product_name"] + # Backward compatibility for old projects + # TODO remove in future 0.4.4 onwards + product_name = profile.get("product_name") + # TODO: this should be required after backward compatibility is removed + fallback_data = None + published_product_data = profile.get("published_product") + + if product_name is None and published_product_data is None: + log.warning("Product name or published product is missing.") + return None + + if published_product_data: + product_name = published_product_data["product_name"] + fallback_data = published_product_data["fallback"] + if folder_id is None: folder_entity = ayon_api.get_folder_by_path( project_name, folder_path, fields={"id"} @@ -798,6 +828,13 @@ def _get_global_config_data( ) } if not product_entities_by_name: + # TODO: make this required in future 0.4.4 onwards + if fallback_data: + # in case no product was found we need to use fallback + fallback_type = fallback_data["type"] + return _get_config_path_from_profile_data( + fallback_data, fallback_type, template_data + ) log.debug( f"No product entities were found for folder '{folder_path}' with" f" product name filter '{product_name}'." diff --git a/server/settings/conversion.py b/server/settings/conversion.py index f513738603..b29c827cae 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -4,6 +4,43 @@ from typing import Any from .publish_plugins import DEFAULT_PUBLISH_VALUES +def _convert_imageio_configs_0_4_3(overrides): + """Imageio config settings did change to profiles since 0.4.3.""" + imageio_overrides = overrides.get("imageio") or {} + + # make sure settings are already converted to profiles + if ( + "ocio_config_profiles" not in imageio_overrides + ): + return + + ocio_config_profiles = imageio_overrides["ocio_config_profiles"] + + for inx, profile in enumerate(ocio_config_profiles): + if profile["type"] != "product_name": + continue + + # create new profile + new_profile = { + "type": "published_product", + "published_product": { + "product_name": profile["product_name"], + "fallback": { + "type": "builtin_path", + "builtin_path": "{BUILTIN_OCIO_ROOT}/aces_1.2/config.ocio", + }, + }, + "host_names": profile["host_names"], + "task_names": profile["task_names"], + "task_types": profile["task_types"], + "custom_path": profile["custom_path"], + "builtin_path": profile["builtin_path"], + } + + # replace old profile with new profile + ocio_config_profiles[inx] = new_profile + + def _convert_imageio_configs_0_3_1(overrides): """Imageio config settings did change to profiles since 0.3.1. .""" imageio_overrides = overrides.get("imageio") or {} @@ -82,5 +119,6 @@ def convert_settings_overrides( overrides: dict[str, Any], ) -> dict[str, Any]: _convert_imageio_configs_0_3_1(overrides) + _convert_imageio_configs_0_4_3(overrides) _conver_publish_plugins(overrides) return overrides diff --git a/server/settings/main.py b/server/settings/main.py index 0972ccdfb9..09c9bf0065 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -58,7 +58,14 @@ def _ocio_config_profile_types(): return [ {"value": "builtin_path", "label": "AYON built-in OCIO config"}, {"value": "custom_path", "label": "Path to OCIO config"}, - {"value": "product_name", "label": "Published product"}, + {"value": "published_product", "label": "Published product"}, + ] + + +def _fallback_ocio_config_profile_types(): + return [ + {"value": "builtin_path", "label": "AYON built-in OCIO config"}, + {"value": "custom_path", "label": "Path to OCIO config"}, ] @@ -76,6 +83,49 @@ def _ocio_built_in_paths(): ] +class FallbackProductModel(BaseSettingsModel): + _layout = "expanded" + type: str = SettingsField( + title="Fallback config type", + enum_resolver=_fallback_ocio_config_profile_types, + conditionalEnum=True, + default="builtin_path", + description=( + "Type of config which needs to be used in case published " + "product is not found." + ), + ) + builtin_path: str = SettingsField( + "ACES 1.2", + title="Built-in OCIO config", + enum_resolver=_ocio_built_in_paths, + description=( + "AYON ocio addon distributed OCIO config. " + "Activated addon in bundle is required: 'ayon_ocio' >= 1.1.1" + ), + ) + custom_path: str = SettingsField( + "", + title="OCIO config path", + description="Path to OCIO config. Anatomy formatting is supported.", + ) + + +class PublishedProductModel(BaseSettingsModel): + _layout = "expanded" + product_name: str = SettingsField( + "", + title="Product name", + description=( + "Context related published product name to get OCIO config from. " + "Partial match is supported via use of regex expression." + ), + ) + fallback: FallbackProductModel = SettingsField( + default_factory=FallbackProductModel, + ) + + class CoreImageIOConfigProfilesModel(BaseSettingsModel): _layout = "expanded" host_names: list[str] = SettingsField( @@ -102,19 +152,19 @@ class CoreImageIOConfigProfilesModel(BaseSettingsModel): "ACES 1.2", title="Built-in OCIO config", enum_resolver=_ocio_built_in_paths, + description=( + "AYON ocio addon distributed OCIO config. " + "Activated addon in bundle is required: 'ayon_ocio' >= 1.1.1" + ), ) custom_path: str = SettingsField( "", title="OCIO config path", description="Path to OCIO config. Anatomy formatting is supported.", ) - product_name: str = SettingsField( - "", - title="Product name", - description=( - "Published product name to get OCIO config from. " - "Partial match is supported." - ), + published_product: PublishedProductModel = SettingsField( + default_factory=PublishedProductModel, + title="Published product", ) @@ -294,7 +344,14 @@ DEFAULT_VALUES = { "type": "builtin_path", "builtin_path": "{BUILTIN_OCIO_ROOT}/aces_1.2/config.ocio", "custom_path": "", - "product_name": "", + "published_product": { + "product_name": "", + "fallback": { + "type": "builtin_path", + "builtin_path": "ACES 1.2", + "custom_path": "" + } + } } ], "file_rules": { From 9375b8bee2985f251e5aad1b6b4309caa1c4c5d1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 31 Jul 2024 13:57:47 +0200 Subject: [PATCH 02/45] Update version references for future removal and conversion functions in colorspace and conversion modules. - Update version references to 0.4.5 for future removal in colorspace module. - Update function name and version reference to 0.4.4 for imageio config conversion in the conversion module. --- client/ayon_core/pipeline/colorspace.py | 4 ++-- server/settings/conversion.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 1986e2d407..106d43d55a 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -795,7 +795,7 @@ def _get_global_config_data( folder_path = folder_info["path"] # Backward compatibility for old projects - # TODO remove in future 0.4.4 onwards + # TODO remove in future 0.4.5 onwards product_name = profile.get("product_name") # TODO: this should be required after backward compatibility is removed fallback_data = None @@ -828,7 +828,7 @@ def _get_global_config_data( ) } if not product_entities_by_name: - # TODO: make this required in future 0.4.4 onwards + # TODO: make this required in future 0.4.5 onwards if fallback_data: # in case no product was found we need to use fallback fallback_type = fallback_data["type"] diff --git a/server/settings/conversion.py b/server/settings/conversion.py index b29c827cae..d99483d21f 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -4,8 +4,8 @@ from typing import Any from .publish_plugins import DEFAULT_PUBLISH_VALUES -def _convert_imageio_configs_0_4_3(overrides): - """Imageio config settings did change to profiles since 0.4.3.""" +def _convert_imageio_configs_0_4_4(overrides): + """Imageio config settings did change to profiles since 0.4.4.""" imageio_overrides = overrides.get("imageio") or {} # make sure settings are already converted to profiles @@ -119,6 +119,6 @@ def convert_settings_overrides( overrides: dict[str, Any], ) -> dict[str, Any]: _convert_imageio_configs_0_3_1(overrides) - _convert_imageio_configs_0_4_3(overrides) + _convert_imageio_configs_0_4_4(overrides) _conver_publish_plugins(overrides) return overrides From 1e026d8fcb10e053615b1795edc5157600935e32 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 1 Aug 2024 15:50:45 +0200 Subject: [PATCH 03/45] Refactor config data retrieval logic in colorspace module - Removed redundant folder info handling - Added fallback mechanism for missing folder info --- client/ayon_core/pipeline/colorspace.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 106d43d55a..57a36286db 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -788,12 +788,6 @@ def _get_global_config_data( # TODO decide if this is the right name for representation repre_name = "ocioconfig" - folder_info = template_data.get("folder") - if not folder_info: - log.warning("Folder info is missing.") - return None - folder_path = folder_info["path"] - # Backward compatibility for old projects # TODO remove in future 0.4.5 onwards product_name = profile.get("product_name") @@ -809,6 +803,23 @@ def _get_global_config_data( product_name = published_product_data["product_name"] fallback_data = published_product_data["fallback"] + folder_info = template_data.get("folder") + if not folder_info: + log.warning("Folder info is missing.") + + # TODO: this fallback should be required after backward compatibility + # is removed + if fallback_data: + log.info("Using fallback data for ocio config path.") + # in case no product was found we need to use fallback + fallback_type = fallback_data["type"] + return _get_config_path_from_profile_data( + fallback_data, fallback_type, template_data + ) + return None + + folder_path = folder_info["path"] + if folder_id is None: folder_entity = ayon_api.get_folder_by_path( project_name, folder_path, fields={"id"} @@ -827,6 +838,7 @@ def _get_global_config_data( fields={"id", "name"} ) } + if not product_entities_by_name: # TODO: make this required in future 0.4.5 onwards if fallback_data: @@ -874,6 +886,7 @@ def _get_global_config_data( path = get_representation_path_with_anatomy(repre_entity, anatomy) template = repre_entity["attrib"]["template"] + return { "path": path, "template": template, From f9e0b05bf2dac3f596a9c53f32368219072970dd Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Aug 2024 13:27:05 +0200 Subject: [PATCH 04/45] Refactor fallback handling in colorspace module Improve handling of fallback data for OCIO config path. Simplified logic and error messages for better clarity. --- client/ayon_core/pipeline/colorspace.py | 51 +++++++++---------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 57a36286db..82025fabaf 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -788,35 +788,27 @@ def _get_global_config_data( # TODO decide if this is the right name for representation repre_name = "ocioconfig" - # Backward compatibility for old projects - # TODO remove in future 0.4.5 onwards - product_name = profile.get("product_name") - # TODO: this should be required after backward compatibility is removed - fallback_data = None - published_product_data = profile.get("published_product") + published_product_data = profile["published_product"] + product_name = published_product_data["product_name"] + fallback_data = published_product_data["fallback"] - if product_name is None and published_product_data is None: - log.warning("Product name or published product is missing.") + if product_name == "": + log.error( + "Colorspace OCIO config path cannot be set. " + "Profile is set to published product but `Product name` is empty." + ) return None - if published_product_data: - product_name = published_product_data["product_name"] - fallback_data = published_product_data["fallback"] - folder_info = template_data.get("folder") if not folder_info: log.warning("Folder info is missing.") - # TODO: this fallback should be required after backward compatibility - # is removed - if fallback_data: - log.info("Using fallback data for ocio config path.") - # in case no product was found we need to use fallback - fallback_type = fallback_data["type"] - return _get_config_path_from_profile_data( - fallback_data, fallback_type, template_data - ) - return None + log.info("Using fallback data for ocio config path.") + # in case no product was found we need to use fallback + fallback_type = fallback_data["type"] + return _get_config_path_from_profile_data( + fallback_data, fallback_type, template_data + ) folder_path = folder_info["path"] @@ -840,18 +832,11 @@ def _get_global_config_data( } if not product_entities_by_name: - # TODO: make this required in future 0.4.5 onwards - if fallback_data: - # in case no product was found we need to use fallback - fallback_type = fallback_data["type"] - return _get_config_path_from_profile_data( - fallback_data, fallback_type, template_data - ) - log.debug( - f"No product entities were found for folder '{folder_path}' with" - f" product name filter '{product_name}'." + # in case no product was found we need to use fallback + fallback_type = fallback_data["type"] + return _get_config_path_from_profile_data( + fallback_data, fallback_type, template_data ) - return None # Try to use exact match first, otherwise use first available product product_entity = product_entities_by_name.get(product_name) From d51a04c8ac2b43f2b181709c728e7bda895e7658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Fri, 9 Aug 2024 13:31:25 +0200 Subject: [PATCH 05/45] Update client/ayon_core/pipeline/colorspace.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/colorspace.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 82025fabaf..867c3ec22a 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -700,18 +700,19 @@ def get_ocio_config_views(config_path): def _get_config_path_from_profile_data( - data, data_type, template_data) -> dict: + profile, profile_type, template_data +): """Get config path from profile data. Args: - data (dict[str, Any]): Profile data. - data_type (str): Profile type. + profile(dict[str, Any]): Profile data. + profile_type (str): Profile type. template_data (dict[str, Any]): Template data. Returns: dict[str, str]: Config data with path and template. """ - template = data[data_type] + template = profile[profile_type] result = StringTemplate.format_strict_template( template, template_data ) From f87057506a63c12fcc9835e4a093f60da9e1c52c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 9 Aug 2024 13:34:46 +0200 Subject: [PATCH 06/45] adding space --- client/ayon_core/pipeline/colorspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 867c3ec22a..1e6f98f272 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -705,7 +705,7 @@ def _get_config_path_from_profile_data( """Get config path from profile data. Args: - profile(dict[str, Any]): Profile data. + profile (dict[str, Any]): Profile data. profile_type (str): Profile type. template_data (dict[str, Any]): Template data. From fe9cef1c8fe931324ac880d4bb68a66e431b84d5 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Thu, 22 Aug 2024 23:07:14 +0300 Subject: [PATCH 07/45] Easier settings for ExtractOIIOTranscode --- server/settings/publish_plugins.py | 42 +++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/server/settings/publish_plugins.py b/server/settings/publish_plugins.py index 8ca96432f4..cce1891fda 100644 --- a/server/settings/publish_plugins.py +++ b/server/settings/publish_plugins.py @@ -57,7 +57,7 @@ class CollectFramesFixDefModel(BaseSettingsModel): True, title="Show 'Rewrite latest version' toggle" ) - + class ContributionLayersModel(BaseSettingsModel): _layout = "compact" @@ -256,8 +256,8 @@ class ExtractThumbnailModel(BaseSettingsModel): def _extract_oiio_transcoding_type(): return [ - {"value": "colorspace", "label": "Use Colorspace"}, - {"value": "display", "label": "Use Display&View"} + {"value": "use_colorspace", "label": "Use Colorspace"}, + {"value": "use_display_view", "label": "Use Display&View"} ] @@ -266,6 +266,17 @@ class OIIOToolArgumentsModel(BaseSettingsModel): default_factory=list, title="Arguments") +class UseColorspaceModel(BaseSettingsModel): + _layout = "expanded" + colorspace: str = SettingsField("", title="Target Colorspace") + + +class UseDisplayViewModel(BaseSettingsModel): + _layout = "expanded" + display: str = SettingsField("", title="Target Display") + view: str = SettingsField("", title="Target View") + + class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): _layout = "expanded" name: str = SettingsField( @@ -276,13 +287,20 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): ) extension: str = SettingsField("", title="Extension") transcoding_type: str = SettingsField( - "colorspace", + "use_colorspace", title="Transcoding type", - enum_resolver=_extract_oiio_transcoding_type + enum_resolver=_extract_oiio_transcoding_type, + conditionalEnum=True ) - colorspace: str = SettingsField("", title="Colorspace") - display: str = SettingsField("", title="Display") - view: str = SettingsField("", title="View") + use_colorspace: UseColorspaceModel = SettingsField( + title="Use Colorspace", + default_factory=UseColorspaceModel + ) + use_display_view: UseDisplayViewModel = SettingsField( + title="Use Display&View", + default_factory=UseDisplayViewModel + ) + oiiotool_args: OIIOToolArgumentsModel = SettingsField( default_factory=OIIOToolArgumentsModel, title="OIIOtool arguments") @@ -360,7 +378,7 @@ class ExtractReviewFFmpegModel(BaseSettingsModel): def extract_review_filter_enum(): return [ { - "value": "everytime", + "value": "everytime", # codespell:ignore everytime "label": "Always" }, { @@ -382,7 +400,7 @@ class ExtractReviewFilterModel(BaseSettingsModel): default_factory=list, title="Custom Tags" ) single_frame_filter: str = SettingsField( - "everytime", + "everytime", # codespell:ignore everytime description=( "Use output always / only if input is 1 frame" " image / only if has 2+ frames or is video" @@ -780,7 +798,7 @@ class IntegrateHeroVersionModel(BaseSettingsModel): class CleanUpModel(BaseSettingsModel): _isGroup = True - paterns: list[str] = SettingsField( + paterns: list[str] = SettingsField( # codespell:ignore paterns default_factory=list, title="Patterns (regex)" ) @@ -1200,7 +1218,7 @@ DEFAULT_PUBLISH_VALUES = { "use_hardlinks": False }, "CleanUp": { - "paterns": [], + "paterns": [], # codespell:ignore paterns "remove_temp_renders": False }, "CleanUpFarm": { From 00d7f6c378225b7643450605587bab0724243a3a Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Thu, 22 Aug 2024 23:08:42 +0300 Subject: [PATCH 08/45] adopt to the new settings --- .../plugins/publish/extract_color_transcode.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index a28a761e7e..c5f90e4c99 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -122,12 +122,15 @@ class ExtractOIIOTranscode(publish.Extractor): transcoding_type = output_def["transcoding_type"] target_colorspace = view = display = None - if transcoding_type == "colorspace": - target_colorspace = (output_def["colorspace"] or + # NOTE: we use colorspace_data as the fallback values for the target colorspace. + if transcoding_type == "use_colorspace": + # TODO: Should we fallback to the colorspace (which used as source above) ? + # or should we compute the target colorspace from current view and display ? + target_colorspace = (output_def["use_colorspace"]["colorspace"] or colorspace_data.get("colorspace")) - else: - view = output_def["view"] or colorspace_data.get("view") - display = (output_def["display"] or + elif transcoding_type == "use_display_view": + view = output_def["use_display_view"]["view"] or colorspace_data.get("view") + display = (output_def["use_display_view"]["display"] or colorspace_data.get("display")) # both could be already collected by DCC, @@ -192,7 +195,7 @@ class ExtractOIIOTranscode(publish.Extractor): new_repre["files"] = new_repre["files"][0] # If the source representation has "review" tag, but its not - # part of the output defintion tags, then both the + # part of the output definition tags, then both the # representations will be transcoded in ExtractReview and # their outputs will clash in integration. if "review" in repre.get("tags", []): From d90310079cbd7a81a032be3f3d50f7346805ff76 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Tue, 3 Sep 2024 15:13:36 +0300 Subject: [PATCH 09/45] refactor settings names --- .../plugins/publish/extract_color_transcode.py | 15 +++++++++------ server/settings/publish_plugins.py | 18 +++++------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index c5f90e4c99..77ab6d0428 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -123,15 +123,18 @@ class ExtractOIIOTranscode(publish.Extractor): target_colorspace = view = display = None # NOTE: we use colorspace_data as the fallback values for the target colorspace. - if transcoding_type == "use_colorspace": + if transcoding_type == "colorspace": # TODO: Should we fallback to the colorspace (which used as source above) ? # or should we compute the target colorspace from current view and display ? - target_colorspace = (output_def["use_colorspace"]["colorspace"] or + target_colorspace = (output_def["colorspace"] or colorspace_data.get("colorspace")) - elif transcoding_type == "use_display_view": - view = output_def["use_display_view"]["view"] or colorspace_data.get("view") - display = (output_def["use_display_view"]["display"] or - colorspace_data.get("display")) + elif transcoding_type == "display_view": + display_view = output_def["display_view"] + view = display_view["view"] or colorspace_data.get("view") + display = ( + display_view["display"] + or colorspace_data.get("display") + ) # both could be already collected by DCC, # but could be overwritten when transcoding diff --git a/server/settings/publish_plugins.py b/server/settings/publish_plugins.py index cbfab4c5c6..98a50e6101 100644 --- a/server/settings/publish_plugins.py +++ b/server/settings/publish_plugins.py @@ -267,8 +267,8 @@ class ExtractThumbnailModel(BaseSettingsModel): def _extract_oiio_transcoding_type(): return [ - {"value": "use_colorspace", "label": "Use Colorspace"}, - {"value": "use_display_view", "label": "Use Display&View"} + {"value": "colorspace", "label": "Use Colorspace"}, + {"value": "display_view", "label": "Use Display&View"} ] @@ -277,11 +277,6 @@ class OIIOToolArgumentsModel(BaseSettingsModel): default_factory=list, title="Arguments") -class UseColorspaceModel(BaseSettingsModel): - _layout = "expanded" - colorspace: str = SettingsField("", title="Target Colorspace") - - class UseDisplayViewModel(BaseSettingsModel): _layout = "expanded" display: str = SettingsField("", title="Target Display") @@ -298,16 +293,13 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): ) extension: str = SettingsField("", title="Extension") transcoding_type: str = SettingsField( - "use_colorspace", + "colorspace", title="Transcoding type", enum_resolver=_extract_oiio_transcoding_type, conditionalEnum=True ) - use_colorspace: UseColorspaceModel = SettingsField( - title="Use Colorspace", - default_factory=UseColorspaceModel - ) - use_display_view: UseDisplayViewModel = SettingsField( + colorspace: str = SettingsField("", title="Target Colorspace") + display_view: UseDisplayViewModel = SettingsField( title="Use Display&View", default_factory=UseDisplayViewModel ) From 49448fc3ffb13d04281e9bb691ab443f0ed2dbd2 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Thu, 5 Sep 2024 16:08:03 +0300 Subject: [PATCH 10/45] fix running builder on start if scene is the default scene --- .../workfile/workfile_template_builder.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 7b15dff049..1fce07a722 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -533,29 +533,20 @@ class AbstractTemplateBuilder(ABC): if create_first_version: created_version_workfile = self.create_first_workfile_version() - # if first version is created, import template - # and populate placeholders + # Build the template if ( create_first_version and workfile_creation_enabled - and created_version_workfile + and (created_version_workfile or self.host.get_current_workfile() is None) ): self.import_template(template_path) self.populate_scene_placeholders( level_limit, keep_placeholders) - # save workfile after template is populated + # save workfile if a workfile was created. + if created_version_workfile: self.save_workfile(created_version_workfile) - # ignore process if first workfile is enabled - # but a version is already created - if workfile_creation_enabled: - return - - self.import_template(template_path) - self.populate_scene_placeholders( - level_limit, keep_placeholders) - def rebuild_template(self): """Go through existing placeholders in scene and update them. From e4881e2fec7599863e311fd9124db89bc35fd2b8 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Thu, 5 Sep 2024 22:04:55 +0300 Subject: [PATCH 11/45] refactor and simplify `build_template` method --- .../workfile/workfile_template_builder.py | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 1fce07a722..283408452e 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -512,39 +512,28 @@ class AbstractTemplateBuilder(ABC): process if version is created """ - if any( - value is None - for value in [ - template_path, - keep_placeholders, - create_first_version, - ] - ): - template_preset = self.get_template_preset() - if template_path is None: - template_path = template_preset["path"] + # Get default values if not provided + if template_path is None or keep_placeholders is None or create_first_version is None: + preset = self.get_template_preset() + template_path = template_path or preset["path"] if keep_placeholders is None: - keep_placeholders = template_preset["keep_placeholder"] + keep_placeholders = preset["keep_placeholder"] if create_first_version is None: - create_first_version = template_preset["create_first_version"] + create_first_version = preset["create_first_version"] # check if first version is created - created_version_workfile = False - if create_first_version: - created_version_workfile = self.create_first_workfile_version() + created_version_workfile = create_first_version and self.create_first_workfile_version() # Build the template if ( - create_first_version - and workfile_creation_enabled - and (created_version_workfile or self.host.get_current_workfile() is None) + not self.host.get_current_workfile() or created_version_workfile ): self.import_template(template_path) self.populate_scene_placeholders( level_limit, keep_placeholders) # save workfile if a workfile was created. - if created_version_workfile: + if workfile_creation_enabled and created_version_workfile: self.save_workfile(created_version_workfile) def rebuild_template(self): From 274585ced6e7a7c1c4582e0c28ddf0346b4a9039 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Fri, 6 Sep 2024 11:41:21 +0300 Subject: [PATCH 12/45] Optimize the code --- .../workfile/workfile_template_builder.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 283408452e..abee71f8ec 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -521,19 +521,26 @@ class AbstractTemplateBuilder(ABC): if create_first_version is None: create_first_version = preset["create_first_version"] - # check if first version is created + # Create, open and return the first workfile version. + # This only works if there are no existent files and + # create_first_version is enabled. created_version_workfile = create_first_version and self.create_first_workfile_version() - # Build the template - if ( - not self.host.get_current_workfile() or created_version_workfile - ): + # # Abort the process if workfile_creation_enabled. + # if workfile_creation_enabled: + # return + + # Build the template if the current scene is empty + # or if we have created new file. + # which basically avoids any + if not self.host.get_current_workfile() or created_version_workfile: + self.log.info(f"Building the workfile template: {template_path}") self.import_template(template_path) self.populate_scene_placeholders( level_limit, keep_placeholders) # save workfile if a workfile was created. - if workfile_creation_enabled and created_version_workfile: + if created_version_workfile: self.save_workfile(created_version_workfile) def rebuild_template(self): @@ -833,11 +840,10 @@ class AbstractTemplateBuilder(ABC): keep_placeholder = True if not path: - raise TemplateLoadFailed(( - "Template path is not set.\n" - "Path need to be set in {}\\Template Workfile Build " - "Settings\\Profiles" - ).format(host_name.title())) + self.log.info( + "Template path is not set." + ) + return # Try fill path with environments and anatomy roots anatomy = Anatomy(project_name) From df203db3d7fa5c9e2d8877f550372c96b8496616 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 6 Sep 2024 17:16:28 +0200 Subject: [PATCH 13/45] Refactor the `build_template` function for readability --- .../workfile/workfile_template_builder.py | 68 +++++++++++-------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index abee71f8ec..c535173038 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -15,6 +15,7 @@ import os import re import collections import copy +import warnings from abc import ABC, abstractmethod from ayon_api import ( @@ -489,7 +490,7 @@ class AbstractTemplateBuilder(ABC): level_limit=None, keep_placeholders=None, create_first_version=None, - workfile_creation_enabled=False + workfile_creation_enabled=None ): """Main callback for building workfile from template path. @@ -507,41 +508,49 @@ class AbstractTemplateBuilder(ABC): hosts to decide if they want to remove placeholder after it is used. create_first_version (bool): create first version of a workfile - workfile_creation_enabled (bool): If True, it might create - first version but ignore - process if version is created + workfile_creation_enabled (Any): Deprecated unused variable. """ + + if workfile_creation_enabled is not None: + warnings.warn( + ( + "Using deprecated argument `workfile_creation_enabled`. " + "It has been removed because it remained unused." + ), + category=DeprecationWarning + ) + # Get default values if not provided - if template_path is None or keep_placeholders is None or create_first_version is None: + if ( + template_path is None + or keep_placeholders is None + or create_first_version is None + ): preset = self.get_template_preset() - template_path = template_path or preset["path"] + template_path: str = template_path or preset["path"] if keep_placeholders is None: - keep_placeholders = preset["keep_placeholder"] + keep_placeholders: bool = preset["keep_placeholder"] if create_first_version is None: - create_first_version = preset["create_first_version"] + create_first_version: bool = preset["create_first_version"] - # Create, open and return the first workfile version. - # This only works if there are no existent files and - # create_first_version is enabled. - created_version_workfile = create_first_version and self.create_first_workfile_version() - - # # Abort the process if workfile_creation_enabled. - # if workfile_creation_enabled: - # return - - # Build the template if the current scene is empty - # or if we have created new file. - # which basically avoids any - if not self.host.get_current_workfile() or created_version_workfile: + # Build the template if current workfile is a new unsaved file + # (that's detected by checking if it returns any current filepath) + if not self.host.get_current_workfile(): self.log.info(f"Building the workfile template: {template_path}") self.import_template(template_path) self.populate_scene_placeholders( level_limit, keep_placeholders) - # save workfile if a workfile was created. - if created_version_workfile: - self.save_workfile(created_version_workfile) + if not create_first_version: + # Do not save whatsoever + return + + # If there is no existing workfile, save the first version + workfile_path = self.get_first_workfile_version_to_create() + if workfile_path: + self.log.info("Saving first workfile: %s", workfile_path) + self.save_workfile(workfile_path) def rebuild_template(self): """Go through existing placeholders in scene and update them. @@ -595,7 +604,7 @@ class AbstractTemplateBuilder(ABC): pass - def create_first_workfile_version(self): + def get_first_workfile_version_to_create(self): """ Create first version of workfile. @@ -605,7 +614,12 @@ class AbstractTemplateBuilder(ABC): Args: template_path (str): Fullpath for current task and host's template file. + + Return: + Optional[str]: Filepath to save to, if we should save. """ + # AYON_LAST_WORKFILE will be set to the last existing workfile OR + # if none exist it will be set to the first version. last_workfile_path = os.environ.get("AYON_LAST_WORKFILE") self.log.info("__ last_workfile_path: {}".format(last_workfile_path)) if os.path.exists(last_workfile_path): @@ -613,10 +627,6 @@ class AbstractTemplateBuilder(ABC): self.log.info("Workfile already exists, skipping creation.") return False - # Create first version - self.log.info("Creating first version of workfile.") - self.save_workfile(last_workfile_path) - # Confirm creation of first version return last_workfile_path From a653ed64e45a20f35f449e8d84a180d5651e4a78 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 6 Sep 2024 17:31:24 +0200 Subject: [PATCH 14/45] :tada: `workfile_creation_enabled` argument was actually useful after all --- .../workfile/workfile_template_builder.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index c535173038..9986e8ddae 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -15,7 +15,6 @@ import os import re import collections import copy -import warnings from abc import ABC, abstractmethod from ayon_api import ( @@ -508,19 +507,12 @@ class AbstractTemplateBuilder(ABC): hosts to decide if they want to remove placeholder after it is used. create_first_version (bool): create first version of a workfile - workfile_creation_enabled (Any): Deprecated unused variable. + workfile_creation_enabled (Any): Whether we are creating a new + workfile. If False, we assume we just want to build the + template in our current scene. """ - if workfile_creation_enabled is not None: - warnings.warn( - ( - "Using deprecated argument `workfile_creation_enabled`. " - "It has been removed because it remained unused." - ), - category=DeprecationWarning - ) - # Get default values if not provided if ( template_path is None @@ -542,6 +534,12 @@ class AbstractTemplateBuilder(ABC): self.populate_scene_placeholders( level_limit, keep_placeholders) + if not workfile_creation_enabled: + # Do not consider saving a first workfile version, if this + # is not set to be a "workfile creation". Then we assume + # only the template should get build. + return + if not create_first_version: # Do not save whatsoever return From b470ff214d2c05dbd367e9bf6438caa768aaf39c Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 6 Sep 2024 17:33:55 +0200 Subject: [PATCH 15/45] Elaborate `create_first_version` argument to differentiate from `workfile_creation_enabled` --- .../ayon_core/pipeline/workfile/workfile_template_builder.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 9986e8ddae..c28b156e46 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -506,7 +506,10 @@ class AbstractTemplateBuilder(ABC): keep_placeholders (bool): Add flag to placeholder data for hosts to decide if they want to remove placeholder after it is used. - create_first_version (bool): create first version of a workfile + create_first_version (bool): Create first version of a workfile. + When set to True, this option initiates the saving of the + workfile for an initial version. It will skip saving if + a version already exists. workfile_creation_enabled (Any): Whether we are creating a new workfile. If False, we assume we just want to build the template in our current scene. From 0ca2e1044f0efdd3e25cde6ce391b2cd4ab6d8f7 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 6 Sep 2024 17:35:43 +0200 Subject: [PATCH 16/45] Fix argument signature --- .../ayon_core/pipeline/workfile/workfile_template_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index c28b156e46..e2c06db328 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -489,7 +489,7 @@ class AbstractTemplateBuilder(ABC): level_limit=None, keep_placeholders=None, create_first_version=None, - workfile_creation_enabled=None + workfile_creation_enabled=False ): """Main callback for building workfile from template path. @@ -510,7 +510,7 @@ class AbstractTemplateBuilder(ABC): When set to True, this option initiates the saving of the workfile for an initial version. It will skip saving if a version already exists. - workfile_creation_enabled (Any): Whether we are creating a new + workfile_creation_enabled (bool): Whether we are creating a new workfile. If False, we assume we just want to build the template in our current scene. From 0faaadab51b0e7b74c483a32e406f90b49f45636 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Fri, 6 Sep 2024 17:47:25 +0200 Subject: [PATCH 17/45] Fix case of explicit load import and improve docstring --- .../workfile/workfile_template_builder.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index b24a4ab653..8bd9c00773 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -511,8 +511,10 @@ class AbstractTemplateBuilder(ABC): workfile for an initial version. It will skip saving if a version already exists. workfile_creation_enabled (bool): Whether we are creating a new - workfile. If False, we assume we just want to build the - template in our current scene. + workfile. If False, we assume we explicitly want to build the + template in our current scene. Otherwise we only build if the + current file is not an existing saved workfile but a "new" + file. """ @@ -531,7 +533,14 @@ class AbstractTemplateBuilder(ABC): # Build the template if current workfile is a new unsaved file # (that's detected by checking if it returns any current filepath) - if not self.host.get_current_workfile(): + if ( + # If not a workfile creation, an explicit load template + # was requested, so we always want to build the template + not workfile_creation_enabled + # Or if workfile creation, but we're not in an active file + # we still need to build the "new workfile template" + or not self.host.get_current_workfile() + ): self.log.info(f"Building the workfile template: {template_path}") self.import_template(template_path) self.populate_scene_placeholders( From a27157f46d90b8f08fc858583dc4dd85ffc15940 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 15:58:20 +0200 Subject: [PATCH 18/45] Please @fabiaserra's eyes with better indentation --- .../pipeline/workfile/workfile_template_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 8bd9c00773..aaa8a951c2 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -520,9 +520,9 @@ class AbstractTemplateBuilder(ABC): # Get default values if not provided if ( - template_path is None - or keep_placeholders is None - or create_first_version is None + template_path is None + or keep_placeholders is None + or create_first_version is None ): preset = self.get_template_preset() template_path: str = template_path or preset["path"] From 3df68ef2a5b1e6c8bda457b96ebfe153480e1bbe Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 15:59:32 +0200 Subject: [PATCH 19/45] `workfile_creation_enabled` docstring, first explain the `True` case. --- .../pipeline/workfile/workfile_template_builder.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index aaa8a951c2..9537096794 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -511,10 +511,11 @@ class AbstractTemplateBuilder(ABC): workfile for an initial version. It will skip saving if a version already exists. workfile_creation_enabled (bool): Whether we are creating a new - workfile. If False, we assume we explicitly want to build the - template in our current scene. Otherwise we only build if the - current file is not an existing saved workfile but a "new" - file. + workfile. When True, we only build if the current file is not + an existing saved workfile but a "new" file. When False, the + default value, we assume we explicitly want to build the + template in our current scene regardless of current scene + state. """ From e7f7eaba680a779323f1a4cd1b28d68d8c346593 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 16:01:57 +0200 Subject: [PATCH 20/45] Elaborate more --- .../workfile/workfile_template_builder.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 9537096794..86cb4b3f86 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -510,12 +510,15 @@ class AbstractTemplateBuilder(ABC): When set to True, this option initiates the saving of the workfile for an initial version. It will skip saving if a version already exists. - workfile_creation_enabled (bool): Whether we are creating a new - workfile. When True, we only build if the current file is not - an existing saved workfile but a "new" file. When False, the - default value, we assume we explicitly want to build the - template in our current scene regardless of current scene - state. + workfile_creation_enabled (bool): Whether the call is part of + creating a new workfile. + When True, we only build if the current file is not + an existing saved workfile but a "new" file. Basically when + enabled we assume the user tries to load it only into a + "New File" (unsaved empty workfile). + When False, the default value, we assume we explicitly want to + build the template in our current scene regardless of current + scene state. """ From 522b205a92625eb5c2188f8f6e0d5b6a7bb6e72b Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 16:07:17 +0200 Subject: [PATCH 21/45] Refactor logic with better function names --- .../workfile/workfile_template_builder.py | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 86cb4b3f86..7ea69b3f8f 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -561,10 +561,14 @@ class AbstractTemplateBuilder(ABC): return # If there is no existing workfile, save the first version - workfile_path = self.get_first_workfile_version_to_create() - if workfile_path: + workfile_path = self.get_workfile_path() + if not os.path.exists(workfile_path): self.log.info("Saving first workfile: %s", workfile_path) self.save_workfile(workfile_path) + else: + self.log.info( + "A workfile already exists. Skipping save of workfile as " + "initial version.") def rebuild_template(self): """Go through existing placeholders in scene and update them. @@ -618,30 +622,16 @@ class AbstractTemplateBuilder(ABC): pass - def get_first_workfile_version_to_create(self): - """ - Create first version of workfile. - - Should load the content of template into scene so - 'populate_scene_placeholders' can be started. - - Args: - template_path (str): Fullpath for current task and - host's template file. + def get_workfile_path(self): + """Return last known workfile path or the first workfile path create. Return: - Optional[str]: Filepath to save to, if we should save. + str: Last workfile path, or first version to create if none exist. """ # AYON_LAST_WORKFILE will be set to the last existing workfile OR # if none exist it will be set to the first version. last_workfile_path = os.environ.get("AYON_LAST_WORKFILE") self.log.info("__ last_workfile_path: {}".format(last_workfile_path)) - if os.path.exists(last_workfile_path): - # ignore in case workfile existence - self.log.info("Workfile already exists, skipping creation.") - return False - - # Confirm creation of first version return last_workfile_path def save_workfile(self, workfile_path): From d65b84cf2fea40c615fdf04a14831175e1e37c59 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 16:09:21 +0200 Subject: [PATCH 22/45] Tweak comment to describe both the cases in the if statement --- .../ayon_core/pipeline/workfile/workfile_template_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 7ea69b3f8f..8495529924 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -535,8 +535,8 @@ class AbstractTemplateBuilder(ABC): if create_first_version is None: create_first_version: bool = preset["create_first_version"] - # Build the template if current workfile is a new unsaved file - # (that's detected by checking if it returns any current filepath) + # Build the template if we are explicitly requesting it or if it's + # an unsaved "new file". if ( # If not a workfile creation, an explicit load template # was requested, so we always want to build the template From cfcce2138347a0928e42021384960a87797bc567 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 16:14:25 +0200 Subject: [PATCH 23/45] Separate into variables for readability --- .../pipeline/workfile/workfile_template_builder.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 8495529924..673ca7922e 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -537,14 +537,9 @@ class AbstractTemplateBuilder(ABC): # Build the template if we are explicitly requesting it or if it's # an unsaved "new file". - if ( - # If not a workfile creation, an explicit load template - # was requested, so we always want to build the template - not workfile_creation_enabled - # Or if workfile creation, but we're not in an active file - # we still need to build the "new workfile template" - or not self.host.get_current_workfile() - ): + is_new_file = not self.host.get_current_workfile() + explicit_build_requested = not workfile_creation_enabled + if is_new_file or explicit_build_requested: self.log.info(f"Building the workfile template: {template_path}") self.import_template(template_path) self.populate_scene_placeholders( From 057a5fffcf96a6c7dc48efb845e21c47de935d6a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Mon, 9 Sep 2024 17:23:26 +0200 Subject: [PATCH 24/45] Simplify logic --- .../pipeline/workfile/workfile_template_builder.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 673ca7922e..026eacc19f 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -545,14 +545,9 @@ class AbstractTemplateBuilder(ABC): self.populate_scene_placeholders( level_limit, keep_placeholders) - if not workfile_creation_enabled: - # Do not consider saving a first workfile version, if this - # is not set to be a "workfile creation". Then we assume - # only the template should get build. - return - - if not create_first_version: - # Do not save whatsoever + # Do not consider saving a first workfile version, if this is not set + # to be a "workfile creation" or `create_first_version` is disabled. + if explicit_build_requested or not create_first_version: return # If there is no existing workfile, save the first version From e29ade4e7d36d3f0def6bf4b8c998b3fd3fb446d Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Wed, 11 Sep 2024 11:49:09 +0300 Subject: [PATCH 25/45] Add tool tips for `ExtractOIIOTranscode` settings --- server/settings/publish_plugins.py | 72 ++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/server/settings/publish_plugins.py b/server/settings/publish_plugins.py index 98a50e6101..cdcd28a9ce 100644 --- a/server/settings/publish_plugins.py +++ b/server/settings/publish_plugins.py @@ -274,13 +274,30 @@ def _extract_oiio_transcoding_type(): class OIIOToolArgumentsModel(BaseSettingsModel): additional_command_args: list[str] = SettingsField( - default_factory=list, title="Arguments") + default_factory=list, + title="Arguments", + description="Additional command line arguments for *oiiotool*." + ) class UseDisplayViewModel(BaseSettingsModel): _layout = "expanded" - display: str = SettingsField("", title="Target Display") - view: str = SettingsField("", title="Target View") + display: str = SettingsField( + "", + title="Target Display", + description=( + "Display of the target transform. If left empty, the" + " source Display value will be used." + ) + ) + view: str = SettingsField( + "", + title="Target View", + description=( + "View of the target transform. If left empty, the" + " source View value will be used." + ) + ) class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): @@ -291,14 +308,35 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): description="Output name (no space)", regex=r"[a-zA-Z0-9_]([a-zA-Z0-9_\.\-]*[a-zA-Z0-9_])?$", ) - extension: str = SettingsField("", title="Extension") + extension: str = SettingsField( + "", + title="Extension", + description=( + "Target extension. If left empty, original" + " extension is used." + ), + ) transcoding_type: str = SettingsField( "colorspace", title="Transcoding type", enum_resolver=_extract_oiio_transcoding_type, - conditionalEnum=True + conditionalEnum=True, + description=( + "Select the transcoding type for your output, choosing either " + "*Colorspace* or *Display&View* transform." + " Only one option can be applied per output definition." + ), + ) + colorspace: str = SettingsField( + "", + title="Target Colorspace", + description=( + "Choose the desired target colorspace, confirming its availability" + " in the active OCIO config. If left empty, the" + " source colorspace value will be used, resulting in no" + " colorspace conversion." + ) ) - colorspace: str = SettingsField("", title="Target Colorspace") display_view: UseDisplayViewModel = SettingsField( title="Use Display&View", default_factory=UseDisplayViewModel @@ -308,9 +346,19 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): default_factory=OIIOToolArgumentsModel, title="OIIOtool arguments") - tags: list[str] = SettingsField(default_factory=list, title="Tags") + tags: list[str] = SettingsField( + default_factory=list, + title="Tags", + description=( + "Additional tags that will be added to the created representation." + "\nAdd *review* tag to create review from the transcoded" + " representation instead of the original." + ) + ) custom_tags: list[str] = SettingsField( - default_factory=list, title="Custom Tags" + default_factory=list, + title="Custom Tags", + description="Additional custom tags that will be added to the created representation." ) @@ -338,7 +386,13 @@ class ExtractOIIOTranscodeProfileModel(BaseSettingsModel): ) delete_original: bool = SettingsField( True, - title="Delete Original Representation" + title="Delete Original Representation", + description=( + "Choose to preserve or remove the original representation.\n" + "Keep in mind that if the transcoded representation includes" + " a `review` tag, it will take precedence over" + " the original for creating reviews." + ), ) outputs: list[ExtractOIIOTranscodeOutputModel] = SettingsField( default_factory=list, From ab498dd7e1edd5f4db0003dcb314aa016dfa889a Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 12 Sep 2024 15:03:14 +0200 Subject: [PATCH 26/45] Update client/ayon_core/pipeline/workfile/workfile_template_builder.py Co-authored-by: Mustafa Jafar --- .../pipeline/workfile/workfile_template_builder.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index a908e76222..17debdb2e8 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -844,10 +844,11 @@ class AbstractTemplateBuilder(ABC): keep_placeholder = True if not path: - self.log.info( - "Template path is not set." - ) - return + raise TemplateLoadFailed(( + "Template path is not set.\n" + "Path need to be set in {}\\Template Workfile Build " + "Settings\\Profiles" + ).format(host_name.title())) # Try to fill path with environments and anatomy roots anatomy = Anatomy(project_name) From 3c5603c9241560b578b28d4b27ab8eb3da486085 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Mon, 16 Sep 2024 23:47:41 +0300 Subject: [PATCH 27/45] fix line lengths --- .../ayon_core/plugins/publish/extract_color_transcode.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index 77ab6d0428..3e54d324e3 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -122,10 +122,13 @@ class ExtractOIIOTranscode(publish.Extractor): transcoding_type = output_def["transcoding_type"] target_colorspace = view = display = None - # NOTE: we use colorspace_data as the fallback values for the target colorspace. + # NOTE: we use colorspace_data as the fallback values for + # the target colorspace. if transcoding_type == "colorspace": - # TODO: Should we fallback to the colorspace (which used as source above) ? - # or should we compute the target colorspace from current view and display ? + # TODO: Should we fallback to the colorspace + # (which used as source above) ? + # or should we compute the target colorspace from + # current view and display ? target_colorspace = (output_def["colorspace"] or colorspace_data.get("colorspace")) elif transcoding_type == "display_view": From 98918b1dd4e4140f93f23de8f16178d43494cd65 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Fri, 20 Sep 2024 00:23:28 +0300 Subject: [PATCH 28/45] Add `ExtractOIIOTranscode` settings override --- server/settings/conversion.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index f513738603..7eed13bb69 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -71,10 +71,36 @@ def _convert_validate_version_0_3_3(publish_overrides): validate_version["plugin_state_profiles"] = [profile] +def _convert_oiio_transcode_0_4_5(publish_overrides): + """ExtractOIIOTranscode plugin changed in 0.4.5.""" + if "ExtractOIIOTranscode" not in publish_overrides: + return + + transcode_profiles = publish_overrides["ExtractOIIOTranscode"]["profiles"] + + for profile in transcode_profiles: + for output in profile["outputs"]: + transcode_type = output["transcoding_type"] + if transcode_type == "display": + output["transcoding_type"] = "display_view" + # Already new settings + if "display_view" in output: + continue + + output["display_view"] = {} + if output["display"]: + output["display_view"].update({"display": output["display"]}) + output.pop("display") + if output["view"]: + output["display_view"].update({"view": output["view"]}) + output.pop("view") + + def _conver_publish_plugins(overrides): if "publish" not in overrides: return _convert_validate_version_0_3_3(overrides["publish"]) + _convert_oiio_transcode_0_4_5(overrides["publish"]) def convert_settings_overrides( From 451d5e77cd4edeadd5cc18bcf14fdc9d350c3a5c Mon Sep 17 00:00:00 2001 From: Mustafa Jafar Date: Mon, 23 Sep 2024 12:13:23 +0300 Subject: [PATCH 29/45] exit _convert_oiio_transcode_0_4_5 if there are no profiles Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/conversion.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 7eed13bb69..06a1c2c02b 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -76,7 +76,9 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): if "ExtractOIIOTranscode" not in publish_overrides: return - transcode_profiles = publish_overrides["ExtractOIIOTranscode"]["profiles"] + transcode_profiles = publish_overrides["ExtractOIIOTranscode"].get("profiles") + if not transcode_profiles: + return for profile in transcode_profiles: for output in profile["outputs"]: From 453995ac5795489bd5eb7390717021c832a321b1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Sep 2024 14:16:39 +0200 Subject: [PATCH 30/45] Update imageio config conversion to 0.4.5 version Adjust imageio config conversion function to handle changes in settings from 0.4.4 to 0.4.5, ensuring proper profile usage and plugin conversion consistency. --- server/settings/conversion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index d99483d21f..634e5ab438 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -4,8 +4,8 @@ from typing import Any from .publish_plugins import DEFAULT_PUBLISH_VALUES -def _convert_imageio_configs_0_4_4(overrides): - """Imageio config settings did change to profiles since 0.4.4.""" +def _convert_imageio_configs_0_4_5(overrides): + """Imageio config settings did change to profiles since 0.4.5.""" imageio_overrides = overrides.get("imageio") or {} # make sure settings are already converted to profiles @@ -119,6 +119,6 @@ def convert_settings_overrides( overrides: dict[str, Any], ) -> dict[str, Any]: _convert_imageio_configs_0_3_1(overrides) - _convert_imageio_configs_0_4_4(overrides) + _convert_imageio_configs_0_4_5(overrides) _conver_publish_plugins(overrides) return overrides From 6085b6bd8237b7049714ecf5f03c20cfd738d086 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Sep 2024 14:17:26 +0200 Subject: [PATCH 31/45] Refactor imageio settings conversion logic Simplified conditional check for ocio_config_profiles presence. --- server/settings/conversion.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 634e5ab438..3b51e46dba 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -9,9 +9,7 @@ def _convert_imageio_configs_0_4_5(overrides): imageio_overrides = overrides.get("imageio") or {} # make sure settings are already converted to profiles - if ( - "ocio_config_profiles" not in imageio_overrides - ): + if "ocio_config_profiles" not in imageio_overrides: return ocio_config_profiles = imageio_overrides["ocio_config_profiles"] From 07ea80d2d70fd611b7687eba45e8d9a3fe294405 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Sep 2024 14:23:31 +0200 Subject: [PATCH 32/45] Update fallback type field names in colorspace and settings modules The commit updates the field name from "type" to "fallback_type" for consistency in the colorspace and settings modules. --- client/ayon_core/pipeline/colorspace.py | 2 +- server/settings/main.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/colorspace.py b/client/ayon_core/pipeline/colorspace.py index 1e6f98f272..8c4f97ab1c 100644 --- a/client/ayon_core/pipeline/colorspace.py +++ b/client/ayon_core/pipeline/colorspace.py @@ -806,7 +806,7 @@ def _get_global_config_data( log.info("Using fallback data for ocio config path.") # in case no product was found we need to use fallback - fallback_type = fallback_data["type"] + fallback_type = fallback_data["fallback_type"] return _get_config_path_from_profile_data( fallback_data, fallback_type, template_data ) diff --git a/server/settings/main.py b/server/settings/main.py index 09c9bf0065..249bab85fd 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -85,7 +85,7 @@ def _ocio_built_in_paths(): class FallbackProductModel(BaseSettingsModel): _layout = "expanded" - type: str = SettingsField( + fallback_type: str = SettingsField( title="Fallback config type", enum_resolver=_fallback_ocio_config_profile_types, conditionalEnum=True, @@ -347,7 +347,7 @@ DEFAULT_VALUES = { "published_product": { "product_name": "", "fallback": { - "type": "builtin_path", + "fallback_type": "builtin_path", "builtin_path": "ACES 1.2", "custom_path": "" } From 6aa31e8788dbeb316b2c3efcc480675aa9386cc0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Sep 2024 14:24:52 +0200 Subject: [PATCH 33/45] fixing typo Changed function name from '_conver_publish_plugins' to '_convert_publish_plugins' for consistency and clarity. Updated the function call accordingly in 'convert_settings_overrides'. --- server/settings/conversion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 3b51e46dba..b933d5856f 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -106,7 +106,7 @@ def _convert_validate_version_0_3_3(publish_overrides): validate_version["plugin_state_profiles"] = [profile] -def _conver_publish_plugins(overrides): +def _convert_publish_plugins(overrides): if "publish" not in overrides: return _convert_validate_version_0_3_3(overrides["publish"]) @@ -118,5 +118,5 @@ def convert_settings_overrides( ) -> dict[str, Any]: _convert_imageio_configs_0_3_1(overrides) _convert_imageio_configs_0_4_5(overrides) - _conver_publish_plugins(overrides) + _convert_publish_plugins(overrides) return overrides From 85752510f0fe90a91e5f6a1c774264a8b788e64f Mon Sep 17 00:00:00 2001 From: Mustafa Jafar Date: Thu, 26 Sep 2024 11:57:49 +0300 Subject: [PATCH 34/45] optimize `_convert_oiio_transcode_0_4_5` Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/conversion.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 06a1c2c02b..0a345e2059 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -81,21 +81,25 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): return for profile in transcode_profiles: - for output in profile["outputs"]: - transcode_type = output["transcoding_type"] + outputs = profile.get("outputs") + if outputs is None: + return + + for output in outputs : + # Already new settings + if "display" not in output and "view" not in output: + break + + # Fix 'display' -> 'display_view' in 'transcoding_type' + transcode_type = output.get("transcoding_type") if transcode_type == "display": output["transcoding_type"] = "display_view" - # Already new settings - if "display_view" in output: - continue - - output["display_view"] = {} - if output["display"]: - output["display_view"].update({"display": output["display"]}) - output.pop("display") - if output["view"]: - output["display_view"].update({"view": output["view"]}) - output.pop("view") + + # Convert 'display' and 'view' to new values + output["display_view"] = { + "display": output.pop("display", ""), + "view": output.pop("view", ""), + } def _conver_publish_plugins(overrides): From 6f2e69c3f7f24a0499a430d6f282647a672d4428 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Thu, 26 Sep 2024 12:02:39 +0300 Subject: [PATCH 35/45] fix code linting --- server/settings/conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 0a345e2059..2337492ef2 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -94,7 +94,7 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): transcode_type = output.get("transcoding_type") if transcode_type == "display": output["transcoding_type"] = "display_view" - + # Convert 'display' and 'view' to new values output["display_view"] = { "display": output.pop("display", ""), From fa54805d94152b92bcb3c0a7903f2a10528e6409 Mon Sep 17 00:00:00 2001 From: Mustafa Jafar Date: Thu, 26 Sep 2024 12:06:28 +0300 Subject: [PATCH 36/45] enhance a condition in `_convert_oiio_transcode_0_4_5` Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 2337492ef2..c855863591 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -87,7 +87,7 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): for output in outputs : # Already new settings - if "display" not in output and "view" not in output: + if "display_view" in output: break # Fix 'display' -> 'display_view' in 'transcoding_type' From e299b405fc01fc6d5daac5d40a6da1b3955853c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 2 Oct 2024 10:01:33 +0200 Subject: [PATCH 37/45] Update server/settings/conversion.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/conversion.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index b933d5856f..aabf41f8d3 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -14,30 +14,19 @@ def _convert_imageio_configs_0_4_5(overrides): ocio_config_profiles = imageio_overrides["ocio_config_profiles"] - for inx, profile in enumerate(ocio_config_profiles): - if profile["type"] != "product_name": + for profile in ocio_config_profiles: + if profile.get("type") != "product_name": continue - - # create new profile - new_profile = { - "type": "published_product", - "published_product": { - "product_name": profile["product_name"], - "fallback": { - "type": "builtin_path", - "builtin_path": "{BUILTIN_OCIO_ROOT}/aces_1.2/config.ocio", - }, + + profile["type"] = "published_product" + profile["published_product"] = { + "product_name": profile.pop("product_name"), + "fallback": { + "type": "builtin_path", + "builtin_path": "{BUILTIN_OCIO_ROOT}/aces_1.2/config.ocio", }, - "host_names": profile["host_names"], - "task_names": profile["task_names"], - "task_types": profile["task_types"], - "custom_path": profile["custom_path"], - "builtin_path": profile["builtin_path"], } - # replace old profile with new profile - ocio_config_profiles[inx] = new_profile - def _convert_imageio_configs_0_3_1(overrides): """Imageio config settings did change to profiles since 0.3.1. .""" From 8b14e79629058d4c44f2ecc89f1262bb0a157825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Wed, 2 Oct 2024 10:03:28 +0200 Subject: [PATCH 38/45] Update server/settings/conversion.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/conversion.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index aabf41f8d3..315f5a2027 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -9,11 +9,10 @@ def _convert_imageio_configs_0_4_5(overrides): imageio_overrides = overrides.get("imageio") or {} # make sure settings are already converted to profiles - if "ocio_config_profiles" not in imageio_overrides: + ocio_config_profiles = imageio_overrides.get("ocio_config_profiles") + if not ocio_config_profiles: return - ocio_config_profiles = imageio_overrides["ocio_config_profiles"] - for profile in ocio_config_profiles: if profile.get("type") != "product_name": continue From 589a642d69e85d4a9b9b6364b57cfa3a4d20624b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 2 Oct 2024 10:10:57 +0200 Subject: [PATCH 39/45] Merge branch 'develop' into enhancement/AY-6198_OCIO-fallback-for-profiles-and-templated-values --- .github/workflows/release_trigger.yml | 12 +++ client/ayon_core/hooks/pre_ocio_hook.py | 3 +- client/ayon_core/lib/path_tools.py | 5 +- client/ayon_core/pipeline/__init__.py | 2 - client/ayon_core/pipeline/constants.py | 17 ---- client/ayon_core/plugins/load/delivery.py | 18 +++- .../publish/extract_color_transcode.py | 19 +++- client/ayon_core/plugins/publish/integrate.py | 7 +- .../plugins/publish/validate_file_saved.py | 3 +- client/ayon_core/tools/publisher/window.py | 5 +- server/settings/conversion.py | 35 ++++++- server/settings/publish_plugins.py | 92 ++++++++++++++++--- 12 files changed, 172 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/release_trigger.yml diff --git a/.github/workflows/release_trigger.yml b/.github/workflows/release_trigger.yml new file mode 100644 index 0000000000..01a3b3a682 --- /dev/null +++ b/.github/workflows/release_trigger.yml @@ -0,0 +1,12 @@ +name: 🚀 Release Trigger + +on: + workflow_dispatch: + +jobs: + call-release-trigger: + uses: ynput/ops-repo-automation/.github/workflows/release_trigger.yml@main + secrets: + token: ${{ secrets.YNPUT_BOT_TOKEN }} + email: ${{ secrets.CI_EMAIL }} + user: ${{ secrets.CI_USER }} diff --git a/client/ayon_core/hooks/pre_ocio_hook.py b/client/ayon_core/hooks/pre_ocio_hook.py index 6c30b267bc..7406aa42cf 100644 --- a/client/ayon_core/hooks/pre_ocio_hook.py +++ b/client/ayon_core/hooks/pre_ocio_hook.py @@ -19,7 +19,8 @@ class OCIOEnvHook(PreLaunchHook): "nuke", "hiero", "resolve", - "openrv" + "openrv", + "cinema4d" } launch_types = set() diff --git a/client/ayon_core/lib/path_tools.py b/client/ayon_core/lib/path_tools.py index a65f0f8e13..5c81fbfebf 100644 --- a/client/ayon_core/lib/path_tools.py +++ b/client/ayon_core/lib/path_tools.py @@ -81,7 +81,10 @@ def collect_frames(files): dict: {'/folder/product_v001.0001.png': '0001', ....} """ - patterns = [clique.PATTERNS["frames"]] + # clique.PATTERNS["frames"] supports only `.1001.exr` not `_1001.exr` so + # we use a customized pattern. + pattern = "[_.](?P(?P0*)\\d+)\\.\\D+\\d?$" + patterns = [pattern] collections, remainder = clique.assemble( files, minimum_items=1, patterns=patterns) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 4fcea60d5e..8e89029e7b 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -3,7 +3,6 @@ from .constants import ( AVALON_INSTANCE_ID, AYON_CONTAINER_ID, AYON_INSTANCE_ID, - HOST_WORKFILE_EXTENSIONS, ) from .anatomy import Anatomy @@ -114,7 +113,6 @@ __all__ = ( "AVALON_INSTANCE_ID", "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", - "HOST_WORKFILE_EXTENSIONS", # --- Anatomy --- "Anatomy", diff --git a/client/ayon_core/pipeline/constants.py b/client/ayon_core/pipeline/constants.py index 7a08cbb3aa..e6156b3138 100644 --- a/client/ayon_core/pipeline/constants.py +++ b/client/ayon_core/pipeline/constants.py @@ -4,20 +4,3 @@ AYON_INSTANCE_ID = "ayon.create.instance" # Backwards compatibility AVALON_CONTAINER_ID = "pyblish.avalon.container" AVALON_INSTANCE_ID = "pyblish.avalon.instance" - -# TODO get extensions from host implementations -HOST_WORKFILE_EXTENSIONS = { - "blender": [".blend"], - "celaction": [".scn"], - "tvpaint": [".tvpp"], - "fusion": [".comp"], - "harmony": [".zip"], - "houdini": [".hip", ".hiplc", ".hipnc"], - "maya": [".ma", ".mb"], - "nuke": [".nk"], - "hiero": [".hrox"], - "photoshop": [".psd", ".psb"], - "premiere": [".prproj"], - "resolve": [".drp"], - "aftereffects": [".aep"] -} diff --git a/client/ayon_core/plugins/load/delivery.py b/client/ayon_core/plugins/load/delivery.py index 6a0947cc42..5c53d170eb 100644 --- a/client/ayon_core/plugins/load/delivery.py +++ b/client/ayon_core/plugins/load/delivery.py @@ -230,6 +230,11 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): self.log ] + # TODO: This will currently incorrectly detect 'resources' + # that are published along with the publish, because those should + # not adhere to the template directly but are ingested in a + # customized way. For example, maya look textures or any publish + # that directly adds files into `instance.data["transfers"]` src_paths = [] for repre_file in repre["files"]: src_path = self.anatomy.fill_root(repre_file["path"]) @@ -261,7 +266,18 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): frame = dst_frame if frame is not None: - anatomy_data["frame"] = frame + if repre["context"].get("frame"): + anatomy_data["frame"] = frame + elif repre["context"].get("udim"): + anatomy_data["udim"] = frame + else: + # Fallback + self.log.warning( + "Representation context has no frame or udim" + " data. Supplying sequence frame to '{frame}'" + " formatting data." + ) + anatomy_data["frame"] = frame new_report_items, uploaded = deliver_single_file(*args) report_items.update(new_report_items) self._update_progress(uploaded) diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index a28a761e7e..3e54d324e3 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -122,13 +122,22 @@ class ExtractOIIOTranscode(publish.Extractor): transcoding_type = output_def["transcoding_type"] target_colorspace = view = display = None + # NOTE: we use colorspace_data as the fallback values for + # the target colorspace. if transcoding_type == "colorspace": + # TODO: Should we fallback to the colorspace + # (which used as source above) ? + # or should we compute the target colorspace from + # current view and display ? target_colorspace = (output_def["colorspace"] or colorspace_data.get("colorspace")) - else: - view = output_def["view"] or colorspace_data.get("view") - display = (output_def["display"] or - colorspace_data.get("display")) + elif transcoding_type == "display_view": + display_view = output_def["display_view"] + view = display_view["view"] or colorspace_data.get("view") + display = ( + display_view["display"] + or colorspace_data.get("display") + ) # both could be already collected by DCC, # but could be overwritten when transcoding @@ -192,7 +201,7 @@ class ExtractOIIOTranscode(publish.Extractor): new_repre["files"] = new_repre["files"][0] # If the source representation has "review" tag, but its not - # part of the output defintion tags, then both the + # part of the output definition tags, then both the # representations will be transcoded in ExtractReview and # their outputs will clash in integration. if "review" in repre.get("tags", []): diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index d3f6c04333..e8fe09bab7 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -509,8 +509,11 @@ class IntegrateAsset(pyblish.api.InstancePlugin): if not is_sequence_representation: files = [files] - if any(os.path.isabs(fname) for fname in files): - raise KnownPublishError("Given file names contain full paths") + for fname in files: + if os.path.isabs(fname): + raise KnownPublishError( + f"Representation file names contains full paths: {fname}" + ) if not is_sequence_representation: return diff --git a/client/ayon_core/plugins/publish/validate_file_saved.py b/client/ayon_core/plugins/publish/validate_file_saved.py index d132ba8d3a..f52998cef3 100644 --- a/client/ayon_core/plugins/publish/validate_file_saved.py +++ b/client/ayon_core/plugins/publish/validate_file_saved.py @@ -36,7 +36,8 @@ class ValidateCurrentSaveFile(pyblish.api.ContextPlugin): label = "Validate File Saved" order = pyblish.api.ValidatorOrder - 0.1 - hosts = ["fusion", "houdini", "max", "maya", "nuke", "substancepainter"] + hosts = ["fusion", "houdini", "max", "maya", "nuke", "substancepainter", + "cinema4d"] actions = [SaveByVersionUpAction, ShowWorkfilesAction] def process(self, context): diff --git a/client/ayon_core/tools/publisher/window.py b/client/ayon_core/tools/publisher/window.py index a8ca605ecb..434c2ca602 100644 --- a/client/ayon_core/tools/publisher/window.py +++ b/client/ayon_core/tools/publisher/window.py @@ -439,10 +439,13 @@ class PublisherWindow(QtWidgets.QDialog): def make_sure_is_visible(self): if self._window_is_visible: self.setWindowState(QtCore.Qt.WindowActive) - else: self.show() + self.raise_() + self.activateWindow() + self.showNormal() + def showEvent(self, event): self._window_is_visible = True super().showEvent(event) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 315f5a2027..34820b5b32 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -16,7 +16,7 @@ def _convert_imageio_configs_0_4_5(overrides): for profile in ocio_config_profiles: if profile.get("type") != "product_name": continue - + profile["type"] = "published_product" profile["published_product"] = { "product_name": profile.pop("product_name"), @@ -94,10 +94,43 @@ def _convert_validate_version_0_3_3(publish_overrides): validate_version["plugin_state_profiles"] = [profile] +def _convert_oiio_transcode_0_4_5(publish_overrides): + """ExtractOIIOTranscode plugin changed in 0.4.5.""" + if "ExtractOIIOTranscode" not in publish_overrides: + return + + transcode_profiles = publish_overrides["ExtractOIIOTranscode"].get( + "profiles") + if not transcode_profiles: + return + + for profile in transcode_profiles: + outputs = profile.get("outputs") + if outputs is None: + return + + for output in outputs: + # Already new settings + if "display_view" in output: + break + + # Fix 'display' -> 'display_view' in 'transcoding_type' + transcode_type = output.get("transcoding_type") + if transcode_type == "display": + output["transcoding_type"] = "display_view" + + # Convert 'display' and 'view' to new values + output["display_view"] = { + "display": output.pop("display", ""), + "view": output.pop("view", ""), + } + + def _convert_publish_plugins(overrides): if "publish" not in overrides: return _convert_validate_version_0_3_3(overrides["publish"]) + _convert_oiio_transcode_0_4_5(overrides["publish"]) def convert_settings_overrides( diff --git a/server/settings/publish_plugins.py b/server/settings/publish_plugins.py index 61972e64c4..cdcd28a9ce 100644 --- a/server/settings/publish_plugins.py +++ b/server/settings/publish_plugins.py @@ -268,13 +268,36 @@ class ExtractThumbnailModel(BaseSettingsModel): def _extract_oiio_transcoding_type(): return [ {"value": "colorspace", "label": "Use Colorspace"}, - {"value": "display", "label": "Use Display&View"} + {"value": "display_view", "label": "Use Display&View"} ] class OIIOToolArgumentsModel(BaseSettingsModel): additional_command_args: list[str] = SettingsField( - default_factory=list, title="Arguments") + default_factory=list, + title="Arguments", + description="Additional command line arguments for *oiiotool*." + ) + + +class UseDisplayViewModel(BaseSettingsModel): + _layout = "expanded" + display: str = SettingsField( + "", + title="Target Display", + description=( + "Display of the target transform. If left empty, the" + " source Display value will be used." + ) + ) + view: str = SettingsField( + "", + title="Target View", + description=( + "View of the target transform. If left empty, the" + " source View value will be used." + ) + ) class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): @@ -285,22 +308,57 @@ class ExtractOIIOTranscodeOutputModel(BaseSettingsModel): description="Output name (no space)", regex=r"[a-zA-Z0-9_]([a-zA-Z0-9_\.\-]*[a-zA-Z0-9_])?$", ) - extension: str = SettingsField("", title="Extension") + extension: str = SettingsField( + "", + title="Extension", + description=( + "Target extension. If left empty, original" + " extension is used." + ), + ) transcoding_type: str = SettingsField( "colorspace", title="Transcoding type", - enum_resolver=_extract_oiio_transcoding_type + enum_resolver=_extract_oiio_transcoding_type, + conditionalEnum=True, + description=( + "Select the transcoding type for your output, choosing either " + "*Colorspace* or *Display&View* transform." + " Only one option can be applied per output definition." + ), ) - colorspace: str = SettingsField("", title="Colorspace") - display: str = SettingsField("", title="Display") - view: str = SettingsField("", title="View") + colorspace: str = SettingsField( + "", + title="Target Colorspace", + description=( + "Choose the desired target colorspace, confirming its availability" + " in the active OCIO config. If left empty, the" + " source colorspace value will be used, resulting in no" + " colorspace conversion." + ) + ) + display_view: UseDisplayViewModel = SettingsField( + title="Use Display&View", + default_factory=UseDisplayViewModel + ) + oiiotool_args: OIIOToolArgumentsModel = SettingsField( default_factory=OIIOToolArgumentsModel, title="OIIOtool arguments") - tags: list[str] = SettingsField(default_factory=list, title="Tags") + tags: list[str] = SettingsField( + default_factory=list, + title="Tags", + description=( + "Additional tags that will be added to the created representation." + "\nAdd *review* tag to create review from the transcoded" + " representation instead of the original." + ) + ) custom_tags: list[str] = SettingsField( - default_factory=list, title="Custom Tags" + default_factory=list, + title="Custom Tags", + description="Additional custom tags that will be added to the created representation." ) @@ -328,7 +386,13 @@ class ExtractOIIOTranscodeProfileModel(BaseSettingsModel): ) delete_original: bool = SettingsField( True, - title="Delete Original Representation" + title="Delete Original Representation", + description=( + "Choose to preserve or remove the original representation.\n" + "Keep in mind that if the transcoded representation includes" + " a `review` tag, it will take precedence over" + " the original for creating reviews." + ), ) outputs: list[ExtractOIIOTranscodeOutputModel] = SettingsField( default_factory=list, @@ -371,7 +435,7 @@ class ExtractReviewFFmpegModel(BaseSettingsModel): def extract_review_filter_enum(): return [ { - "value": "everytime", + "value": "everytime", # codespell:ignore everytime "label": "Always" }, { @@ -393,7 +457,7 @@ class ExtractReviewFilterModel(BaseSettingsModel): default_factory=list, title="Custom Tags" ) single_frame_filter: str = SettingsField( - "everytime", + "everytime", # codespell:ignore everytime description=( "Use output always / only if input is 1 frame" " image / only if has 2+ frames or is video" @@ -791,7 +855,7 @@ class IntegrateHeroVersionModel(BaseSettingsModel): class CleanUpModel(BaseSettingsModel): _isGroup = True - paterns: list[str] = SettingsField( + paterns: list[str] = SettingsField( # codespell:ignore paterns default_factory=list, title="Patterns (regex)" ) @@ -1225,7 +1289,7 @@ DEFAULT_PUBLISH_VALUES = { "use_hardlinks": False }, "CleanUp": { - "paterns": [], + "paterns": [], # codespell:ignore paterns "remove_temp_renders": False }, "CleanUpFarm": { From 0b9cea926cf8d061128321708d10f395453cf18c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 2 Oct 2024 10:12:51 +0200 Subject: [PATCH 40/45] Refactor code for better readability and consistency. - Refactored code to improve readability by adjusting indentation and line breaks. - Made changes to ensure consistent formatting of the code. --- server/settings/conversion.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/settings/conversion.py b/server/settings/conversion.py index 04ab0f8d81..34820b5b32 100644 --- a/server/settings/conversion.py +++ b/server/settings/conversion.py @@ -99,7 +99,8 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): if "ExtractOIIOTranscode" not in publish_overrides: return - transcode_profiles = publish_overrides["ExtractOIIOTranscode"].get("profiles") + transcode_profiles = publish_overrides["ExtractOIIOTranscode"].get( + "profiles") if not transcode_profiles: return @@ -108,7 +109,7 @@ def _convert_oiio_transcode_0_4_5(publish_overrides): if outputs is None: return - for output in outputs : + for output in outputs: # Already new settings if "display_view" in output: break From 7298ddb745e003ca8e317eae3baccab246a2ad37 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Wed, 2 Oct 2024 12:27:32 +0300 Subject: [PATCH 41/45] add note about using more accurate variable names --- .../ayon_core/pipeline/workfile/workfile_template_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/workfile/workfile_template_builder.py b/client/ayon_core/pipeline/workfile/workfile_template_builder.py index 17debdb2e8..4412e4489b 100644 --- a/client/ayon_core/pipeline/workfile/workfile_template_builder.py +++ b/client/ayon_core/pipeline/workfile/workfile_template_builder.py @@ -521,6 +521,9 @@ class AbstractTemplateBuilder(ABC): scene state. """ + # More accurate variable name + # - logic related to workfile creation should be moved out in future + explicit_build_requested = not workfile_creation_enabled # Get default values if not provided if ( @@ -538,7 +541,6 @@ class AbstractTemplateBuilder(ABC): # Build the template if we are explicitly requesting it or if it's # an unsaved "new file". is_new_file = not self.host.get_current_workfile() - explicit_build_requested = not workfile_creation_enabled if is_new_file or explicit_build_requested: self.log.info(f"Building the workfile template: {template_path}") self.import_template(template_path) From db41e53511870112df72631ad5123566ca04e943 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:40:01 +0200 Subject: [PATCH 42/45] updated create package script --- create_package.py | 514 +++++++++++++++++++++++++++++----------------- 1 file changed, 326 insertions(+), 188 deletions(-) diff --git a/create_package.py b/create_package.py index 48952c43c5..843e993de1 100644 --- a/create_package.py +++ b/create_package.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + """Prepares server package from addon repo to upload to server. Requires Python 3.9. (Or at least 3.8+). @@ -22,32 +24,39 @@ client side code zipped in `private` subfolder. import os import sys import re +import io import shutil -import argparse import platform +import argparse import logging import collections import zipfile -import hashlib +import subprocess +from typing import Optional, Iterable, Pattern, Union, List, Tuple -from typing import Optional +import package -CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) -PACKAGE_PATH = os.path.join(CURRENT_DIR, "package.py") -package_content = {} -with open(PACKAGE_PATH, "r") as stream: - exec(stream.read(), package_content) +FileMapping = Tuple[Union[str, io.BytesIO], str] +ADDON_NAME: str = package.name +ADDON_VERSION: str = package.version +ADDON_CLIENT_DIR: Union[str, None] = getattr(package, "client_dir", None) -ADDON_VERSION = package_content["version"] -ADDON_NAME = package_content["name"] -ADDON_CLIENT_DIR = package_content["client_dir"] -CLIENT_VERSION_CONTENT = '''# -*- coding: utf-8 -*- -"""Package declaring AYON core addon version.""" -__version__ = "{}" +CURRENT_ROOT: str = os.path.dirname(os.path.abspath(__file__)) +SERVER_ROOT: str = os.path.join(CURRENT_ROOT, "server") +FRONTEND_ROOT: str = os.path.join(CURRENT_ROOT, "frontend") +FRONTEND_DIST_ROOT: str = os.path.join(FRONTEND_ROOT, "dist") +DST_DIST_DIR: str = os.path.join("frontend", "dist") +PRIVATE_ROOT: str = os.path.join(CURRENT_ROOT, "private") +PUBLIC_ROOT: str = os.path.join(CURRENT_ROOT, "public") +CLIENT_ROOT: str = os.path.join(CURRENT_ROOT, "client") + +VERSION_PY_CONTENT = f'''# -*- coding: utf-8 -*- +"""Package declaring AYON addon '{ADDON_NAME}' version.""" +__version__ = "{ADDON_VERSION}" ''' # Patterns of directories to be skipped for server part of addon -IGNORE_DIR_PATTERNS = [ +IGNORE_DIR_PATTERNS: List[Pattern] = [ re.compile(pattern) for pattern in { # Skip directories starting with '.' @@ -58,7 +67,7 @@ IGNORE_DIR_PATTERNS = [ ] # Patterns of files to be skipped for server part of addon -IGNORE_FILE_PATTERNS = [ +IGNORE_FILE_PATTERNS: List[Pattern] = [ re.compile(pattern) for pattern in { # Skip files starting with '.' @@ -70,15 +79,6 @@ IGNORE_FILE_PATTERNS = [ ] -def calculate_file_checksum(filepath, hash_algorithm, chunk_size=10000): - func = getattr(hashlib, hash_algorithm) - hash_obj = func() - with open(filepath, "rb") as f: - for chunk in iter(lambda: f.read(chunk_size), b""): - hash_obj.update(chunk) - return hash_obj.hexdigest() - - class ZipFileLongPaths(zipfile.ZipFile): """Allows longer paths in zip files. @@ -97,12 +97,28 @@ class ZipFileLongPaths(zipfile.ZipFile): else: tpath = "\\\\?\\" + tpath - return super(ZipFileLongPaths, self)._extract_member( - member, tpath, pwd - ) + return super()._extract_member(member, tpath, pwd) -def safe_copy_file(src_path, dst_path): +def _get_yarn_executable() -> Union[str, None]: + cmd = "which" + if platform.system().lower() == "windows": + cmd = "where" + + for line in subprocess.check_output( + [cmd, "yarn"], encoding="utf-8" + ).splitlines(): + if not line or not os.path.exists(line): + continue + try: + subprocess.call([line, "--version"]) + return line + except OSError: + continue + return None + + +def safe_copy_file(src_path: str, dst_path: str): """Copy file and make sure destination directory exists. Ignore if destination already contains directories from source. @@ -115,210 +131,335 @@ def safe_copy_file(src_path, dst_path): if src_path == dst_path: return - dst_dir = os.path.dirname(dst_path) - try: - os.makedirs(dst_dir) - except Exception: - pass + dst_dir: str = os.path.dirname(dst_path) + os.makedirs(dst_dir, exist_ok=True) shutil.copy2(src_path, dst_path) -def _value_match_regexes(value, regexes): - for regex in regexes: - if regex.search(value): - return True - return False +def _value_match_regexes(value: str, regexes: Iterable[Pattern]) -> bool: + return any( + regex.search(value) + for regex in regexes + ) def find_files_in_subdir( - src_path, - ignore_file_patterns=None, - ignore_dir_patterns=None -): + src_path: str, + ignore_file_patterns: Optional[List[Pattern]] = None, + ignore_dir_patterns: Optional[List[Pattern]] = None +) -> List[Tuple[str, str]]: + """Find all files to copy in subdirectories of given path. + + All files that match any of the patterns in 'ignore_file_patterns' will + be skipped and any directories that match any of the patterns in + 'ignore_dir_patterns' will be skipped with all subfiles. + + Args: + src_path (str): Path to directory to search in. + ignore_file_patterns (Optional[list[Pattern]]): List of regexes + to match files to ignore. + ignore_dir_patterns (Optional[list[Pattern]]): List of regexes + to match directories to ignore. + + Returns: + list[tuple[str, str]]: List of tuples with path to file and parent + directories relative to 'src_path'. + """ + if ignore_file_patterns is None: ignore_file_patterns = IGNORE_FILE_PATTERNS if ignore_dir_patterns is None: ignore_dir_patterns = IGNORE_DIR_PATTERNS - output = [] + output: List[Tuple[str, str]] = [] + if not os.path.exists(src_path): + return output - hierarchy_queue = collections.deque() + hierarchy_queue: collections.deque = collections.deque() hierarchy_queue.append((src_path, [])) while hierarchy_queue: - item = hierarchy_queue.popleft() + item: Tuple[str, str] = hierarchy_queue.popleft() dirpath, parents = item for name in os.listdir(dirpath): - path = os.path.join(dirpath, name) + path: str = os.path.join(dirpath, name) if os.path.isfile(path): if not _value_match_regexes(name, ignore_file_patterns): - items = list(parents) + items: List[str] = list(parents) items.append(name) output.append((path, os.path.sep.join(items))) continue if not _value_match_regexes(name, ignore_dir_patterns): - items = list(parents) + items: List[str] = list(parents) items.append(name) hierarchy_queue.append((path, items)) return output -def copy_server_content(addon_output_dir, current_dir, log): +def update_client_version(logger): + """Update version in client code if version.py is present.""" + if not ADDON_CLIENT_DIR: + return + + version_path: str = os.path.join( + CLIENT_ROOT, ADDON_CLIENT_DIR, "version.py" + ) + if not os.path.exists(version_path): + logger.debug("Did not find version.py in client directory") + return + + logger.info("Updating client version") + with open(version_path, "w") as stream: + stream.write(VERSION_PY_CONTENT) + + +def update_pyproject_toml(logger): + filepath = os.path.join(CURRENT_ROOT, "pyproject.toml") + new_lines = [] + with open(filepath, "r") as stream: + version_found = False + for line in stream.readlines(): + if not version_found and line.startswith("version ="): + line = f'version = "{ADDON_VERSION}"\n' + version_found = True + + new_lines.append(line) + + with open(filepath, "w") as stream: + stream.write("".join(new_lines)) + + +def build_frontend(): + yarn_executable = _get_yarn_executable() + if yarn_executable is None: + raise RuntimeError("Yarn executable was not found.") + + subprocess.run([yarn_executable, "install"], cwd=FRONTEND_ROOT) + subprocess.run([yarn_executable, "build"], cwd=FRONTEND_ROOT) + if not os.path.exists(FRONTEND_DIST_ROOT): + raise RuntimeError( + "Frontend build failed. Did not find 'dist' folder." + ) + + +def get_client_files_mapping() -> List[Tuple[str, str]]: + """Mapping of source client code files to destination paths. + + Example output: + [ + ( + "C:/addons/MyAddon/version.py", + "my_addon/version.py" + ), + ( + "C:/addons/MyAddon/client/my_addon/__init__.py", + "my_addon/__init__.py" + ) + ] + + Returns: + list[tuple[str, str]]: List of path mappings to copy. The destination + path is relative to expected output directory. + + """ + # Add client code content to zip + client_code_dir: str = os.path.join(CLIENT_ROOT, ADDON_CLIENT_DIR) + mapping = [ + (path, os.path.join(ADDON_CLIENT_DIR, sub_path)) + for path, sub_path in find_files_in_subdir(client_code_dir) + ] + + license_path = os.path.join(CURRENT_ROOT, "LICENSE") + if os.path.exists(license_path): + mapping.append((license_path, f"{ADDON_CLIENT_DIR}/LICENSE")) + return mapping + + +def get_client_zip_content(log) -> io.BytesIO: + log.info("Preparing client code zip") + files_mapping: List[Tuple[str, str]] = get_client_files_mapping() + stream = io.BytesIO() + with ZipFileLongPaths(stream, "w", zipfile.ZIP_DEFLATED) as zipf: + for src_path, subpath in files_mapping: + zipf.write(src_path, subpath) + stream.seek(0) + return stream + + +def get_base_files_mapping() -> List[FileMapping]: + filepaths_to_copy: List[FileMapping] = [ + ( + os.path.join(CURRENT_ROOT, "package.py"), + "package.py" + ) + ] + # Add license file to package if exists + license_path = os.path.join(CURRENT_ROOT, "LICENSE") + if os.path.exists(license_path): + filepaths_to_copy.append((license_path, "LICENSE")) + + # Go through server, private and public directories and find all files + for dirpath in (SERVER_ROOT, PRIVATE_ROOT, PUBLIC_ROOT): + if not os.path.exists(dirpath): + continue + + dirname = os.path.basename(dirpath) + for src_file, subpath in find_files_in_subdir(dirpath): + dst_subpath = os.path.join(dirname, subpath) + filepaths_to_copy.append((src_file, dst_subpath)) + + if os.path.exists(FRONTEND_DIST_ROOT): + for src_file, subpath in find_files_in_subdir(FRONTEND_DIST_ROOT): + dst_subpath = os.path.join(DST_DIST_DIR, subpath) + filepaths_to_copy.append((src_file, dst_subpath)) + + pyproject_toml = os.path.join(CLIENT_ROOT, "pyproject.toml") + if os.path.exists(pyproject_toml): + filepaths_to_copy.append( + (pyproject_toml, "private/pyproject.toml") + ) + + return filepaths_to_copy + + +def copy_client_code(output_dir: str, log: logging.Logger): """Copies server side folders to 'addon_package_dir' Args: - addon_output_dir (str): package dir in addon repo dir - current_dir (str): addon repo dir + output_dir (str): Output directory path. log (logging.Logger) + """ + log.info(f"Copying client for {ADDON_NAME}-{ADDON_VERSION}") - log.info("Copying server content") + full_output_path = os.path.join( + output_dir, f"{ADDON_NAME}_{ADDON_VERSION}" + ) + if os.path.exists(full_output_path): + shutil.rmtree(full_output_path) + os.makedirs(full_output_path, exist_ok=True) - filepaths_to_copy = [] - server_dirpath = os.path.join(current_dir, "server") - - for item in find_files_in_subdir(server_dirpath): - src_path, dst_subpath = item - dst_path = os.path.join(addon_output_dir, "server", dst_subpath) - filepaths_to_copy.append((src_path, dst_path)) - - # Copy files - for src_path, dst_path in filepaths_to_copy: + for src_path, dst_subpath in get_client_files_mapping(): + dst_path = os.path.join(full_output_path, dst_subpath) safe_copy_file(src_path, dst_path) - -def _update_client_version(client_addon_dir): - """Write version.py file to 'client' directory. - - Make sure the version in client dir is the same as in package.py. - - Args: - client_addon_dir (str): Directory path of client addon. - """ - - dst_version_path = os.path.join(client_addon_dir, "version.py") - with open(dst_version_path, "w") as stream: - stream.write(CLIENT_VERSION_CONTENT.format(ADDON_VERSION)) + log.info("Client copy finished") -def zip_client_side(addon_package_dir, current_dir, log): - """Copy and zip `client` content into 'addon_package_dir'. - - Args: - addon_package_dir (str): Output package directory path. - current_dir (str): Directory path of addon source. - log (logging.Logger): Logger object. - """ - - client_dir = os.path.join(current_dir, "client") - client_addon_dir = os.path.join(client_dir, ADDON_CLIENT_DIR) - if not os.path.isdir(client_addon_dir): - raise ValueError( - f"Failed to find client directory '{client_addon_dir}'" - ) - - log.info("Preparing client code zip") - private_dir = os.path.join(addon_package_dir, "private") - - if not os.path.exists(private_dir): - os.makedirs(private_dir) - - _update_client_version(client_addon_dir) - - zip_filepath = os.path.join(os.path.join(private_dir, "client.zip")) - with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf: - # Add client code content to zip - for path, sub_path in find_files_in_subdir(client_addon_dir): - sub_path = os.path.join(ADDON_CLIENT_DIR, sub_path) - zipf.write(path, sub_path) - - shutil.copy(os.path.join(client_dir, "pyproject.toml"), private_dir) - - -def create_server_package( +def copy_addon_package( output_dir: str, - addon_output_dir: str, + files_mapping: List[FileMapping], log: logging.Logger ): - """Create server package zip file. - - The zip file can be installed to a server using UI or rest api endpoints. + """Copy client code to output directory. Args: - output_dir (str): Directory path to output zip file. - addon_output_dir (str): Directory path to addon output directory. + output_dir (str): Directory path to output client code. + files_mapping (List[FileMapping]): List of tuples with source file + and destination subpath. log (logging.Logger): Logger object. - """ - log.info("Creating server package") + """ + log.info(f"Copying package for {ADDON_NAME}-{ADDON_VERSION}") + + # Add addon name and version to output directory + addon_output_dir: str = os.path.join( + output_dir, ADDON_NAME, ADDON_VERSION + ) + if os.path.isdir(addon_output_dir): + log.info(f"Purging {addon_output_dir}") + shutil.rmtree(addon_output_dir) + + os.makedirs(addon_output_dir, exist_ok=True) + + # Copy server content + for src_file, dst_subpath in files_mapping: + dst_path: str = os.path.join(addon_output_dir, dst_subpath) + dst_dir: str = os.path.dirname(dst_path) + os.makedirs(dst_dir, exist_ok=True) + if isinstance(src_file, io.BytesIO): + with open(dst_path, "wb") as stream: + stream.write(src_file.getvalue()) + else: + safe_copy_file(src_file, dst_path) + + log.info("Package copy finished") + + +def create_addon_package( + output_dir: str, + files_mapping: List[FileMapping], + log: logging.Logger +): + log.info(f"Creating package for {ADDON_NAME}-{ADDON_VERSION}") + + os.makedirs(output_dir, exist_ok=True) output_path = os.path.join( output_dir, f"{ADDON_NAME}-{ADDON_VERSION}.zip" ) + with ZipFileLongPaths(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: - # Move addon content to zip into 'addon' directory - addon_output_dir_offset = len(addon_output_dir) + 1 - for root, _, filenames in os.walk(addon_output_dir): - if not filenames: - continue + # Copy server content + for src_file, dst_subpath in files_mapping: + if isinstance(src_file, io.BytesIO): + zipf.writestr(dst_subpath, src_file.getvalue()) + else: + zipf.write(src_file, dst_subpath) - dst_root = None - if root != addon_output_dir: - dst_root = root[addon_output_dir_offset:] - for filename in filenames: - src_path = os.path.join(root, filename) - dst_path = filename - if dst_root: - dst_path = os.path.join(dst_root, dst_path) - zipf.write(src_path, dst_path) - - log.info(f"Output package can be found: {output_path}") + log.info("Package created") def main( - output_dir: Optional[str]=None, - skip_zip: bool=False, - keep_sources: bool=False, - clear_output_dir: bool=False + output_dir: Optional[str] = None, + skip_zip: Optional[bool] = False, + only_client: Optional[bool] = False ): - log = logging.getLogger("create_package") - log.info("Start creating package") + log: logging.Logger = logging.getLogger("create_package") + log.info("Package creation started") - current_dir = os.path.dirname(os.path.abspath(__file__)) if not output_dir: - output_dir = os.path.join(current_dir, "package") + output_dir = os.path.join(CURRENT_ROOT, "package") + has_client_code = bool(ADDON_CLIENT_DIR) + if has_client_code: + client_dir: str = os.path.join(CLIENT_ROOT, ADDON_CLIENT_DIR) + if not os.path.exists(client_dir): + raise RuntimeError( + f"Client directory was not found '{client_dir}'." + " Please check 'client_dir' in 'package.py'." + ) + update_client_version(log) - new_created_version_dir = os.path.join( - output_dir, ADDON_NAME, ADDON_VERSION - ) + update_pyproject_toml(log) - if os.path.isdir(new_created_version_dir) and clear_output_dir: - log.info(f"Purging {new_created_version_dir}") - shutil.rmtree(output_dir) + if only_client: + if not has_client_code: + raise RuntimeError("Client code is not available. Skipping") + + copy_client_code(output_dir, log) + return log.info(f"Preparing package for {ADDON_NAME}-{ADDON_VERSION}") - addon_output_root = os.path.join(output_dir, ADDON_NAME) - addon_output_dir = os.path.join(addon_output_root, ADDON_VERSION) - if not os.path.exists(addon_output_dir): - os.makedirs(addon_output_dir) + if os.path.exists(FRONTEND_ROOT): + build_frontend() - copy_server_content(addon_output_dir, current_dir, log) - safe_copy_file( - PACKAGE_PATH, - os.path.join(addon_output_dir, os.path.basename(PACKAGE_PATH)) - ) - zip_client_side(addon_output_dir, current_dir, log) + files_mapping: List[FileMapping] = [] + files_mapping.extend(get_base_files_mapping()) + + if has_client_code: + files_mapping.append( + (get_client_zip_content(log), "private/client.zip") + ) # Skip server zipping - if not skip_zip: - create_server_package(output_dir, addon_output_dir, log) - # Remove sources only if zip file is created - if not keep_sources: - log.info("Removing source files for server package") - shutil.rmtree(addon_output_root) + if skip_zip: + copy_addon_package(output_dir, files_mapping, log) + else: + create_addon_package(output_dir, files_mapping, log) + log.info("Package creation finished") @@ -333,23 +474,6 @@ if __name__ == "__main__": " server folder structure." ) ) - parser.add_argument( - "--keep-sources", - dest="keep_sources", - action="store_true", - help=( - "Keep folder structure when server package is created." - ) - ) - parser.add_argument( - "-c", "--clear-output-dir", - dest="clear_output_dir", - action="store_true", - help=( - "Clear output directory before package creation." - ) - ) - parser.add_argument( "-o", "--output", dest="output_dir", @@ -359,11 +483,25 @@ if __name__ == "__main__": " (Will be purged if already exists!)" ) ) + parser.add_argument( + "--only-client", + dest="only_client", + action="store_true", + help=( + "Extract only client code. This is useful for development." + " Requires '-o', '--output' argument to be filled." + ) + ) + parser.add_argument( + "--debug", + dest="debug", + action="store_true", + help="Debug log messages." + ) args = parser.parse_args(sys.argv[1:]) - main( - args.output_dir, - args.skip_zip, - args.keep_sources, - args.clear_output_dir - ) + level = logging.INFO + if args.debug: + level = logging.DEBUG + logging.basicConfig(level=level) + main(args.output_dir, args.skip_zip, args.only_client) From ba002d56377a5c221c80bb91fd515bade2c87dbf Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:40:13 +0200 Subject: [PATCH 43/45] bump version to '1.0.0' --- client/ayon_core/version.py | 2 +- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 3ee3c976b9..0b6322645f 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring AYON core addon version.""" -__version__ = "0.4.5-dev.1" +__version__ = "1.0.0" diff --git a/package.py b/package.py index 26c004ae84..b06959d5cf 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "0.4.5-dev.1" +version = "1.0.0" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index db98ee4eba..091cdc273d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "0.4.3-dev.1" +version = "1.0.0" description = "" authors = ["Ynput Team "] readme = "README.md" From 10ffb37518349101598a23399878c8a86957fd5e Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:41:27 +0200 Subject: [PATCH 44/45] bump version to '1.0.0+dev' --- client/ayon_core/version.py | 4 ++-- package.py | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/version.py b/client/ayon_core/version.py index 0b6322645f..75116c703e 100644 --- a/client/ayon_core/version.py +++ b/client/ayon_core/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -"""Package declaring AYON core addon version.""" -__version__ = "1.0.0" +"""Package declaring AYON addon 'core' version.""" +__version__ = "1.0.0+dev" diff --git a/package.py b/package.py index b06959d5cf..1466031daa 100644 --- a/package.py +++ b/package.py @@ -1,6 +1,6 @@ name = "core" title = "Core" -version = "1.0.0" +version = "1.0.0+dev" client_dir = "ayon_core" diff --git a/pyproject.toml b/pyproject.toml index 091cdc273d..4a63529c67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ [tool.poetry] name = "ayon-core" -version = "1.0.0" +version = "1.0.0+dev" description = "" authors = ["Ynput Team "] readme = "README.md" From a834e2c6639c22e617caa6cab0cb4dd3ee9f74f4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Oct 2024 12:43:29 +0000 Subject: [PATCH 45/45] chore(): update bug report / version --- .github/ISSUE_TEMPLATE/bug_report.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index e6badf936a..54f5d68b98 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ name: Bug Report description: File a bug report -title: 'Your issue title here' +title: Your issue title here labels: - 'type: bug' body: @@ -36,6 +36,16 @@ body: description: What version are you running? Look to AYON Tray options: - 1.0.0 + - 0.4.4 + - 0.4.3 + - 0.4.2 + - 0.4.1 + - 0.4.0 + - 0.3.2 + - 0.3.1 + - 0.3.0 + - 0.2.1 + - 0.2.0 validations: required: true - type: dropdown