From eda080d86d987775697728949973019d8d2c1a1d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 22 Jul 2024 15:59:16 +0200 Subject: [PATCH 01/11] Added profile to filter environment variables on farm --- server/settings/main.py | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 40e16e7e91..1329f465e0 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -169,6 +169,46 @@ class VersionStartCategoryModel(BaseSettingsModel): ) +class EnvironmentReplacementModel(BaseSettingsModel): + environment_key: str = SettingsField("", title="Enviroment variable") + pattern: str = SettingsField("", title="Pattern") + replacement: str = SettingsField("", title="Replacement") + + +class FilterFarmEnvironmentModel(BaseSettingsModel): + _layout = "expanded" + + hosts: list[str] = SettingsField( + default_factory=list, + title="Host names" + ) + + task_types: list[str] = SettingsField( + default_factory=list, + title="Task types", + enum_resolver=task_types_enum + ) + + task_names: list[str] = SettingsField( + default_factory=list, + title="Task names" + ) + + folders: list[str] = SettingsField( + default_factory=list, + title="Folders" + ) + + skip_environment: list[str] = SettingsField( + default_factory=list, + title="Skip environment variables" + ) + replace_in_environment: list[EnvironmentReplacementModel] = SettingsField( + default_factory=list, + title="Replace values in environment" + ) + + class CoreSettings(BaseSettingsModel): studio_name: str = SettingsField("", title="Studio name", scope=["studio"]) studio_code: str = SettingsField("", title="Studio code", scope=["studio"]) @@ -219,6 +259,9 @@ class CoreSettings(BaseSettingsModel): title="Project environments", section="---" ) + filter_farm_environment: list[FilterFarmEnvironmentModel] = SettingsField( + default_factory=list, + ) @validator( "environments", @@ -313,5 +356,6 @@ DEFAULT_VALUES = { "project_environments": json.dumps( {}, indent=4 - ) + ), + "filter_farm_environment": [], } From 274ed655e9631aacc616695028eb29b59637b226 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 22 Jul 2024 16:00:14 +0200 Subject: [PATCH 02/11] Added hook filtering farm environment variables Should be triggered only on farm. Used to modify env var on farm machines like license path etc. --- .../hooks/pre_filter_farm_environments.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 client/ayon_core/hooks/pre_filter_farm_environments.py diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py new file mode 100644 index 0000000000..9a52f53950 --- /dev/null +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -0,0 +1,78 @@ +import copy +import re + +from ayon_applications import PreLaunchHook, LaunchTypes +from ayon_core.lib import filter_profiles + + +class FilterFarmEnvironments(PreLaunchHook): + """Filter or modify calculated environment variables for farm rendering. + + This hook must run last, only after all other hooks are finished to get + correct environment for launch context. + + Implemented modifications to self.launch_context.env: + - skipping (list) of environment variable keys + - removing value in environment variable: + - supports regular expression in pattern + - doesn't remove env var if value empty! + """ + order = 1000 + + launch_types = {LaunchTypes.farm_publish} + + def execute(self): + data = self.launch_context.data + project_settings = data["project_settings"] + filter_env_profiles = ( + project_settings["core"]["filter_farm_environment"]) + + if not filter_env_profiles: + self.log.debug("No profiles found for env var filtering") + return + + task_entity = data["task_entity"] + + filter_data = { + "hosts": self.host_name, + "task_types": task_entity["taskType"], + "tasks": task_entity["name"], + "folders": data["folder_path"] + } + matching_profile = filter_profiles( + filter_env_profiles, filter_data, logger=self.log + ) + if not matching_profile: + self.log.debug("No matching profile found for env var filtering " + f"for {filter_data}") + return + + calculated_env = copy.deepcopy(self.launch_context.env) + + calculated_env = self._skip_environment_variables( + calculated_env, matching_profile) + + calculated_env = self._modify_environment_variables( + calculated_env, matching_profile) + + self.launch_context.env = calculated_env + + def _modify_environment_variables(self, calculated_env, matching_profile): + """Modify environment variable values.""" + for env_item in matching_profile["replace_in_environment"]: + value = calculated_env.get(env_item["environment_key"]) + if not value: + continue + + value = re.sub(value, env_item["pattern"], env_item["replacement"]) + calculated_env[env_item["environment_key"]] = value + + return calculated_env + + def _skip_environment_variables(self, calculated_env, matching_profile): + """Skips list of environment variable names""" + for skip_env in matching_profile["skip_environment"]: + self.log.info(f"Skipping {skip_env}") + calculated_env.pop(skip_env) + + return calculated_env From 3f4a491e8f9ce83458fc5ae76305b495690f0c57 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 22 Jul 2024 16:39:28 +0200 Subject: [PATCH 03/11] Update variable name for skipped env vars Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 1329f465e0..717897a70b 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -199,7 +199,7 @@ class FilterFarmEnvironmentModel(BaseSettingsModel): title="Folders" ) - skip_environment: list[str] = SettingsField( + skip_env_keys: list[str] = SettingsField( default_factory=list, title="Skip environment variables" ) From 291e3eaa4c217cdacde15456080ad6c963263194 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 22 Jul 2024 16:46:41 +0200 Subject: [PATCH 04/11] Update names of profile fields to be more descriptive --- client/ayon_core/hooks/pre_filter_farm_environments.py | 6 +++--- server/settings/main.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py index 9a52f53950..837116d5eb 100644 --- a/client/ayon_core/hooks/pre_filter_farm_environments.py +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -34,10 +34,10 @@ class FilterFarmEnvironments(PreLaunchHook): task_entity = data["task_entity"] filter_data = { - "hosts": self.host_name, + "host_names": self.host_name, "task_types": task_entity["taskType"], - "tasks": task_entity["name"], - "folders": data["folder_path"] + "task_names": task_entity["name"], + "folder_paths": data["folder_path"] } matching_profile = filter_profiles( filter_env_profiles, filter_data, logger=self.log diff --git a/server/settings/main.py b/server/settings/main.py index 1329f465e0..b6cfbe36ae 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -178,7 +178,7 @@ class EnvironmentReplacementModel(BaseSettingsModel): class FilterFarmEnvironmentModel(BaseSettingsModel): _layout = "expanded" - hosts: list[str] = SettingsField( + host_names: list[str] = SettingsField( default_factory=list, title="Host names" ) @@ -194,9 +194,9 @@ class FilterFarmEnvironmentModel(BaseSettingsModel): title="Task names" ) - folders: list[str] = SettingsField( + folder_paths: list[str] = SettingsField( default_factory=list, - title="Folders" + title="Folder paths" ) skip_environment: list[str] = SettingsField( From 3dc12c7954fae1dacba9e90db6505292a4583ce6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 22 Jul 2024 16:49:33 +0200 Subject: [PATCH 05/11] Simplified methods for manipulating environments --- .../hooks/pre_filter_farm_environments.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py index 837116d5eb..95ddec990c 100644 --- a/client/ayon_core/hooks/pre_filter_farm_environments.py +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -1,4 +1,3 @@ -import copy import re from ayon_applications import PreLaunchHook, LaunchTypes @@ -47,15 +46,11 @@ class FilterFarmEnvironments(PreLaunchHook): f"for {filter_data}") return - calculated_env = copy.deepcopy(self.launch_context.env) + self._skip_environment_variables( + self.launch_context.env, matching_profile) - calculated_env = self._skip_environment_variables( - calculated_env, matching_profile) - - calculated_env = self._modify_environment_variables( - calculated_env, matching_profile) - - self.launch_context.env = calculated_env + self._modify_environment_variables( + self.launch_context.env, matching_profile) def _modify_environment_variables(self, calculated_env, matching_profile): """Modify environment variable values.""" @@ -67,12 +62,8 @@ class FilterFarmEnvironments(PreLaunchHook): value = re.sub(value, env_item["pattern"], env_item["replacement"]) calculated_env[env_item["environment_key"]] = value - return calculated_env - def _skip_environment_variables(self, calculated_env, matching_profile): """Skips list of environment variable names""" for skip_env in matching_profile["skip_environment"]: self.log.info(f"Skipping {skip_env}") calculated_env.pop(skip_env) - - return calculated_env From f473e987dfa44b0985dbef9380b0a5fbe9d45cf0 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jul 2024 11:27:29 +0200 Subject: [PATCH 06/11] Fix key name from Settings --- client/ayon_core/hooks/pre_filter_farm_environments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py index 95ddec990c..cabd705d81 100644 --- a/client/ayon_core/hooks/pre_filter_farm_environments.py +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -64,6 +64,6 @@ class FilterFarmEnvironments(PreLaunchHook): def _skip_environment_variables(self, calculated_env, matching_profile): """Skips list of environment variable names""" - for skip_env in matching_profile["skip_environment"]: + for skip_env in matching_profile["skip_env_keys"]: self.log.info(f"Skipping {skip_env}") calculated_env.pop(skip_env) From 07d0bcc7526ee643b6f8c04e00a254493070b88c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Jul 2024 12:10:11 +0200 Subject: [PATCH 07/11] Remove empty environment variable --- client/ayon_core/hooks/pre_filter_farm_environments.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py index cabd705d81..0f83c0d3e0 100644 --- a/client/ayon_core/hooks/pre_filter_farm_environments.py +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -14,7 +14,6 @@ class FilterFarmEnvironments(PreLaunchHook): - skipping (list) of environment variable keys - removing value in environment variable: - supports regular expression in pattern - - doesn't remove env var if value empty! """ order = 1000 @@ -55,12 +54,16 @@ class FilterFarmEnvironments(PreLaunchHook): def _modify_environment_variables(self, calculated_env, matching_profile): """Modify environment variable values.""" for env_item in matching_profile["replace_in_environment"]: - value = calculated_env.get(env_item["environment_key"]) + key = env_item["environment_key"] + value = calculated_env.get(key) if not value: continue value = re.sub(value, env_item["pattern"], env_item["replacement"]) - calculated_env[env_item["environment_key"]] = value + if value: + calculated_env[key] = value + else: + calculated_env.pop(key) def _skip_environment_variables(self, calculated_env, matching_profile): """Skips list of environment variable names""" From aad7e2902dba233aedb99585c95192882c5b16be Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Jul 2024 14:33:55 +0200 Subject: [PATCH 08/11] Change name of variable Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 6596a9ecba..564dd92bd2 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -259,7 +259,7 @@ class CoreSettings(BaseSettingsModel): title="Project environments", section="---" ) - filter_farm_environment: list[FilterFarmEnvironmentModel] = SettingsField( + filter_env_profiles: list[FilterEnvsProfileModel] = SettingsField( default_factory=list, ) From 6b66de7daadd93cae586c8829468a7886f758cac Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Jul 2024 14:34:22 +0200 Subject: [PATCH 09/11] Change name of variable Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 564dd92bd2..986a9ed1c5 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -357,5 +357,5 @@ DEFAULT_VALUES = { {}, indent=4 ), - "filter_farm_environment": [], + "filter_env_profiles": [], } From 6a4196c5b48e133fade1aec7bc396691f42f6469 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Jul 2024 14:34:35 +0200 Subject: [PATCH 10/11] Change name of variable Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/hooks/pre_filter_farm_environments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hooks/pre_filter_farm_environments.py b/client/ayon_core/hooks/pre_filter_farm_environments.py index 0f83c0d3e0..d231acf5e9 100644 --- a/client/ayon_core/hooks/pre_filter_farm_environments.py +++ b/client/ayon_core/hooks/pre_filter_farm_environments.py @@ -23,7 +23,7 @@ class FilterFarmEnvironments(PreLaunchHook): data = self.launch_context.data project_settings = data["project_settings"] filter_env_profiles = ( - project_settings["core"]["filter_farm_environment"]) + project_settings["core"]["filter_env_profiles"]) if not filter_env_profiles: self.log.debug("No profiles found for env var filtering") From 2003ae81b40d7c12585c6ef60de250af923eff2b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Jul 2024 14:35:06 +0200 Subject: [PATCH 11/11] Change name of model Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- server/settings/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/settings/main.py b/server/settings/main.py index 986a9ed1c5..0972ccdfb9 100644 --- a/server/settings/main.py +++ b/server/settings/main.py @@ -175,7 +175,7 @@ class EnvironmentReplacementModel(BaseSettingsModel): replacement: str = SettingsField("", title="Replacement") -class FilterFarmEnvironmentModel(BaseSettingsModel): +class FilterEnvsProfileModel(BaseSettingsModel): _layout = "expanded" host_names: list[str] = SettingsField(