From 5c65a1de895b9481f0871b49cab153889e14a657 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 15 Feb 2024 09:29:58 +0100 Subject: [PATCH 01/27] moved some dependencies to runtime dependencies --- client/pyproject.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/pyproject.toml b/client/pyproject.toml index 7b4329a31a..1a0ad7e5f2 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -4,19 +4,19 @@ description="AYON core addon." [tool.poetry.dependencies] python = ">=3.9.1,<3.10" -aiohttp_json_rpc = "*" # TVPaint server -aiohttp-middlewares = "^2.0.0" -wsrpc_aiohttp = "^3.1.1" # websocket server -Click = "^8" clique = "1.6.*" jsonschema = "^2.6.0" pyblish-base = "^1.8.11" -pynput = "^1.7.2" # Timers manager - TODO remove speedcopy = "^2.1" six = "^1.15" qtawesome = "0.7.3" [ayon.runtimeDependencies] +aiohttp_json_rpc = "*" # TVPaint server +aiohttp-middlewares = "^2.0.0" +wsrpc_aiohttp = "^3.1.1" # websocket server +Click = "^8" OpenTimelineIO = "0.14.1" opencolorio = "2.2.1" Pillow = "9.5.0" +pynput = "^1.7.2" # Timers manager - TODO remove From 7d5c9c71716af7b1038274ab5d48b70ac91658d2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 12:42:59 +0100 Subject: [PATCH 02/27] use 'reformatted' tag name instead of 'reformated' --- client/ayon_core/hosts/nuke/api/plugin.py | 2 +- client/ayon_core/plugins/publish/extract_review.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client/ayon_core/hosts/nuke/api/plugin.py b/client/ayon_core/hosts/nuke/api/plugin.py index 650b67dd2c..5501e225cb 100644 --- a/client/ayon_core/hosts/nuke/api/plugin.py +++ b/client/ayon_core/hosts/nuke/api/plugin.py @@ -911,7 +911,7 @@ class ExporterReviewMov(ExporterReview): node, product_name, "Reposition node... `{}`" ) # append reformatted tag - add_tags.append("reformated") + add_tags.append("reformatted") # only create colorspace baking if toggled on if bake_viewer_process: diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 905158c851..8afb77b436 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -1225,7 +1225,11 @@ class ExtractReview(pyblish.api.InstancePlugin): filters = [] # if reformat input video file is already reforamted from upstream - reformat_in_baking = bool("reformated" in new_repre["tags"]) + reformat_in_baking = ( + "reformatted" in new_repre["tags"] + # Backwards compatibility + or "reformated" in new_repre["tags"] + ) self.log.debug("reformat_in_baking: `{}`".format(reformat_in_baking)) # Get instance data From 02cb96a1c1af0ebe54494d7a104daa5aca6ae643 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 15:36:34 +0100 Subject: [PATCH 03/27] initial addon commit --- .../addons/ayon_applications/__init__.py | 6 + .../addons/ayon_applications/addon.py | 5 + .../addons/ayon_applications/constants.py | 2 + .../addons/ayon_applications/defs.py | 404 +++++++++++ .../addons/ayon_applications/exceptions.py | 50 ++ .../addons/ayon_applications/hooks.py | 150 ++++ .../addons/ayon_applications/manager.py | 682 ++++++++++++++++++ .../addons/ayon_applications/utils.py | 609 ++++++++++++++++ 8 files changed, 1908 insertions(+) create mode 100644 client/ayon_core/addons/ayon_applications/__init__.py create mode 100644 client/ayon_core/addons/ayon_applications/addon.py create mode 100644 client/ayon_core/addons/ayon_applications/constants.py create mode 100644 client/ayon_core/addons/ayon_applications/defs.py create mode 100644 client/ayon_core/addons/ayon_applications/exceptions.py create mode 100644 client/ayon_core/addons/ayon_applications/hooks.py create mode 100644 client/ayon_core/addons/ayon_applications/manager.py create mode 100644 client/ayon_core/addons/ayon_applications/utils.py diff --git a/client/ayon_core/addons/ayon_applications/__init__.py b/client/ayon_core/addons/ayon_applications/__init__.py new file mode 100644 index 0000000000..8ecffad159 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/__init__.py @@ -0,0 +1,6 @@ +from .addon import ApplicationsAddon + + +__all__ = ( + "ApplicationsAddon", +) diff --git a/client/ayon_core/addons/ayon_applications/addon.py b/client/ayon_core/addons/ayon_applications/addon.py new file mode 100644 index 0000000000..149f547861 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/addon.py @@ -0,0 +1,5 @@ +from ayon_core.addon import AYONAddon + + +class ApplicationsAddon(AYONAddon): + name = "applications" diff --git a/client/ayon_core/addons/ayon_applications/constants.py b/client/ayon_core/addons/ayon_applications/constants.py new file mode 100644 index 0000000000..03112ee422 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/constants.py @@ -0,0 +1,2 @@ +PLATFORM_NAMES = {"windows", "linux", "darwin"} +DEFAULT_ENV_SUBGROUP = "standard" diff --git a/client/ayon_core/addons/ayon_applications/defs.py b/client/ayon_core/addons/ayon_applications/defs.py new file mode 100644 index 0000000000..5cc36041a1 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/defs.py @@ -0,0 +1,404 @@ +import os +import platform +import json +import copy + +from ayon_core.lib import find_executable + + +class LaunchTypes: + """Launch types are filters for pre/post-launch hooks. + + Please use these variables in case they'll change values. + """ + + # Local launch - application is launched on local machine + local = "local" + # Farm render job - application is on farm + farm_render = "farm-render" + # Farm publish job - integration post-render job + farm_publish = "farm-publish" + # Remote launch - application is launched on remote machine from which + # can be started publishing + remote = "remote" + # Automated launch - application is launched with automated publishing + automated = "automated" + + +class ApplicationExecutable: + """Representation of executable loaded from settings.""" + + def __init__(self, executable): + # Try to format executable with environments + try: + executable = executable.format(**os.environ) + except Exception: + pass + + # On MacOS check if exists path to executable when ends with `.app` + # - it is common that path will lead to "/Applications/Blender" but + # real path is "/Applications/Blender.app" + if platform.system().lower() == "darwin": + executable = self.macos_executable_prep(executable) + + self.executable_path = executable + + def __str__(self): + return self.executable_path + + def __repr__(self): + return "<{}> {}".format(self.__class__.__name__, self.executable_path) + + @staticmethod + def macos_executable_prep(executable): + """Try to find full path to executable file. + + Real executable is stored in '*.app/Contents/MacOS/'. + + Having path to '*.app' gives ability to read it's plist info and + use "CFBundleExecutable" key from plist to know what is "executable." + + Plist is stored in '*.app/Contents/Info.plist'. + + This is because some '*.app' directories don't have same permissions + as real executable. + """ + # Try to find if there is `.app` file + if not os.path.exists(executable): + _executable = executable + ".app" + if os.path.exists(_executable): + executable = _executable + + # Try to find real executable if executable has `Contents` subfolder + contents_dir = os.path.join(executable, "Contents") + if os.path.exists(contents_dir): + executable_filename = None + # Load plist file and check for bundle executable + plist_filepath = os.path.join(contents_dir, "Info.plist") + if os.path.exists(plist_filepath): + import plistlib + + if hasattr(plistlib, "load"): + with open(plist_filepath, "rb") as stream: + parsed_plist = plistlib.load(stream) + else: + parsed_plist = plistlib.readPlist(plist_filepath) + executable_filename = parsed_plist.get("CFBundleExecutable") + + if executable_filename: + executable = os.path.join( + contents_dir, "MacOS", executable_filename + ) + + return executable + + def as_args(self): + return [self.executable_path] + + def _realpath(self): + """Check if path is valid executable path.""" + # Check for executable in PATH + result = find_executable(self.executable_path) + if result is not None: + return result + + # This is not 100% validation but it is better than remove ability to + # launch .bat, .sh or extentionless files + if os.path.exists(self.executable_path): + return self.executable_path + return None + + def exists(self): + if not self.executable_path: + return False + return bool(self._realpath()) + + +class UndefinedApplicationExecutable(ApplicationExecutable): + """Some applications do not require executable path from settings. + + In that case this class is used to "fake" existing executable. + """ + def __init__(self): + pass + + def __str__(self): + return self.__class__.__name__ + + def __repr__(self): + return "<{}>".format(self.__class__.__name__) + + def as_args(self): + return [] + + def exists(self): + return True + + +class ApplicationGroup: + """Hold information about application group. + + Application group wraps different versions(variants) of application. + e.g. "maya" is group and "maya_2020" is variant. + + Group hold `host_name` which is implementation name used in AYON. Also + holds `enabled` if whole app group is enabled or `icon` for application + icon path in resources. + + Group has also `environment` which hold same environments for all variants. + + Args: + name (str): Groups' name. + data (dict): Group defying data loaded from settings. + manager (ApplicationManager): Manager that created the group. + """ + + def __init__(self, name, data, manager): + self.name = name + self.manager = manager + self._data = data + + self.enabled = data["enabled"] + self.label = data["label"] or None + self.icon = data["icon"] or None + env = {} + try: + env = json.loads(data["environment"]) + except Exception: + pass + self._environment = env + + host_name = data["host_name"] or None + self.is_host = host_name is not None + self.host_name = host_name + + settings_variants = data["variants"] + variants = {} + for variant_data in settings_variants: + app_variant = Application(variant_data, self) + variants[app_variant.name] = app_variant + + self.variants = variants + + def __repr__(self): + return "<{}> - {}".format(self.__class__.__name__, self.name) + + def __iter__(self): + for variant in self.variants.values(): + yield variant + + @property + def environment(self): + return copy.deepcopy(self._environment) + + +class Application: + """Hold information about application. + + Object by itself does nothing special. + + Args: + data (dict): Data for the version containing information about + executables, variant label or if is enabled. + Only required key is `executables`. + group (ApplicationGroup): App group object that created the application + and under which application belongs. + + """ + def __init__(self, data, group): + self._data = data + name = data["name"] + label = data["label"] or name + enabled = False + if group.enabled: + enabled = data.get("enabled", True) + + if group.label: + full_label = " ".join((group.label, label)) + else: + full_label = label + env = {} + try: + env = json.loads(data["environment"]) + except Exception: + pass + + arguments = data["arguments"] + if isinstance(arguments, dict): + arguments = arguments.get(platform.system().lower()) + + if not arguments: + arguments = [] + + _executables = data["executables"].get(platform.system().lower(), []) + executables = [ + ApplicationExecutable(executable) + for executable in _executables + ] + + self.group = group + + self.name = name + self.label = label + self.enabled = enabled + self.use_python_2 = data.get("use_python_2", False) + + self.full_name = "/".join((group.name, name)) + self.full_label = full_label + self.arguments = arguments + self.executables = executables + self._environment = env + + def __repr__(self): + return "<{}> - {}".format(self.__class__.__name__, self.full_name) + + @property + def environment(self): + return copy.deepcopy(self._environment) + + @property + def manager(self): + return self.group.manager + + @property + def host_name(self): + return self.group.host_name + + @property + def icon(self): + return self.group.icon + + @property + def is_host(self): + return self.group.is_host + + def find_executable(self): + """Try to find existing executable for application. + + Returns (str): Path to executable from `executables` or None if any + exists. + """ + for executable in self.executables: + if executable.exists(): + return executable + return None + + def launch(self, *args, **kwargs): + """Launch the application. + + For this purpose is used manager's launch method to keep logic at one + place. + + Arguments must match with manager's launch method. That's why *args + **kwargs are used. + + Returns: + subprocess.Popen: Return executed process as Popen object. + """ + return self.manager.launch(self.full_name, *args, **kwargs) + + +class EnvironmentToolGroup: + """Hold information about environment tool group. + + Environment tool group may hold different variants of same tool and set + environments that are same for all of them. + + e.g. "mtoa" may have different versions but all environments except one + are same. + + Args: + data (dict): Group information with variants. + manager (ApplicationManager): Manager that creates the group. + """ + + def __init__(self, data, manager): + name = data["name"] + label = data["label"] + + self.name = name + self.label = label + self._data = data + self.manager = manager + + environment = {} + try: + environment = json.loads(data["environment"]) + except Exception: + pass + self._environment = environment + + variants = data.get("variants") or [] + variants_by_name = {} + for variant_data in variants: + tool = EnvironmentTool(variant_data, self) + variants_by_name[tool.name] = tool + self.variants = variants_by_name + + def __repr__(self): + return "<{}> - {}".format(self.__class__.__name__, self.name) + + def __iter__(self): + for variant in self.variants.values(): + yield variant + + @property + def environment(self): + return copy.deepcopy(self._environment) + + +class EnvironmentTool: + """Hold information about application tool. + + Structure of tool information. + + Args: + variant_data (dict): Variant data with environments and + host and app variant filters. + group (EnvironmentToolGroup): Name of group which wraps tool. + """ + + def __init__(self, variant_data, group): + # Backwards compatibility 3.9.1 - 3.9.2 + # - 'variant_data' contained only environments but contain also host + # and application variant filters + name = variant_data["name"] + label = variant_data["label"] + host_names = variant_data["host_names"] + app_variants = variant_data["app_variants"] + + environment = {} + try: + environment = json.loads(variant_data["environment"]) + except Exception: + pass + + self.host_names = host_names + self.app_variants = app_variants + self.name = name + self.variant_label = label + self.label = " ".join((group.label, label)) + self.group = group + + self._environment = environment + self.full_name = "/".join((group.name, name)) + + def __repr__(self): + return "<{}> - {}".format(self.__class__.__name__, self.full_name) + + @property + def environment(self): + return copy.deepcopy(self._environment) + + def is_valid_for_app(self, app): + """Is tool valid for application. + + Args: + app (Application): Application for which are prepared environments. + """ + if self.app_variants and app.full_name not in self.app_variants: + return False + + if self.host_names and app.host_name not in self.host_names: + return False + return True diff --git a/client/ayon_core/addons/ayon_applications/exceptions.py b/client/ayon_core/addons/ayon_applications/exceptions.py new file mode 100644 index 0000000000..c24ba8c158 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/exceptions.py @@ -0,0 +1,50 @@ +class ApplicationNotFound(Exception): + """Application was not found in ApplicationManager by name.""" + + def __init__(self, app_name): + self.app_name = app_name + super(ApplicationNotFound, self).__init__( + "Application \"{}\" was not found.".format(app_name) + ) + + +class ApplictionExecutableNotFound(Exception): + """Defined executable paths are not available on the machine.""" + + def __init__(self, application): + self.application = application + details = None + if not application.executables: + msg = ( + "Executable paths for application \"{}\"({}) are not set." + ) + else: + msg = ( + "Defined executable paths for application \"{}\"({})" + " are not available on this machine." + ) + details = "Defined paths:" + for executable in application.executables: + details += "\n- " + executable.executable_path + + self.msg = msg.format(application.full_label, application.full_name) + self.details = details + + exc_mgs = str(self.msg) + if details: + # Is good idea to pass new line symbol to exception message? + exc_mgs += "\n" + details + self.exc_msg = exc_mgs + super(ApplictionExecutableNotFound, self).__init__(exc_mgs) + + +class ApplicationLaunchFailed(Exception): + """Application launch failed due to known reason. + + Message should be self explanatory as traceback won't be shown. + """ + pass + + +class MissingRequiredKey(KeyError): + pass diff --git a/client/ayon_core/addons/ayon_applications/hooks.py b/client/ayon_core/addons/ayon_applications/hooks.py new file mode 100644 index 0000000000..6aa12a210a --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/hooks.py @@ -0,0 +1,150 @@ +import platform +from abc import ABCMeta, abstractmethod + +import six + +from ayon_core.lib import Logger + +from .defs import LaunchTypes + + +@six.add_metaclass(ABCMeta) +class LaunchHook: + """Abstract base class of launch hook.""" + # Order of prelaunch hook, will be executed as last if set to None. + order = None + # List of host implementations, skipped if empty. + hosts = set() + # Set of application groups + app_groups = set() + # Set of specific application names + app_names = set() + # Set of platform availability + platforms = set() + # Set of launch types for which is available + # - if empty then is available for all launch types + # - by default has 'local' which is most common reason for launc hooks + launch_types = {LaunchTypes.local} + + def __init__(self, launch_context): + """Constructor of launch hook. + + Always should be called + """ + self.log = Logger.get_logger(self.__class__.__name__) + + self.launch_context = launch_context + + is_valid = self.class_validation(launch_context) + if is_valid: + is_valid = self.validate() + + self.is_valid = is_valid + + @classmethod + def class_validation(cls, launch_context): + """Validation of class attributes by launch context. + + Args: + launch_context (ApplicationLaunchContext): Context of launching + application. + + Returns: + bool: Is launch hook valid for the context by class attributes. + """ + if cls.platforms: + low_platforms = tuple( + _platform.lower() + for _platform in cls.platforms + ) + if platform.system().lower() not in low_platforms: + return False + + if cls.hosts: + if launch_context.host_name not in cls.hosts: + return False + + if cls.app_groups: + if launch_context.app_group.name not in cls.app_groups: + return False + + if cls.app_names: + if launch_context.app_name not in cls.app_names: + return False + + if cls.launch_types: + if launch_context.launch_type not in cls.launch_types: + return False + + return True + + @property + def data(self): + return self.launch_context.data + + @property + def application(self): + return getattr(self.launch_context, "application", None) + + @property + def manager(self): + return getattr(self.application, "manager", None) + + @property + def host_name(self): + return getattr(self.application, "host_name", None) + + @property + def app_group(self): + return getattr(self.application, "group", None) + + @property + def app_name(self): + return getattr(self.application, "full_name", None) + + @property + def addons_manager(self): + return getattr(self.launch_context, "addons_manager", None) + + @property + def modules_manager(self): + """ + Deprecated: + Use 'addons_wrapper' instead. + """ + return self.addons_manager + + def validate(self): + """Optional validation of launch hook on initialization. + + Returns: + bool: Hook is valid (True) or invalid (False). + """ + # QUESTION Not sure if this method has any usable potential. + # - maybe result can be based on settings + return True + + @abstractmethod + def execute(self, *args, **kwargs): + """Abstract execute method where logic of hook is.""" + pass + + +class PreLaunchHook(LaunchHook): + """Abstract class of prelaunch hook. + + This launch hook will be processed before application is launched. + + If any exception will happen during processing the application won't be + launched. + """ + + +class PostLaunchHook(LaunchHook): + """Abstract class of postlaunch hook. + + This launch hook will be processed after application is launched. + + Nothing will happen if any exception will happen during processing. And + processing of other postlaunch hooks won't stop either. + """ diff --git a/client/ayon_core/addons/ayon_applications/manager.py b/client/ayon_core/addons/ayon_applications/manager.py new file mode 100644 index 0000000000..6b1474b6ab --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/manager.py @@ -0,0 +1,682 @@ +import os +import sys +import copy +import json +import tempfile +import platform +import inspect +import subprocess + +import six + +from ayon_core import AYON_CORE_ROOT +from ayon_core.settings import get_studio_settings +from ayon_core.lib import ( + Logger, + modules_from_path, + classes_from_module, + get_linux_launcher_args, +) +from ayon_core.addon import AddonsManager + +from .constants import DEFAULT_ENV_SUBGROUP +from .exceptions import ( + ApplicationNotFound, + ApplictionExecutableNotFound, +) +from .hooks import PostLaunchHook, PreLaunchHook +from .defs import EnvironmentToolGroup, ApplicationGroup, LaunchTypes + +_logger = None + +CUSTOM_LAUNCH_APP_GROUPS = { + "djvview" +} + + +class ApplicationManager: + """Load applications and tools and store them by their full name. + + Args: + studio_settings (dict): Preloaded studio settings. When passed manager + will always use these values. Gives ability to create manager + using different settings. + """ + + def __init__(self, studio_settings=None): + self.log = Logger.get_logger(self.__class__.__name__) + + self.app_groups = {} + self.applications = {} + self.tool_groups = {} + self.tools = {} + + self._studio_settings = studio_settings + + self.refresh() + + def set_studio_settings(self, studio_settings): + """Ability to change init system settings. + + This will trigger refresh of manager. + """ + self._studio_settings = studio_settings + + self.refresh() + + def refresh(self): + """Refresh applications from settings.""" + self.app_groups.clear() + self.applications.clear() + self.tool_groups.clear() + self.tools.clear() + + if self._studio_settings is not None: + settings = copy.deepcopy(self._studio_settings) + else: + settings = get_studio_settings( + clear_metadata=False, exclude_locals=False + ) + + applications_addon_settings = settings["applications"] + + # Prepare known applications + app_defs = applications_addon_settings["applications"] + additional_apps = app_defs.pop("additional_apps") + for additional_app in additional_apps: + app_name = additional_app.pop("name") + if app_name in app_defs: + self.log.warning(( + "Additional application '{}' is already" + " in built-in applications." + ).format(app_name)) + app_defs[app_name] = additional_app + + for group_name, variant_defs in app_defs.items(): + group = ApplicationGroup(group_name, variant_defs, self) + self.app_groups[group_name] = group + for app in group: + self.applications[app.full_name] = app + + tools_definitions = applications_addon_settings["tool_groups"] + for tool_group_data in tools_definitions: + group = EnvironmentToolGroup(tool_group_data, self) + self.tool_groups[group.name] = group + for tool in group: + self.tools[tool.full_name] = tool + + def find_latest_available_variant_for_group(self, group_name): + group = self.app_groups.get(group_name) + if group is None or not group.enabled: + return None + + output = None + for _, variant in reversed(sorted(group.variants.items())): + executable = variant.find_executable() + if executable: + output = variant + break + return output + + def create_launch_context(self, app_name, **data): + """Prepare launch context for application. + + Args: + app_name (str): Name of application that should be launched. + **data (Any): Any additional data. Data may be used during + + Returns: + ApplicationLaunchContext: Launch context for application. + + Raises: + ApplicationNotFound: Application was not found by entered name. + """ + + app = self.applications.get(app_name) + if not app: + raise ApplicationNotFound(app_name) + + executable = app.find_executable() + + return ApplicationLaunchContext( + app, executable, **data + ) + + def launch_with_context(self, launch_context): + """Launch application using existing launch context. + + Args: + launch_context (ApplicationLaunchContext): Prepared launch + context. + """ + + if not launch_context.executable: + raise ApplictionExecutableNotFound(launch_context.application) + return launch_context.launch() + + def launch(self, app_name, **data): + """Launch procedure. + + For host application it's expected to contain "project_name", + "folder_path" and "task_name". + + Args: + app_name (str): Name of application that should be launched. + **data (dict): Any additional data. Data may be used during + preparation to store objects usable in multiple places. + + Raises: + ApplicationNotFound: Application was not found by entered + argument `app_name`. + ApplictionExecutableNotFound: Executables in application definition + were not found on this machine. + ApplicationLaunchFailed: Something important for application launch + failed. Exception should contain explanation message, + traceback should not be needed. + """ + + context = self.create_launch_context(app_name, **data) + return self.launch_with_context(context) + + +class ApplicationLaunchContext: + """Context of launching application. + + Main purpose of context is to prepare launch arguments and keyword + arguments for new process. Most important part of keyword arguments + preparations are environment variables. + + During the whole process is possible to use `data` attribute to store + object usable in multiple places. + + Launch arguments are strings in list. It is possible to "chain" argument + when order of them matters. That is possible to do with adding list where + order is right and should not change. + NOTE: This is recommendation, not requirement. + e.g.: `["nuke.exe", "--NukeX"]` -> In this case any part of process may + insert argument between `nuke.exe` and `--NukeX`. To keep them together + it is better to wrap them in another list: `[["nuke.exe", "--NukeX"]]`. + + Notes: + It is possible to use launch context only to prepare environment + variables. In that case `executable` may be None and can be used + 'run_prelaunch_hooks' method to run prelaunch hooks which prepare + them. + + Args: + application (Application): Application definition. + executable (ApplicationExecutable): Object with path to executable. + env_group (Optional[str]): Environment variable group. If not set + 'DEFAULT_ENV_SUBGROUP' is used. + launch_type (Optional[str]): Launch type. If not set 'local' is used. + **data (dict): Any additional data. Data may be used during + preparation to store objects usable in multiple places. + """ + + def __init__( + self, + application, + executable, + env_group=None, + launch_type=None, + **data + ): + # Application object + self.application = application + + self.addons_manager = AddonsManager() + + # Logger + logger_name = "{}-{}".format(self.__class__.__name__, + self.application.full_name) + self.log = Logger.get_logger(logger_name) + + self.executable = executable + + if launch_type is None: + launch_type = LaunchTypes.local + self.launch_type = launch_type + + if env_group is None: + env_group = DEFAULT_ENV_SUBGROUP + + self.env_group = env_group + + self.data = dict(data) + + launch_args = [] + if executable is not None: + launch_args = executable.as_args() + # subprocess.Popen launch arguments (first argument in constructor) + self.launch_args = launch_args + self.launch_args.extend(application.arguments) + if self.data.get("app_args"): + self.launch_args.extend(self.data.pop("app_args")) + + # Handle launch environemtns + src_env = self.data.pop("env", None) + if src_env is not None and not isinstance(src_env, dict): + self.log.warning(( + "Passed `env` kwarg has invalid type: {}. Expected: `dict`." + " Using `os.environ` instead." + ).format(str(type(src_env)))) + src_env = None + + if src_env is None: + src_env = os.environ + + ignored_env = {"QT_API", } + env = { + key: str(value) + for key, value in src_env.items() + if key not in ignored_env + } + # subprocess.Popen keyword arguments + self.kwargs = {"env": env} + + if platform.system().lower() == "windows": + # Detach new process from currently running process on Windows + flags = ( + subprocess.CREATE_NEW_PROCESS_GROUP + | subprocess.DETACHED_PROCESS + ) + self.kwargs["creationflags"] = flags + + if not sys.stdout: + self.kwargs["stdout"] = subprocess.DEVNULL + self.kwargs["stderr"] = subprocess.DEVNULL + + self.prelaunch_hooks = None + self.postlaunch_hooks = None + + self.process = None + self._prelaunch_hooks_executed = False + + @property + def env(self): + if ( + "env" not in self.kwargs + or self.kwargs["env"] is None + ): + self.kwargs["env"] = {} + return self.kwargs["env"] + + @env.setter + def env(self, value): + if not isinstance(value, dict): + raise ValueError( + "'env' attribute expect 'dict' object. Got: {}".format( + str(type(value)) + ) + ) + self.kwargs["env"] = value + + @property + def modules_manager(self): + """ + Deprecated: + Use 'addons_manager' instead. + + """ + return self.addons_manager + + def _collect_addons_launch_hook_paths(self): + """Helper to collect application launch hooks from addons. + + Module have to have implemented 'get_launch_hook_paths' method which + can expect application as argument or nothing. + + Returns: + List[str]: Paths to launch hook directories. + """ + + expected_types = (list, tuple, set) + + output = [] + for module in self.addons_manager.get_enabled_addons(): + # Skip module if does not have implemented 'get_launch_hook_paths' + func = getattr(module, "get_launch_hook_paths", None) + if func is None: + continue + + func = module.get_launch_hook_paths + if hasattr(inspect, "signature"): + sig = inspect.signature(func) + expect_args = len(sig.parameters) > 0 + else: + expect_args = len(inspect.getargspec(func)[0]) > 0 + + # Pass application argument if method expect it. + try: + if expect_args: + hook_paths = func(self.application) + else: + hook_paths = func() + except Exception: + self.log.warning( + "Failed to call 'get_launch_hook_paths'", + exc_info=True + ) + continue + + if not hook_paths: + continue + + # Convert string to list + if isinstance(hook_paths, six.string_types): + hook_paths = [hook_paths] + + # Skip invalid types + if not isinstance(hook_paths, expected_types): + self.log.warning(( + "Result of `get_launch_hook_paths`" + " has invalid type {}. Expected {}" + ).format(type(hook_paths), expected_types)) + continue + + output.extend(hook_paths) + return output + + def paths_to_launch_hooks(self): + """Directory paths where to look for launch hooks.""" + # This method has potential to be part of application manager (maybe). + paths = [] + + # TODO load additional studio paths from settings + global_hooks_dir = os.path.join(AYON_CORE_ROOT, "hooks") + + hooks_dirs = [ + global_hooks_dir + ] + if self.host_name: + # If host requires launch hooks and is module then launch hooks + # should be collected using 'collect_launch_hook_paths' + # - module have to implement 'get_launch_hook_paths' + host_module = self.addons_manager.get_host_addon(self.host_name) + if not host_module: + hooks_dirs.append(os.path.join( + AYON_CORE_ROOT, "hosts", self.host_name, "hooks" + )) + + for path in hooks_dirs: + if ( + os.path.exists(path) + and os.path.isdir(path) + and path not in paths + ): + paths.append(path) + + # Load modules paths + paths.extend(self._collect_addons_launch_hook_paths()) + + return paths + + def discover_launch_hooks(self, force=False): + """Load and prepare launch hooks.""" + if ( + self.prelaunch_hooks is not None + or self.postlaunch_hooks is not None + ): + if not force: + self.log.info("Launch hooks were already discovered.") + return + + self.prelaunch_hooks.clear() + self.postlaunch_hooks.clear() + + self.log.debug("Discovery of launch hooks started.") + + paths = self.paths_to_launch_hooks() + self.log.debug("Paths searched for launch hooks:\n{}".format( + "\n".join("- {}".format(path) for path in paths) + )) + + all_classes = { + "pre": [], + "post": [] + } + for path in paths: + if not os.path.exists(path): + self.log.info( + "Path to launch hooks does not exist: \"{}\"".format(path) + ) + continue + + modules, _crashed = modules_from_path(path) + for _filepath, module in modules: + all_classes["pre"].extend( + classes_from_module(PreLaunchHook, module) + ) + all_classes["post"].extend( + classes_from_module(PostLaunchHook, module) + ) + + for launch_type, classes in all_classes.items(): + hooks_with_order = [] + hooks_without_order = [] + for klass in classes: + try: + hook = klass(self) + if not hook.is_valid: + self.log.debug( + "Skipped hook invalid for current launch context: " + "{}".format(klass.__name__) + ) + continue + + if inspect.isabstract(hook): + self.log.debug("Skipped abstract hook: {}".format( + klass.__name__ + )) + continue + + # Separate hooks by pre/post class + if hook.order is None: + hooks_without_order.append(hook) + else: + hooks_with_order.append(hook) + + except Exception: + self.log.warning( + "Initialization of hook failed: " + "{}".format(klass.__name__), + exc_info=True + ) + + # Sort hooks with order by order + ordered_hooks = list(sorted( + hooks_with_order, key=lambda obj: obj.order + )) + # Extend ordered hooks with hooks without defined order + ordered_hooks.extend(hooks_without_order) + + if launch_type == "pre": + self.prelaunch_hooks = ordered_hooks + else: + self.postlaunch_hooks = ordered_hooks + + self.log.debug("Found {} prelaunch and {} postlaunch hooks.".format( + len(self.prelaunch_hooks), len(self.postlaunch_hooks) + )) + + @property + def app_name(self): + return self.application.name + + @property + def host_name(self): + return self.application.host_name + + @property + def app_group(self): + return self.application.group + + @property + def manager(self): + return self.application.manager + + def _run_process(self): + # Windows and MacOS have easier process start + low_platform = platform.system().lower() + if low_platform in ("windows", "darwin"): + return subprocess.Popen(self.launch_args, **self.kwargs) + + # Linux uses mid process + # - it is possible that the mid process executable is not + # available for this version of AYON in that case use standard + # launch + launch_args = get_linux_launcher_args() + if launch_args is None: + return subprocess.Popen(self.launch_args, **self.kwargs) + + # Prepare data that will be passed to midprocess + # - store arguments to a json and pass path to json as last argument + # - pass environments to set + app_env = self.kwargs.pop("env", {}) + json_data = { + "args": self.launch_args, + "env": app_env + } + if app_env: + # Filter environments of subprocess + self.kwargs["env"] = { + key: value + for key, value in os.environ.items() + if key in app_env + } + + # Create temp file + json_temp = tempfile.NamedTemporaryFile( + mode="w", prefix="op_app_args", suffix=".json", delete=False + ) + json_temp.close() + json_temp_filpath = json_temp.name + with open(json_temp_filpath, "w") as stream: + json.dump(json_data, stream) + + launch_args.append(json_temp_filpath) + + # Create mid-process which will launch application + process = subprocess.Popen(launch_args, **self.kwargs) + # Wait until the process finishes + # - This is important! The process would stay in "open" state. + process.wait() + # Remove the temp file + os.remove(json_temp_filpath) + # Return process which is already terminated + return process + + def run_prelaunch_hooks(self): + """Run prelaunch hooks. + + This method will be executed only once, any future calls will skip + the processing. + """ + + if self._prelaunch_hooks_executed: + self.log.warning("Prelaunch hooks were already executed.") + return + # Discover launch hooks + self.discover_launch_hooks() + + # Execute prelaunch hooks + for prelaunch_hook in self.prelaunch_hooks: + self.log.debug("Executing prelaunch hook: {}".format( + str(prelaunch_hook.__class__.__name__) + )) + prelaunch_hook.execute() + self._prelaunch_hooks_executed = True + + def launch(self): + """Collect data for new process and then create it. + + This method must not be executed more than once. + + Returns: + subprocess.Popen: Created process as Popen object. + """ + if self.process is not None: + self.log.warning("Application was already launched.") + return + + if not self._prelaunch_hooks_executed: + self.run_prelaunch_hooks() + + self.log.debug("All prelaunch hook executed. Starting new process.") + + # Prepare subprocess args + args_len_str = "" + if isinstance(self.launch_args, str): + args = self.launch_args + else: + args = self.clear_launch_args(self.launch_args) + args_len_str = " ({})".format(len(args)) + self.log.info( + "Launching \"{}\" with args{}: {}".format( + self.application.full_name, args_len_str, args + ) + ) + self.launch_args = args + + # Run process + self.process = self._run_process() + + # Process post launch hooks + for postlaunch_hook in self.postlaunch_hooks: + self.log.debug("Executing postlaunch hook: {}".format( + str(postlaunch_hook.__class__.__name__) + )) + + # TODO how to handle errors? + # - store to variable to let them accessible? + try: + postlaunch_hook.execute() + + except Exception: + self.log.warning( + "After launch procedures were not successful.", + exc_info=True + ) + + self.log.debug("Launch of {} finished.".format( + self.application.full_name + )) + + return self.process + + @staticmethod + def clear_launch_args(args): + """Collect launch arguments to final order. + + Launch argument should be list that may contain another lists this + function will upack inner lists and keep ordering. + + ``` + # source + [ [ arg1, [ arg2, arg3 ] ], arg4, [arg5, arg6]] + # result + [ arg1, arg2, arg3, arg4, arg5, arg6] + + Args: + args (list): Source arguments in list may contain inner lists. + + Return: + list: Unpacked arguments. + """ + if isinstance(args, str): + return args + all_cleared = False + while not all_cleared: + all_cleared = True + new_args = [] + for arg in args: + if isinstance(arg, (list, tuple, set)): + all_cleared = False + for _arg in arg: + new_args.append(_arg) + else: + new_args.append(arg) + args = new_args + + return args + diff --git a/client/ayon_core/addons/ayon_applications/utils.py b/client/ayon_core/addons/ayon_applications/utils.py new file mode 100644 index 0000000000..234fa6c683 --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/utils.py @@ -0,0 +1,609 @@ +import os +import copy +import json +import platform +import collections + +import six +import acre + +from ayon_core import AYON_CORE_ROOT +from ayon_core.settings import get_project_settings +from ayon_core.lib import Logger, get_ayon_username +from ayon_core.addon import AddonsManager +from ayon_core.pipeline import HOST_WORKFILE_EXTENSIONS +from ayon_core.pipeline.template_data import get_template_data +from ayon_core.pipeline.workfile import ( + get_workfile_template_key, + get_workdir_with_workdir_data, + get_last_workfile, + should_use_last_workfile_on_launch, + should_open_workfiles_tool_on_launch, +) + +from .constants import PLATFORM_NAMES, DEFAULT_ENV_SUBGROUP +from .exceptions import MissingRequiredKey, ApplicationLaunchFailed +from .manager import ApplicationManager + + +def parse_environments(env_data, env_group=None, platform_name=None): + """Parse environment values from settings byt group and platform. + + Data may contain up to 2 hierarchical levels of dictionaries. At the end + of the last level must be string or list. List is joined using platform + specific joiner (';' for windows and ':' for linux and mac). + + Hierarchical levels can contain keys for subgroups and platform name. + Platform specific values must be always last level of dictionary. Platform + names are "windows" (MS Windows), "linux" (any linux distribution) and + "darwin" (any MacOS distribution). + + Subgroups are helpers added mainly for standard and on farm usage. Farm + may require different environments for e.g. licence related values or + plugins. Default subgroup is "standard". + + Examples: + ``` + { + # Unchanged value + "ENV_KEY1": "value", + # Empty values are kept (unset environment variable) + "ENV_KEY2": "", + + # Join list values with ':' or ';' + "ENV_KEY3": ["value1", "value2"], + + # Environment groups + "ENV_KEY4": { + "standard": "DEMO_SERVER_URL", + "farm": "LICENCE_SERVER_URL" + }, + + # Platform specific (and only for windows and mac) + "ENV_KEY5": { + "windows": "windows value", + "darwin": ["value 1", "value 2"] + }, + + # Environment groups and platform combination + "ENV_KEY6": { + "farm": "FARM_VALUE", + "standard": { + "windows": ["value1", "value2"], + "linux": "value1", + "darwin": "" + } + } + } + ``` + """ + output = {} + if not env_data: + return output + + if not env_group: + env_group = DEFAULT_ENV_SUBGROUP + + if not platform_name: + platform_name = platform.system().lower() + + for key, value in env_data.items(): + if isinstance(value, dict): + # Look if any key is platform key + # - expect that represents environment group if does not contain + # platform keys + if not PLATFORM_NAMES.intersection(set(value.keys())): + # Skip the key if group is not available + if env_group not in value: + continue + value = value[env_group] + + # Check again if value is dictionary + # - this time there should be only platform keys + if isinstance(value, dict): + value = value.get(platform_name) + + # Check if value is list and join it's values + # QUESTION Should empty values be skipped? + if isinstance(value, (list, tuple)): + value = os.pathsep.join(value) + + # Set key to output if value is string + if isinstance(value, six.string_types): + output[key] = value + return output + + +class EnvironmentPrepData(dict): + """Helper dictionary for storin temp data during environment prep. + + Args: + data (dict): Data must contain required keys. + """ + required_keys = ( + "project_entity", "folder_entity", "task_entity", "app", "anatomy" + ) + + def __init__(self, data): + for key in self.required_keys: + if key not in data: + raise MissingRequiredKey(key) + + if not data.get("log"): + data["log"] = Logger.get_logger("EnvironmentPrepData") + + if data.get("env") is None: + data["env"] = os.environ.copy() + + project_name = data["project_entity"]["name"] + if "project_settings" not in data: + data["project_settings"] = get_project_settings(project_name) + + super(EnvironmentPrepData, self).__init__(data) + + +def get_app_environments_for_context( + project_name, + folder_path, + task_name, + app_name, + env_group=None, + launch_type=None, + env=None, + addons_manager=None +): + """Prepare environment variables by context. + Args: + project_name (str): Name of project. + folder_path (str): Folder path. + task_name (str): Name of task. + app_name (str): Name of application that is launched and can be found + by ApplicationManager. + env_group (Optional[str]): Name of environment group. If not passed + default group is used. + launch_type (Optional[str]): Type for which prelaunch hooks are + executed. + env (Optional[dict[str, str]]): Initial environment variables. + `os.environ` is used when not passed. + addons_manager (Optional[AddonsManager]): Initialized modules + manager. + + Returns: + dict: Environments for passed context and application. + """ + + # Prepare app object which can be obtained only from ApplicationManager + app_manager = ApplicationManager() + context = app_manager.create_launch_context( + app_name, + project_name=project_name, + folder_path=folder_path, + task_name=task_name, + env_group=env_group, + launch_type=launch_type, + env=env, + addons_manager=addons_manager, + modules_manager=addons_manager, + ) + context.run_prelaunch_hooks() + return context.env + + +def _merge_env(env, current_env): + """Modified function(merge) from acre module.""" + result = current_env.copy() + for key, value in env.items(): + # Keep missing keys by not filling `missing` kwarg + value = acre.lib.partial_format(value, data=current_env) + result[key] = value + return result + + +def _add_python_version_paths(app, env, logger, addons_manager): + """Add vendor packages specific for a Python version.""" + + for addon in addons_manager.get_enabled_addons(): + addon.modify_application_launch_arguments(app, env) + + # Skip adding if host name is not set + if not app.host_name: + return + + # Add Python 2/3 modules + python_vendor_dir = os.path.join( + AYON_CORE_ROOT, + "vendor", + "python" + ) + if app.use_python_2: + pythonpath = os.path.join(python_vendor_dir, "python_2") + else: + pythonpath = os.path.join(python_vendor_dir, "python_3") + + if not os.path.exists(pythonpath): + return + + logger.debug("Adding Python version specific paths to PYTHONPATH") + python_paths = [pythonpath] + + # Load PYTHONPATH from current launch context + python_path = env.get("PYTHONPATH") + if python_path: + python_paths.append(python_path) + + # Set new PYTHONPATH to launch context environments + env["PYTHONPATH"] = os.pathsep.join(python_paths) + + +def prepare_app_environments( + data, env_group=None, implementation_envs=True, addons_manager=None +): + """Modify launch environments based on launched app and context. + + Args: + data (EnvironmentPrepData): Dictionary where result and intermediate + result will be stored. + + """ + app = data["app"] + log = data["log"] + source_env = data["env"].copy() + + if addons_manager is None: + addons_manager = AddonsManager() + + _add_python_version_paths(app, source_env, log, addons_manager) + + # Use environments from local settings + filtered_local_envs = {} + # NOTE Overrides for environment variables are not implemented in AYON. + # project_settings = data["project_settings"] + # whitelist_envs = project_settings["general"].get("local_env_white_list") + # if whitelist_envs: + # local_settings = get_local_settings() + # local_envs = local_settings.get("environments") or {} + # filtered_local_envs = { + # key: value + # for key, value in local_envs.items() + # if key in whitelist_envs + # } + + # Apply local environment variables for already existing values + for key, value in filtered_local_envs.items(): + if key in source_env: + source_env[key] = value + + # `app_and_tool_labels` has debug purpose + app_and_tool_labels = [app.full_name] + # Environments for application + environments = [ + app.group.environment, + app.environment + ] + + folder_entity = data.get("folder_entity") + # Add tools environments + groups_by_name = {} + tool_by_group_name = collections.defaultdict(dict) + if folder_entity: + # Make sure each tool group can be added only once + for key in folder_entity["attrib"].get("tools") or []: + tool = app.manager.tools.get(key) + if not tool or not tool.is_valid_for_app(app): + continue + groups_by_name[tool.group.name] = tool.group + tool_by_group_name[tool.group.name][tool.name] = tool + + for group_name in sorted(groups_by_name.keys()): + group = groups_by_name[group_name] + environments.append(group.environment) + for tool_name in sorted(tool_by_group_name[group_name].keys()): + tool = tool_by_group_name[group_name][tool_name] + environments.append(tool.environment) + app_and_tool_labels.append(tool.full_name) + + log.debug( + "Will add environments for apps and tools: {}".format( + ", ".join(app_and_tool_labels) + ) + ) + + env_values = {} + for _env_values in environments: + if not _env_values: + continue + + # Choose right platform + tool_env = parse_environments(_env_values, env_group) + + # Apply local environment variables + # - must happen between all values because they may be used during + # merge + for key, value in filtered_local_envs.items(): + if key in tool_env: + tool_env[key] = value + + # Merge dictionaries + env_values = _merge_env(tool_env, env_values) + + merged_env = _merge_env(env_values, source_env) + + loaded_env = acre.compute(merged_env, cleanup=False) + + final_env = None + # Add host specific environments + if app.host_name and implementation_envs: + host_addon = addons_manager.get_host_addon(app.host_name) + add_implementation_envs = None + if host_addon: + add_implementation_envs = getattr( + host_addon, "add_implementation_envs", None + ) + if add_implementation_envs: + # Function may only modify passed dict without returning value + final_env = add_implementation_envs(loaded_env, app) + + if final_env is None: + final_env = loaded_env + + keys_to_remove = set(source_env.keys()) - set(final_env.keys()) + + # Update env + data["env"].update(final_env) + for key in keys_to_remove: + data["env"].pop(key, None) + + +def apply_project_environments_value( + project_name, env, project_settings=None, env_group=None +): + """Apply project specific environments on passed environments. + + The environments are applied on passed `env` argument value so it is not + required to apply changes back. + + Args: + project_name (str): Name of project for which environments should be + received. + env (dict): Environment values on which project specific environments + will be applied. + project_settings (dict): Project settings for passed project name. + Optional if project settings are already prepared. + + Returns: + dict: Passed env values with applied project environments. + + Raises: + KeyError: If project settings do not contain keys for project specific + environments. + + """ + if project_settings is None: + project_settings = get_project_settings(project_name) + + env_value = project_settings["core"]["project_environments"] + if env_value: + env_value = json.loads(env_value) + parsed_value = parse_environments(env_value, env_group) + env.update(acre.compute( + _merge_env(parsed_value, env), + cleanup=False + )) + return env + + +def prepare_context_environments(data, env_group=None, addons_manager=None): + """Modify launch environments with context data for launched host. + + Args: + data (EnvironmentPrepData): Dictionary where result and intermediate + result will be stored. + + """ + # Context environments + log = data["log"] + + project_entity = data["project_entity"] + folder_entity = data["folder_entity"] + task_entity = data["task_entity"] + if not project_entity: + log.info( + "Skipping context environments preparation." + " Launch context does not contain required data." + ) + return + + # Load project specific environments + project_name = project_entity["name"] + project_settings = get_project_settings(project_name) + data["project_settings"] = project_settings + + app = data["app"] + context_env = { + "AYON_PROJECT_NAME": project_entity["name"], + "AYON_APP_NAME": app.full_name + } + if folder_entity: + folder_path = folder_entity["path"] + context_env["AYON_FOLDER_PATH"] = folder_path + + if task_entity: + context_env["AYON_TASK_NAME"] = task_entity["name"] + + log.debug( + "Context environments set:\n{}".format( + json.dumps(context_env, indent=4) + ) + ) + data["env"].update(context_env) + + # Apply project specific environments on current env value + # - apply them once the context environments are set + apply_project_environments_value( + project_name, data["env"], project_settings, env_group + ) + + if not app.is_host: + return + + data["env"]["AYON_HOST_NAME"] = app.host_name + + if not folder_entity or not task_entity: + # QUESTION replace with log.info and skip workfile discovery? + # - technically it should be possible to launch host without context + raise ApplicationLaunchFailed( + "Host launch require folder and task context." + ) + + workdir_data = get_template_data( + project_entity, + folder_entity, + task_entity, + app.host_name, + project_settings + ) + data["workdir_data"] = workdir_data + + anatomy = data["anatomy"] + + task_type = workdir_data["task"]["type"] + # Temp solution how to pass task type to `_prepare_last_workfile` + data["task_type"] = task_type + + try: + workdir = get_workdir_with_workdir_data( + workdir_data, + anatomy.project_name, + anatomy, + project_settings=project_settings + ) + + except Exception as exc: + raise ApplicationLaunchFailed( + "Error in anatomy.format: {}".format(str(exc)) + ) + + if not os.path.exists(workdir): + log.debug( + "Creating workdir folder: \"{}\"".format(workdir) + ) + try: + os.makedirs(workdir) + except Exception as exc: + raise ApplicationLaunchFailed( + "Couldn't create workdir because: {}".format(str(exc)) + ) + + data["env"]["AYON_WORKDIR"] = workdir + + _prepare_last_workfile(data, workdir, addons_manager) + + +def _prepare_last_workfile(data, workdir, addons_manager): + """last workfile workflow preparation. + + Function check if should care about last workfile workflow and tries + to find the last workfile. Both information are stored to `data` and + environments. + + Last workfile is filled always (with version 1) even if any workfile + exists yet. + + Args: + data (EnvironmentPrepData): Dictionary where result and intermediate + result will be stored. + workdir (str): Path to folder where workfiles should be stored. + + """ + if not addons_manager: + addons_manager = AddonsManager() + + log = data["log"] + + _workdir_data = data.get("workdir_data") + if not _workdir_data: + log.info( + "Skipping last workfile preparation." + " Key `workdir_data` not filled." + ) + return + + app = data["app"] + workdir_data = copy.deepcopy(_workdir_data) + project_name = data["project_name"] + task_name = data["task_name"] + task_type = data["task_type"] + + start_last_workfile = data.get("start_last_workfile") + if start_last_workfile is None: + start_last_workfile = should_use_last_workfile_on_launch( + project_name, app.host_name, task_name, task_type + ) + else: + log.info("Opening of last workfile was disabled by user") + + data["start_last_workfile"] = start_last_workfile + + workfile_startup = should_open_workfiles_tool_on_launch( + project_name, app.host_name, task_name, task_type + ) + data["workfile_startup"] = workfile_startup + + # Store boolean as "0"(False) or "1"(True) + data["env"]["AVALON_OPEN_LAST_WORKFILE"] = ( + str(int(bool(start_last_workfile))) + ) + data["env"]["AYON_WORKFILE_TOOL_ON_START"] = ( + str(int(bool(workfile_startup))) + ) + + _sub_msg = "" if start_last_workfile else " not" + log.debug( + "Last workfile should{} be opened on start.".format(_sub_msg) + ) + + # Last workfile path + last_workfile_path = data.get("last_workfile_path") or "" + if not last_workfile_path: + host_addon = addons_manager.get_host_addon(app.host_name) + if host_addon: + extensions = host_addon.get_workfile_extensions() + else: + extensions = HOST_WORKFILE_EXTENSIONS.get(app.host_name) + + if extensions: + anatomy = data["anatomy"] + project_settings = data["project_settings"] + task_type = workdir_data["task"]["type"] + template_key = get_workfile_template_key( + project_name, + task_type, + app.host_name, + project_settings=project_settings + ) + # Find last workfile + file_template = anatomy.get_template_item( + "work", template_key, "file" + ).template + + workdir_data.update({ + "version": 1, + "user": get_ayon_username(), + "ext": extensions[0] + }) + + last_workfile_path = get_last_workfile( + workdir, file_template, workdir_data, extensions, True + ) + + if os.path.exists(last_workfile_path): + log.debug(( + "Workfiles for launch context does not exists" + " yet but path will be set." + )) + log.debug( + "Setting last workfile path: {}".format(last_workfile_path) + ) + + data["env"]["AYON_LAST_WORKFILE"] = last_workfile_path + data["last_workfile_path"] = last_workfile_path From afd0d33e0658eaac0ad9d322323da496227b5019 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 16:32:12 +0100 Subject: [PATCH 04/27] fix typo --- client/ayon_core/addons/ayon_applications/exceptions.py | 4 ++-- client/ayon_core/addons/ayon_applications/manager.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/ayon_core/addons/ayon_applications/exceptions.py b/client/ayon_core/addons/ayon_applications/exceptions.py index c24ba8c158..d5a48d3b6b 100644 --- a/client/ayon_core/addons/ayon_applications/exceptions.py +++ b/client/ayon_core/addons/ayon_applications/exceptions.py @@ -8,7 +8,7 @@ class ApplicationNotFound(Exception): ) -class ApplictionExecutableNotFound(Exception): +class ApplicationExecutableNotFound(Exception): """Defined executable paths are not available on the machine.""" def __init__(self, application): @@ -35,7 +35,7 @@ class ApplictionExecutableNotFound(Exception): # Is good idea to pass new line symbol to exception message? exc_mgs += "\n" + details self.exc_msg = exc_mgs - super(ApplictionExecutableNotFound, self).__init__(exc_mgs) + super(ApplicationExecutableNotFound, self).__init__(exc_mgs) class ApplicationLaunchFailed(Exception): diff --git a/client/ayon_core/addons/ayon_applications/manager.py b/client/ayon_core/addons/ayon_applications/manager.py index 6b1474b6ab..381a467fd4 100644 --- a/client/ayon_core/addons/ayon_applications/manager.py +++ b/client/ayon_core/addons/ayon_applications/manager.py @@ -22,7 +22,7 @@ from ayon_core.addon import AddonsManager from .constants import DEFAULT_ENV_SUBGROUP from .exceptions import ( ApplicationNotFound, - ApplictionExecutableNotFound, + ApplicationExecutableNotFound, ) from .hooks import PostLaunchHook, PreLaunchHook from .defs import EnvironmentToolGroup, ApplicationGroup, LaunchTypes @@ -151,7 +151,7 @@ class ApplicationManager: """ if not launch_context.executable: - raise ApplictionExecutableNotFound(launch_context.application) + raise ApplicationExecutableNotFound(launch_context.application) return launch_context.launch() def launch(self, app_name, **data): @@ -168,7 +168,7 @@ class ApplicationManager: Raises: ApplicationNotFound: Application was not found by entered argument `app_name`. - ApplictionExecutableNotFound: Executables in application definition + ApplicationExecutableNotFound: Executables in application definition were not found on this machine. ApplicationLaunchFailed: Something important for application launch failed. Exception should contain explanation message, From 52bbac3b75b89b76ea8a748fa503d1a6778cb55b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 16:39:53 +0100 Subject: [PATCH 05/27] added applications logic to addon init --- .../addons/ayon_applications/__init__.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/client/ayon_core/addons/ayon_applications/__init__.py b/client/ayon_core/addons/ayon_applications/__init__.py index 8ecffad159..87ad705cda 100644 --- a/client/ayon_core/addons/ayon_applications/__init__.py +++ b/client/ayon_core/addons/ayon_applications/__init__.py @@ -1,6 +1,57 @@ from .addon import ApplicationsAddon +from .constants import ( + DEFAULT_ENV_SUBGROUP, + PLATFORM_NAMES, +) +from .exceptions import ( + ApplicationNotFound, + ApplicationExecutableNotFound, + ApplicationLaunchFailed, + MissingRequiredKey, +) +from .defs import ( + LaunchTypes, + ApplicationExecutable, + UndefinedApplicationExecutable, + ApplicationGroup, + Application, + EnvironmentToolGroup, + EnvironmentTool, +) +from .hooks import ( + LaunchHook, + PreLaunchHook, + PostLaunchHook, +) +from .manager import ( + ApplicationManager, + ApplicationLaunchContext, +) __all__ = ( "ApplicationsAddon", + + "DEFAULT_ENV_SUBGROUP", + "PLATFORM_NAMES", + + "ApplicationNotFound", + "ApplicationExecutableNotFound", + "ApplicationLaunchFailed", + "MissingRequiredKey", + + "LaunchTypes", + "ApplicationExecutable", + "UndefinedApplicationExecutable", + "ApplicationGroup", + "Application", + "EnvironmentToolGroup", + "EnvironmentTool", + + "LaunchHook", + "PreLaunchHook", + "PostLaunchHook", + + "ApplicationManager", + "ApplicationLaunchContext", ) From d253d320baeecde460d0db3716b68ce6d31cb9f0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 16:46:43 +0100 Subject: [PATCH 06/27] pre and post launch hooks are using new import --- client/ayon_core/hooks/pre_add_last_workfile_arg.py | 2 +- client/ayon_core/hooks/pre_copy_template_workfile.py | 2 +- client/ayon_core/hooks/pre_create_extra_workdir_folders.py | 2 +- client/ayon_core/hooks/pre_global_host_data.py | 4 ++-- client/ayon_core/hooks/pre_mac_launch.py | 2 +- client/ayon_core/hooks/pre_new_console_apps.py | 2 +- client/ayon_core/hooks/pre_ocio_hook.py | 2 +- client/ayon_core/hosts/aftereffects/hooks/pre_launch_args.py | 5 +---- .../hosts/blender/hooks/pre_add_run_python_script_arg.py | 2 +- client/ayon_core/hosts/blender/hooks/pre_pyside_install.py | 2 +- client/ayon_core/hosts/blender/hooks/pre_windows_console.py | 2 +- .../ayon_core/hosts/celaction/hooks/pre_celaction_setup.py | 2 +- client/ayon_core/hosts/flame/hooks/pre_flame_setup.py | 2 +- .../ayon_core/hosts/fusion/hooks/pre_fusion_profile_hook.py | 2 +- client/ayon_core/hosts/fusion/hooks/pre_fusion_setup.py | 2 +- client/ayon_core/hosts/fusion/hooks/pre_pyside_install.py | 2 +- client/ayon_core/hosts/harmony/hooks/pre_launch_args.py | 5 +---- client/ayon_core/hosts/houdini/hooks/set_paths.py | 2 +- client/ayon_core/hosts/max/hooks/force_startup_script.py | 2 +- client/ayon_core/hosts/max/hooks/inject_python.py | 2 +- client/ayon_core/hosts/max/hooks/set_paths.py | 2 +- client/ayon_core/hosts/maya/hooks/pre_auto_load_plugins.py | 2 +- client/ayon_core/hosts/maya/hooks/pre_copy_mel.py | 2 +- .../maya/hooks/pre_open_workfile_post_initialization.py | 2 +- client/ayon_core/hosts/nuke/hooks/pre_nukeassist_setup.py | 2 +- client/ayon_core/hosts/photoshop/hooks/pre_launch_args.py | 5 +---- .../hosts/resolve/hooks/pre_resolve_last_workfile.py | 2 +- client/ayon_core/hosts/resolve/hooks/pre_resolve_setup.py | 2 +- client/ayon_core/hosts/resolve/hooks/pre_resolve_startup.py | 2 +- client/ayon_core/hosts/tvpaint/hooks/pre_launch_args.py | 2 +- .../ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py | 2 +- .../modules/timers_manager/launch_hooks/post_start_timer.py | 2 +- 32 files changed, 33 insertions(+), 42 deletions(-) diff --git a/client/ayon_core/hooks/pre_add_last_workfile_arg.py b/client/ayon_core/hooks/pre_add_last_workfile_arg.py index d11bb106d6..74964e0df9 100644 --- a/client/ayon_core/hooks/pre_add_last_workfile_arg.py +++ b/client/ayon_core/hooks/pre_add_last_workfile_arg.py @@ -1,6 +1,6 @@ import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class AddLastWorkfileToLaunchArgs(PreLaunchHook): diff --git a/client/ayon_core/hooks/pre_copy_template_workfile.py b/client/ayon_core/hooks/pre_copy_template_workfile.py index 096ad7dd7e..c884116578 100644 --- a/client/ayon_core/hooks/pre_copy_template_workfile.py +++ b/client/ayon_core/hooks/pre_copy_template_workfile.py @@ -1,7 +1,7 @@ import os import shutil from ayon_core.settings import get_project_settings -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.pipeline.workfile import ( get_custom_workfile_template, get_custom_workfile_template_by_string_context diff --git a/client/ayon_core/hooks/pre_create_extra_workdir_folders.py b/client/ayon_core/hooks/pre_create_extra_workdir_folders.py index 72c6bf2f68..8cbdaa338e 100644 --- a/client/ayon_core/hooks/pre_create_extra_workdir_folders.py +++ b/client/ayon_core/hooks/pre_create_extra_workdir_folders.py @@ -1,5 +1,5 @@ import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.pipeline.workfile import create_workdir_extra_folders diff --git a/client/ayon_core/hooks/pre_global_host_data.py b/client/ayon_core/hooks/pre_global_host_data.py index 27e66450ab..e93b512742 100644 --- a/client/ayon_core/hooks/pre_global_host_data.py +++ b/client/ayon_core/hooks/pre_global_host_data.py @@ -1,7 +1,7 @@ from ayon_api import get_project, get_folder_by_path, get_task_by_name -from ayon_core.lib.applications import ( - PreLaunchHook, +from ayon_applications import PreLaunchHook +from ayon_applications.utils import ( EnvironmentPrepData, prepare_app_environments, prepare_context_environments diff --git a/client/ayon_core/hooks/pre_mac_launch.py b/client/ayon_core/hooks/pre_mac_launch.py index 34680155f1..b234a20310 100644 --- a/client/ayon_core/hooks/pre_mac_launch.py +++ b/client/ayon_core/hooks/pre_mac_launch.py @@ -1,5 +1,5 @@ import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class LaunchWithTerminal(PreLaunchHook): diff --git a/client/ayon_core/hooks/pre_new_console_apps.py b/client/ayon_core/hooks/pre_new_console_apps.py index c81b924573..9777d37900 100644 --- a/client/ayon_core/hooks/pre_new_console_apps.py +++ b/client/ayon_core/hooks/pre_new_console_apps.py @@ -1,5 +1,5 @@ import subprocess -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class LaunchNewConsoleApps(PreLaunchHook): diff --git a/client/ayon_core/hooks/pre_ocio_hook.py b/client/ayon_core/hooks/pre_ocio_hook.py index e135a5bb12..0817afec71 100644 --- a/client/ayon_core/hooks/pre_ocio_hook.py +++ b/client/ayon_core/hooks/pre_ocio_hook.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook +from ayon_applications import PreLaunchHook from ayon_core.pipeline.colorspace import get_imageio_config from ayon_core.pipeline.template_data import get_template_data_with_names diff --git a/client/ayon_core/hosts/aftereffects/hooks/pre_launch_args.py b/client/ayon_core/hosts/aftereffects/hooks/pre_launch_args.py index 979d9ff3e5..a37481566e 100644 --- a/client/ayon_core/hosts/aftereffects/hooks/pre_launch_args.py +++ b/client/ayon_core/hosts/aftereffects/hooks/pre_launch_args.py @@ -6,10 +6,7 @@ from ayon_core.lib import ( get_ayon_launcher_args, is_using_ayon_console, ) -from ayon_core.lib.applications import ( - PreLaunchHook, - LaunchTypes, -) +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.aftereffects import get_launch_script_path diff --git a/client/ayon_core/hosts/blender/hooks/pre_add_run_python_script_arg.py b/client/ayon_core/hosts/blender/hooks/pre_add_run_python_script_arg.py index 00b297f998..9041ef7309 100644 --- a/client/ayon_core/hosts/blender/hooks/pre_add_run_python_script_arg.py +++ b/client/ayon_core/hosts/blender/hooks/pre_add_run_python_script_arg.py @@ -1,6 +1,6 @@ from pathlib import Path -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class AddPythonScriptToLaunchArgs(PreLaunchHook): diff --git a/client/ayon_core/hosts/blender/hooks/pre_pyside_install.py b/client/ayon_core/hosts/blender/hooks/pre_pyside_install.py index c80a1bd669..8f46eea0de 100644 --- a/client/ayon_core/hosts/blender/hooks/pre_pyside_install.py +++ b/client/ayon_core/hosts/blender/hooks/pre_pyside_install.py @@ -2,7 +2,7 @@ import os import re import subprocess from platform import system -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class InstallPySideToBlender(PreLaunchHook): diff --git a/client/ayon_core/hosts/blender/hooks/pre_windows_console.py b/client/ayon_core/hosts/blender/hooks/pre_windows_console.py index e3a8593cd9..47303a7af4 100644 --- a/client/ayon_core/hosts/blender/hooks/pre_windows_console.py +++ b/client/ayon_core/hosts/blender/hooks/pre_windows_console.py @@ -1,5 +1,5 @@ import subprocess -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class BlenderConsoleWindows(PreLaunchHook): diff --git a/client/ayon_core/hosts/celaction/hooks/pre_celaction_setup.py b/client/ayon_core/hosts/celaction/hooks/pre_celaction_setup.py index d94fff8f2b..8350c7b7c8 100644 --- a/client/ayon_core/hosts/celaction/hooks/pre_celaction_setup.py +++ b/client/ayon_core/hosts/celaction/hooks/pre_celaction_setup.py @@ -3,7 +3,7 @@ import shutil import winreg import subprocess from ayon_core.lib import get_ayon_launcher_args -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.celaction import CELACTION_ROOT_DIR diff --git a/client/ayon_core/hosts/flame/hooks/pre_flame_setup.py b/client/ayon_core/hosts/flame/hooks/pre_flame_setup.py index 1ff7ad7ccf..77a9435205 100644 --- a/client/ayon_core/hosts/flame/hooks/pre_flame_setup.py +++ b/client/ayon_core/hosts/flame/hooks/pre_flame_setup.py @@ -9,7 +9,7 @@ from ayon_core.lib import ( get_ayon_username, run_subprocess, ) -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts import flame as opflame diff --git a/client/ayon_core/hosts/fusion/hooks/pre_fusion_profile_hook.py b/client/ayon_core/hosts/fusion/hooks/pre_fusion_profile_hook.py index 10b1c9c45d..1064d0a83a 100644 --- a/client/ayon_core/hosts/fusion/hooks/pre_fusion_profile_hook.py +++ b/client/ayon_core/hosts/fusion/hooks/pre_fusion_profile_hook.py @@ -7,7 +7,7 @@ from ayon_core.hosts.fusion import ( FUSION_VERSIONS_DICT, get_fusion_version, ) -from ayon_core.lib.applications import ( +from ayon_applications import ( PreLaunchHook, LaunchTypes, ApplicationLaunchFailed, diff --git a/client/ayon_core/hosts/fusion/hooks/pre_fusion_setup.py b/client/ayon_core/hosts/fusion/hooks/pre_fusion_setup.py index 5e97ae3de1..ef084b0483 100644 --- a/client/ayon_core/hosts/fusion/hooks/pre_fusion_setup.py +++ b/client/ayon_core/hosts/fusion/hooks/pre_fusion_setup.py @@ -1,5 +1,5 @@ import os -from ayon_core.lib.applications import ( +from ayon_applications import ( PreLaunchHook, LaunchTypes, ApplicationLaunchFailed, diff --git a/client/ayon_core/hosts/fusion/hooks/pre_pyside_install.py b/client/ayon_core/hosts/fusion/hooks/pre_pyside_install.py index a9db39e24e..ab12078c43 100644 --- a/client/ayon_core/hosts/fusion/hooks/pre_pyside_install.py +++ b/client/ayon_core/hosts/fusion/hooks/pre_pyside_install.py @@ -3,7 +3,7 @@ import subprocess import platform import uuid -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class InstallPySideToFusion(PreLaunchHook): diff --git a/client/ayon_core/hosts/harmony/hooks/pre_launch_args.py b/client/ayon_core/hosts/harmony/hooks/pre_launch_args.py index bbad14084a..4d38cd09b3 100644 --- a/client/ayon_core/hosts/harmony/hooks/pre_launch_args.py +++ b/client/ayon_core/hosts/harmony/hooks/pre_launch_args.py @@ -6,10 +6,7 @@ from ayon_core.lib import ( get_ayon_launcher_args, is_using_ayon_console, ) -from ayon_core.lib.applications import ( - PreLaunchHook, - LaunchTypes, -) +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.harmony import get_launch_script_path diff --git a/client/ayon_core/hosts/houdini/hooks/set_paths.py b/client/ayon_core/hosts/houdini/hooks/set_paths.py index 7eb346cc74..4b89ebe944 100644 --- a/client/ayon_core/hosts/houdini/hooks/set_paths.py +++ b/client/ayon_core/hosts/houdini/hooks/set_paths.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class SetPath(PreLaunchHook): diff --git a/client/ayon_core/hosts/max/hooks/force_startup_script.py b/client/ayon_core/hosts/max/hooks/force_startup_script.py index 8ccd658e8f..417f0049ab 100644 --- a/client/ayon_core/hosts/max/hooks/force_startup_script.py +++ b/client/ayon_core/hosts/max/hooks/force_startup_script.py @@ -2,7 +2,7 @@ """Pre-launch to force 3ds max startup script.""" import os from ayon_core.hosts.max import MAX_HOST_DIR -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class ForceStartupScript(PreLaunchHook): diff --git a/client/ayon_core/hosts/max/hooks/inject_python.py b/client/ayon_core/hosts/max/hooks/inject_python.py index b1b36e75bd..fc9626ab87 100644 --- a/client/ayon_core/hosts/max/hooks/inject_python.py +++ b/client/ayon_core/hosts/max/hooks/inject_python.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Pre-launch hook to inject python environment.""" import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class InjectPythonPath(PreLaunchHook): diff --git a/client/ayon_core/hosts/max/hooks/set_paths.py b/client/ayon_core/hosts/max/hooks/set_paths.py index 0ee1b0dab7..f066de092e 100644 --- a/client/ayon_core/hosts/max/hooks/set_paths.py +++ b/client/ayon_core/hosts/max/hooks/set_paths.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class SetPath(PreLaunchHook): diff --git a/client/ayon_core/hosts/maya/hooks/pre_auto_load_plugins.py b/client/ayon_core/hosts/maya/hooks/pre_auto_load_plugins.py index ed294da125..45785ac354 100644 --- a/client/ayon_core/hosts/maya/hooks/pre_auto_load_plugins.py +++ b/client/ayon_core/hosts/maya/hooks/pre_auto_load_plugins.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class MayaPreAutoLoadPlugins(PreLaunchHook): diff --git a/client/ayon_core/hosts/maya/hooks/pre_copy_mel.py b/client/ayon_core/hosts/maya/hooks/pre_copy_mel.py index 3fd81ceff4..683b4c59c7 100644 --- a/client/ayon_core/hosts/maya/hooks/pre_copy_mel.py +++ b/client/ayon_core/hosts/maya/hooks/pre_copy_mel.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.maya.lib import create_workspace_mel diff --git a/client/ayon_core/hosts/maya/hooks/pre_open_workfile_post_initialization.py b/client/ayon_core/hosts/maya/hooks/pre_open_workfile_post_initialization.py index 6bf678474f..a54f17c6c6 100644 --- a/client/ayon_core/hosts/maya/hooks/pre_open_workfile_post_initialization.py +++ b/client/ayon_core/hosts/maya/hooks/pre_open_workfile_post_initialization.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class MayaPreOpenWorkfilePostInitialization(PreLaunchHook): diff --git a/client/ayon_core/hosts/nuke/hooks/pre_nukeassist_setup.py b/client/ayon_core/hosts/nuke/hooks/pre_nukeassist_setup.py index 2f6d121af5..afef3ba843 100644 --- a/client/ayon_core/hosts/nuke/hooks/pre_nukeassist_setup.py +++ b/client/ayon_core/hosts/nuke/hooks/pre_nukeassist_setup.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PreLaunchHook +from ayon_applications import PreLaunchHook class PrelaunchNukeAssistHook(PreLaunchHook): diff --git a/client/ayon_core/hosts/photoshop/hooks/pre_launch_args.py b/client/ayon_core/hosts/photoshop/hooks/pre_launch_args.py index 8358c11ca1..70f8fc730f 100644 --- a/client/ayon_core/hosts/photoshop/hooks/pre_launch_args.py +++ b/client/ayon_core/hosts/photoshop/hooks/pre_launch_args.py @@ -6,10 +6,7 @@ from ayon_core.lib import ( get_ayon_launcher_args, is_using_ayon_console, ) -from ayon_core.lib.applications import ( - PreLaunchHook, - LaunchTypes, -) +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.photoshop import get_launch_script_path diff --git a/client/ayon_core/hosts/resolve/hooks/pre_resolve_last_workfile.py b/client/ayon_core/hosts/resolve/hooks/pre_resolve_last_workfile.py index d82651289c..cf9953bfe9 100644 --- a/client/ayon_core/hosts/resolve/hooks/pre_resolve_last_workfile.py +++ b/client/ayon_core/hosts/resolve/hooks/pre_resolve_last_workfile.py @@ -1,5 +1,5 @@ import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class PreLaunchResolveLastWorkfile(PreLaunchHook): diff --git a/client/ayon_core/hosts/resolve/hooks/pre_resolve_setup.py b/client/ayon_core/hosts/resolve/hooks/pre_resolve_setup.py index c14fd75b2f..f45e28d5ab 100644 --- a/client/ayon_core/hosts/resolve/hooks/pre_resolve_setup.py +++ b/client/ayon_core/hosts/resolve/hooks/pre_resolve_setup.py @@ -1,7 +1,7 @@ import os from pathlib import Path import platform -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes from ayon_core.hosts.resolve.utils import setup diff --git a/client/ayon_core/hosts/resolve/hooks/pre_resolve_startup.py b/client/ayon_core/hosts/resolve/hooks/pre_resolve_startup.py index ab16053450..300564f7cc 100644 --- a/client/ayon_core/hosts/resolve/hooks/pre_resolve_startup.py +++ b/client/ayon_core/hosts/resolve/hooks/pre_resolve_startup.py @@ -1,6 +1,6 @@ import os -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes import ayon_core.hosts.resolve diff --git a/client/ayon_core/hosts/tvpaint/hooks/pre_launch_args.py b/client/ayon_core/hosts/tvpaint/hooks/pre_launch_args.py index 25e324c5cc..691b81e089 100644 --- a/client/ayon_core/hosts/tvpaint/hooks/pre_launch_args.py +++ b/client/ayon_core/hosts/tvpaint/hooks/pre_launch_args.py @@ -1,5 +1,5 @@ from ayon_core.lib import get_ayon_launcher_args -from ayon_core.lib.applications import PreLaunchHook, LaunchTypes +from ayon_applications import PreLaunchHook, LaunchTypes class TvpaintPrelaunchHook(PreLaunchHook): diff --git a/client/ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py b/client/ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py index 54ffba3a63..e38591f65d 100644 --- a/client/ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py +++ b/client/ayon_core/hosts/unreal/hooks/pre_workfile_preparation.py @@ -9,7 +9,7 @@ from pathlib import Path from qtpy import QtCore from ayon_core import resources -from ayon_core.lib.applications import ( +from ayon_applications import ( PreLaunchHook, ApplicationLaunchFailed, LaunchTypes, diff --git a/client/ayon_core/modules/timers_manager/launch_hooks/post_start_timer.py b/client/ayon_core/modules/timers_manager/launch_hooks/post_start_timer.py index da5d430939..b402d4034a 100644 --- a/client/ayon_core/modules/timers_manager/launch_hooks/post_start_timer.py +++ b/client/ayon_core/modules/timers_manager/launch_hooks/post_start_timer.py @@ -1,4 +1,4 @@ -from ayon_core.lib.applications import PostLaunchHook, LaunchTypes +from ayon_applications import PostLaunchHook, LaunchTypes class PostStartTimerHook(PostLaunchHook): From 5abda854566bb38d8a5fefb535cb198be2bc1c48 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:33:48 +0100 Subject: [PATCH 07/27] modified addons discovery to be able to detect addons in addons dir --- client/ayon_core/addon/base.py | 42 +++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/addon/base.py b/client/ayon_core/addon/base.py index 6bac25b8ac..da02691275 100644 --- a/client/ayon_core/addon/base.py +++ b/client/ayon_core/addon/base.py @@ -16,6 +16,7 @@ import six import appdirs import ayon_api +from ayon_core import AYON_CORE_ROOT from ayon_core.lib import Logger, is_dev_mode_enabled from ayon_core.settings import get_studio_settings @@ -335,14 +336,49 @@ def _load_ayon_addons(openpype_modules, modules_key, log): return addons_to_skip_in_core +def _load_ayon_addons_dir(openpype_modules, modules_key, log): + addons_dir = os.path.join(AYON_CORE_ROOT, "addons") + if not os.path.exists(addons_dir): + return + + while addons_dir in sys.path: + sys.path.remove(addons_dir) + sys.path.insert(0, addons_dir) + + imported_modules = [] + for name in os.listdir(addons_dir): + fullpath = os.path.join(addons_dir, name) + basename = os.path.splitext(name)[0] + try: + module = __import__(basename, fromlist=("",)) + for attr_name in dir(module): + attr = getattr(module, attr_name) + if ( + inspect.isclass(attr) + and issubclass(attr, AYONAddon) + ): + new_import_str = "{}.{}".format(modules_key, basename) + sys.modules[new_import_str] = module + setattr(openpype_modules, basename, module) + imported_modules.append(module) + break + + except Exception: + log.error( + "Failed to import addon '{}'.".format(fullpath), + exc_info=True + ) + return imported_modules + + def _load_addons_in_core( ignore_addon_names, openpype_modules, modules_key, log ): + _load_ayon_addons_dir(openpype_modules, modules_key, log) # Add current directory at first place # - has small differences in import logic - current_dir = os.path.abspath(os.path.dirname(__file__)) - hosts_dir = os.path.join(os.path.dirname(current_dir), "hosts") - modules_dir = os.path.join(os.path.dirname(current_dir), "modules") + hosts_dir = os.path.join(AYON_CORE_ROOT, "hosts") + modules_dir = os.path.join(AYON_CORE_ROOT, "modules") ignored_host_names = set(IGNORED_HOSTS_IN_AYON) ignored_module_dir_filenames = ( From 597915e7ea7fed4a609ebbd2e7bebd8a943cb1d2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:33:56 +0100 Subject: [PATCH 08/27] removed unused variables --- client/ayon_core/addons/ayon_applications/manager.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/client/ayon_core/addons/ayon_applications/manager.py b/client/ayon_core/addons/ayon_applications/manager.py index 381a467fd4..dca2ff4491 100644 --- a/client/ayon_core/addons/ayon_applications/manager.py +++ b/client/ayon_core/addons/ayon_applications/manager.py @@ -27,12 +27,6 @@ from .exceptions import ( from .hooks import PostLaunchHook, PreLaunchHook from .defs import EnvironmentToolGroup, ApplicationGroup, LaunchTypes -_logger = None - -CUSTOM_LAUNCH_APP_GROUPS = { - "djvview" -} - class ApplicationManager: """Load applications and tools and store them by their full name. From d6c3540430a2baefe7b8477c2c3e4c6c31147ca1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:34:06 +0100 Subject: [PATCH 09/27] implemeneted addon --- .../addons/ayon_applications/addon.py | 158 +++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/addons/ayon_applications/addon.py b/client/ayon_core/addons/ayon_applications/addon.py index 149f547861..9c4a5a392e 100644 --- a/client/ayon_core/addons/ayon_applications/addon.py +++ b/client/ayon_core/addons/ayon_applications/addon.py @@ -1,5 +1,161 @@ -from ayon_core.addon import AYONAddon +import os +import json + +from ayon_core.addon import AYONAddon, click_wrap + +from .defs import LaunchTypes +from .manager import ApplicationManager class ApplicationsAddon(AYONAddon): name = "applications" + + def get_app_environments_for_context( + self, + project_name, + folder_path, + task_name, + full_app_name, + env_group=None, + launch_type=None, + env=None, + ): + """Calculate environment variables for launch context. + + Args: + project_name (str): Project name. + folder_path (str): Folder path. + task_name (str): Task name. + full_app_name (str): Full application name. + env_group (Optional[str]): Environment group. + launch_type (Optional[str]): Launch type. + env (Optional[dict[str, str]]): Environment variables to update. + + Returns: + dict[str, str]: Environment variables for context. + + """ + from ayon_applications.utils import get_app_environments_for_context + + if not full_app_name: + return {} + + return get_app_environments_for_context( + project_name, + folder_path, + task_name, + full_app_name, + env_group=env_group, + launch_type=launch_type, + env=env, + addons_manager=self.manager + ) + + def get_farm_publish_environment_variables( + self, + project_name, + folder_path, + task_name, + full_app_name=None, + env_group=None, + ): + """Calculate environment variables for farm publish. + + Args: + project_name (str): Project name. + folder_path (str): Folder path. + task_name (str): Task name. + env_group (Optional[str]): Environment group. + full_app_name (Optional[str]): Full application name. Value from + environment variable 'AYON_APP_NAME' is used if 'None' is + passed. + + Returns: + dict[str, str]: Environment variables for farm publish. + + """ + if full_app_name is None: + full_app_name = os.getenv("AYON_APP_NAME") + + return self.get_app_environments_for_context( + project_name, + folder_path, + task_name, + full_app_name, + env_group=env_group, + launch_type=LaunchTypes.farm_publish + ) + + def get_applications_manager(self, settings=None): + """Get applications manager. + + Args: + settings (Optional[dict]): Studio/project settings. + + Returns: + ApplicationManager: Applications manager. + + """ + return ApplicationManager(settings) + + # --- CLI --- + def cli(self, addon_click_group): + main_group = click_wrap.group( + self._cli_main, name=self.name, help="Applications addon" + ) + ( + main_group.command( + self._cli_extract_environments, + name="extractenvironments", + help=( + "Extract environment variables for context into json file" + ) + ) + .argument("output_json_path") + .option("--project", help="Project name", default=None) + .option("--folder", help="Folder path", default=None) + .option("--task", help="Task name", default=None) + .option("--app", help="Application name", default=None) + .option( + "--envgroup", + help="Environment group (e.g. \"farm\")", + default=None + ) + ) + # Convert main command to click object and add it to parent group + addon_click_group.add_command( + main_group.to_click_obj() + ) + + def _cli_main(self): + pass + + def _cli_extract_environments( + self, output_json_path, project, folder, task, app, envgroup + ): + """Produces json file with environment based on project and app. + + Called by farm integration to propagate environment into farm jobs. + + Args: + output_json_path (str): Output json file path. + project (str): Project name. + folder (str): Folder path. + task (str): Task name. + app (str): Full application name e.g. 'maya/2024'. + envgroup (str): Environment group. + + """ + if all((project, folder, task, app)): + env = self.get_farm_publish_environment_variables( + project, folder, task, app, env_group=envgroup, + ) + else: + env = os.environ.copy() + + output_dir = os.path.dirname(output_json_path) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + with open(output_json_path, "w") as file_stream: + json.dump(env, file_stream, indent=4) From 76398753ecd36393189267644534f4dcc35ecb96 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:34:17 +0100 Subject: [PATCH 10/27] removed applications logic from ayon_core.lib --- client/ayon_core/lib/__init__.py | 28 - client/ayon_core/lib/applications.py | 1893 -------------------------- 2 files changed, 1921 deletions(-) delete mode 100644 client/ayon_core/lib/applications.py diff --git a/client/ayon_core/lib/__init__.py b/client/ayon_core/lib/__init__.py index 2ee7eecfe3..408262ca42 100644 --- a/client/ayon_core/lib/__init__.py +++ b/client/ayon_core/lib/__init__.py @@ -120,22 +120,6 @@ from .transcoding import ( get_rescaled_command_arguments, ) -from .applications import ( - ApplicationLaunchFailed, - ApplictionExecutableNotFound, - ApplicationNotFound, - ApplicationManager, - - PreLaunchHook, - PostLaunchHook, - - EnvironmentPrepData, - prepare_app_environments, - prepare_context_environments, - get_app_environments_for_context, - apply_project_environments_value -) - from .plugin_tools import ( prepare_template_data, source_hash, @@ -231,18 +215,6 @@ __all__ = [ "convert_ffprobe_fps_to_float", "get_rescaled_command_arguments", - "ApplicationLaunchFailed", - "ApplictionExecutableNotFound", - "ApplicationNotFound", - "ApplicationManager", - "PreLaunchHook", - "PostLaunchHook", - "EnvironmentPrepData", - "prepare_app_environments", - "prepare_context_environments", - "get_app_environments_for_context", - "apply_project_environments_value", - "compile_list_of_regexes", "filter_profiles", diff --git a/client/ayon_core/lib/applications.py b/client/ayon_core/lib/applications.py deleted file mode 100644 index 2db32cbfaa..0000000000 --- a/client/ayon_core/lib/applications.py +++ /dev/null @@ -1,1893 +0,0 @@ -import os -import sys -import copy -import json -import tempfile -import platform -import collections -import inspect -import subprocess -from abc import ABCMeta, abstractmethod - -import six - -from ayon_core import AYON_CORE_ROOT -from ayon_core.settings import get_project_settings, get_studio_settings -from .log import Logger -from .profiles_filtering import filter_profiles -from .local_settings import get_ayon_username - -from .python_module_tools import ( - modules_from_path, - classes_from_module -) -from .execute import ( - find_executable, - get_linux_launcher_args -) - -_logger = None - -PLATFORM_NAMES = {"windows", "linux", "darwin"} -DEFAULT_ENV_SUBGROUP = "standard" -CUSTOM_LAUNCH_APP_GROUPS = { - "djvview" -} - - -class LaunchTypes: - """Launch types are filters for pre/post-launch hooks. - - Please use these variables in case they'll change values. - """ - - # Local launch - application is launched on local machine - local = "local" - # Farm render job - application is on farm - farm_render = "farm-render" - # Farm publish job - integration post-render job - farm_publish = "farm-publish" - # Remote launch - application is launched on remote machine from which - # can be started publishing - remote = "remote" - # Automated launch - application is launched with automated publishing - automated = "automated" - - -def parse_environments(env_data, env_group=None, platform_name=None): - """Parse environment values from settings byt group and platform. - - Data may contain up to 2 hierarchical levels of dictionaries. At the end - of the last level must be string or list. List is joined using platform - specific joiner (';' for windows and ':' for linux and mac). - - Hierarchical levels can contain keys for subgroups and platform name. - Platform specific values must be always last level of dictionary. Platform - names are "windows" (MS Windows), "linux" (any linux distribution) and - "darwin" (any MacOS distribution). - - Subgroups are helpers added mainly for standard and on farm usage. Farm - may require different environments for e.g. licence related values or - plugins. Default subgroup is "standard". - - Examples: - ``` - { - # Unchanged value - "ENV_KEY1": "value", - # Empty values are kept (unset environment variable) - "ENV_KEY2": "", - - # Join list values with ':' or ';' - "ENV_KEY3": ["value1", "value2"], - - # Environment groups - "ENV_KEY4": { - "standard": "DEMO_SERVER_URL", - "farm": "LICENCE_SERVER_URL" - }, - - # Platform specific (and only for windows and mac) - "ENV_KEY5": { - "windows": "windows value", - "darwin": ["value 1", "value 2"] - }, - - # Environment groups and platform combination - "ENV_KEY6": { - "farm": "FARM_VALUE", - "standard": { - "windows": ["value1", "value2"], - "linux": "value1", - "darwin": "" - } - } - } - ``` - """ - output = {} - if not env_data: - return output - - if not env_group: - env_group = DEFAULT_ENV_SUBGROUP - - if not platform_name: - platform_name = platform.system().lower() - - for key, value in env_data.items(): - if isinstance(value, dict): - # Look if any key is platform key - # - expect that represents environment group if does not contain - # platform keys - if not PLATFORM_NAMES.intersection(set(value.keys())): - # Skip the key if group is not available - if env_group not in value: - continue - value = value[env_group] - - # Check again if value is dictionary - # - this time there should be only platform keys - if isinstance(value, dict): - value = value.get(platform_name) - - # Check if value is list and join it's values - # QUESTION Should empty values be skipped? - if isinstance(value, (list, tuple)): - value = os.pathsep.join(value) - - # Set key to output if value is string - if isinstance(value, six.string_types): - output[key] = value - return output - - -def get_logger(): - """Global lib.applications logger getter.""" - global _logger - if _logger is None: - _logger = Logger.get_logger(__name__) - return _logger - - -class ApplicationNotFound(Exception): - """Application was not found in ApplicationManager by name.""" - - def __init__(self, app_name): - self.app_name = app_name - super(ApplicationNotFound, self).__init__( - "Application \"{}\" was not found.".format(app_name) - ) - - -class ApplictionExecutableNotFound(Exception): - """Defined executable paths are not available on the machine.""" - - def __init__(self, application): - self.application = application - details = None - if not application.executables: - msg = ( - "Executable paths for application \"{}\"({}) are not set." - ) - else: - msg = ( - "Defined executable paths for application \"{}\"({})" - " are not available on this machine." - ) - details = "Defined paths:" - for executable in application.executables: - details += "\n- " + executable.executable_path - - self.msg = msg.format(application.full_label, application.full_name) - self.details = details - - exc_mgs = str(self.msg) - if details: - # Is good idea to pass new line symbol to exception message? - exc_mgs += "\n" + details - self.exc_msg = exc_mgs - super(ApplictionExecutableNotFound, self).__init__(exc_mgs) - - -class ApplicationLaunchFailed(Exception): - """Application launch failed due to known reason. - - Message should be self explanatory as traceback won't be shown. - """ - pass - - -class ApplicationGroup: - """Hold information about application group. - - Application group wraps different versions(variants) of application. - e.g. "maya" is group and "maya_2020" is variant. - - Group hold `host_name` which is implementation name used in AYON. Also - holds `enabled` if whole app group is enabled or `icon` for application - icon path in resources. - - Group has also `environment` which hold same environments for all variants. - - Args: - name (str): Groups' name. - data (dict): Group defying data loaded from settings. - manager (ApplicationManager): Manager that created the group. - """ - - def __init__(self, name, data, manager): - self.name = name - self.manager = manager - self._data = data - - self.enabled = data["enabled"] - self.label = data["label"] or None - self.icon = data["icon"] or None - env = {} - try: - env = json.loads(data["environment"]) - except Exception: - pass - self._environment = env - - host_name = data["host_name"] or None - self.is_host = host_name is not None - self.host_name = host_name - - settings_variants = data["variants"] - variants = {} - for variant_data in settings_variants: - app_variant = Application(variant_data, self) - variants[app_variant.name] = app_variant - - self.variants = variants - - def __repr__(self): - return "<{}> - {}".format(self.__class__.__name__, self.name) - - def __iter__(self): - for variant in self.variants.values(): - yield variant - - @property - def environment(self): - return copy.deepcopy(self._environment) - - -class Application: - """Hold information about application. - - Object by itself does nothing special. - - Args: - data (dict): Data for the version containing information about - executables, variant label or if is enabled. - Only required key is `executables`. - group (ApplicationGroup): App group object that created the application - and under which application belongs. - - """ - def __init__(self, data, group): - self._data = data - name = data["name"] - label = data["label"] or name - enabled = False - if group.enabled: - enabled = data.get("enabled", True) - - if group.label: - full_label = " ".join((group.label, label)) - else: - full_label = label - env = {} - try: - env = json.loads(data["environment"]) - except Exception: - pass - - arguments = data["arguments"] - if isinstance(arguments, dict): - arguments = arguments.get(platform.system().lower()) - - if not arguments: - arguments = [] - - _executables = data["executables"].get(platform.system().lower(), []) - executables = [ - ApplicationExecutable(executable) - for executable in _executables - ] - - self.group = group - - self.name = name - self.label = label - self.enabled = enabled - self.use_python_2 = data.get("use_python_2", False) - - self.full_name = "/".join((group.name, name)) - self.full_label = full_label - self.arguments = arguments - self.executables = executables - self._environment = env - - def __repr__(self): - return "<{}> - {}".format(self.__class__.__name__, self.full_name) - - @property - def environment(self): - return copy.deepcopy(self._environment) - - @property - def manager(self): - return self.group.manager - - @property - def host_name(self): - return self.group.host_name - - @property - def icon(self): - return self.group.icon - - @property - def is_host(self): - return self.group.is_host - - def find_executable(self): - """Try to find existing executable for application. - - Returns (str): Path to executable from `executables` or None if any - exists. - """ - for executable in self.executables: - if executable.exists(): - return executable - return None - - def launch(self, *args, **kwargs): - """Launch the application. - - For this purpose is used manager's launch method to keep logic at one - place. - - Arguments must match with manager's launch method. That's why *args - **kwargs are used. - - Returns: - subprocess.Popen: Return executed process as Popen object. - """ - return self.manager.launch(self.full_name, *args, **kwargs) - - -class ApplicationManager: - """Load applications and tools and store them by their full name. - - Args: - studio_settings (dict): Preloaded studio settings. When passed manager - will always use these values. Gives ability to create manager - using different settings. - """ - - def __init__(self, studio_settings=None): - self.log = Logger.get_logger(self.__class__.__name__) - - self.app_groups = {} - self.applications = {} - self.tool_groups = {} - self.tools = {} - - self._studio_settings = studio_settings - - self.refresh() - - def set_studio_settings(self, studio_settings): - """Ability to change init system settings. - - This will trigger refresh of manager. - """ - self._studio_settings = studio_settings - - self.refresh() - - def refresh(self): - """Refresh applications from settings.""" - self.app_groups.clear() - self.applications.clear() - self.tool_groups.clear() - self.tools.clear() - - if self._studio_settings is not None: - settings = copy.deepcopy(self._studio_settings) - else: - settings = get_studio_settings( - clear_metadata=False, exclude_locals=False - ) - - applications_addon_settings = settings["applications"] - - # Prepare known applications - app_defs = applications_addon_settings["applications"] - additional_apps = app_defs.pop("additional_apps") - for additional_app in additional_apps: - app_name = additional_app.pop("name") - if app_name in app_defs: - self.log.warning(( - "Additional application '{}' is already" - " in built-in applications." - ).format(app_name)) - app_defs[app_name] = additional_app - - for group_name, variant_defs in app_defs.items(): - group = ApplicationGroup(group_name, variant_defs, self) - self.app_groups[group_name] = group - for app in group: - self.applications[app.full_name] = app - - tools_definitions = applications_addon_settings["tool_groups"] - for tool_group_data in tools_definitions: - group = EnvironmentToolGroup(tool_group_data, self) - self.tool_groups[group.name] = group - for tool in group: - self.tools[tool.full_name] = tool - - def find_latest_available_variant_for_group(self, group_name): - group = self.app_groups.get(group_name) - if group is None or not group.enabled: - return None - - output = None - for _, variant in reversed(sorted(group.variants.items())): - executable = variant.find_executable() - if executable: - output = variant - break - return output - - def create_launch_context(self, app_name, **data): - """Prepare launch context for application. - - Args: - app_name (str): Name of application that should be launched. - **data (Any): Any additional data. Data may be used during - - Returns: - ApplicationLaunchContext: Launch context for application. - - Raises: - ApplicationNotFound: Application was not found by entered name. - """ - - app = self.applications.get(app_name) - if not app: - raise ApplicationNotFound(app_name) - - executable = app.find_executable() - - return ApplicationLaunchContext( - app, executable, **data - ) - - def launch_with_context(self, launch_context): - """Launch application using existing launch context. - - Args: - launch_context (ApplicationLaunchContext): Prepared launch - context. - """ - - if not launch_context.executable: - raise ApplictionExecutableNotFound(launch_context.application) - return launch_context.launch() - - def launch(self, app_name, **data): - """Launch procedure. - - For host application it's expected to contain "project_name", - "folder_path" and "task_name". - - Args: - app_name (str): Name of application that should be launched. - **data (dict): Any additional data. Data may be used during - preparation to store objects usable in multiple places. - - Raises: - ApplicationNotFound: Application was not found by entered - argument `app_name`. - ApplictionExecutableNotFound: Executables in application definition - were not found on this machine. - ApplicationLaunchFailed: Something important for application launch - failed. Exception should contain explanation message, - traceback should not be needed. - """ - - context = self.create_launch_context(app_name, **data) - return self.launch_with_context(context) - - - -class EnvironmentToolGroup: - """Hold information about environment tool group. - - Environment tool group may hold different variants of same tool and set - environments that are same for all of them. - - e.g. "mtoa" may have different versions but all environments except one - are same. - - Args: - data (dict): Group information with variants. - manager (ApplicationManager): Manager that creates the group. - """ - - def __init__(self, data, manager): - name = data["name"] - label = data["label"] - - self.name = name - self.label = label - self._data = data - self.manager = manager - - environment = {} - try: - environment = json.loads(data["environment"]) - except Exception: - pass - self._environment = environment - - variants = data.get("variants") or [] - variants_by_name = {} - for variant_data in variants: - tool = EnvironmentTool(variant_data, self) - variants_by_name[tool.name] = tool - self.variants = variants_by_name - - def __repr__(self): - return "<{}> - {}".format(self.__class__.__name__, self.name) - - def __iter__(self): - for variant in self.variants.values(): - yield variant - - @property - def environment(self): - return copy.deepcopy(self._environment) - - -class EnvironmentTool: - """Hold information about application tool. - - Structure of tool information. - - Args: - variant_data (dict): Variant data with environments and - host and app variant filters. - group (EnvironmentToolGroup): Name of group which wraps tool. - """ - - def __init__(self, variant_data, group): - # Backwards compatibility 3.9.1 - 3.9.2 - # - 'variant_data' contained only environments but contain also host - # and application variant filters - name = variant_data["name"] - label = variant_data["label"] - host_names = variant_data["host_names"] - app_variants = variant_data["app_variants"] - - environment = {} - try: - environment = json.loads(variant_data["environment"]) - except Exception: - pass - - self.host_names = host_names - self.app_variants = app_variants - self.name = name - self.variant_label = label - self.label = " ".join((group.label, label)) - self.group = group - - self._environment = environment - self.full_name = "/".join((group.name, name)) - - def __repr__(self): - return "<{}> - {}".format(self.__class__.__name__, self.full_name) - - @property - def environment(self): - return copy.deepcopy(self._environment) - - def is_valid_for_app(self, app): - """Is tool valid for application. - - Args: - app (Application): Application for which are prepared environments. - """ - if self.app_variants and app.full_name not in self.app_variants: - return False - - if self.host_names and app.host_name not in self.host_names: - return False - return True - - -class ApplicationExecutable: - """Representation of executable loaded from settings.""" - - def __init__(self, executable): - # Try to format executable with environments - try: - executable = executable.format(**os.environ) - except Exception: - pass - - # On MacOS check if exists path to executable when ends with `.app` - # - it is common that path will lead to "/Applications/Blender" but - # real path is "/Applications/Blender.app" - if platform.system().lower() == "darwin": - executable = self.macos_executable_prep(executable) - - self.executable_path = executable - - def __str__(self): - return self.executable_path - - def __repr__(self): - return "<{}> {}".format(self.__class__.__name__, self.executable_path) - - @staticmethod - def macos_executable_prep(executable): - """Try to find full path to executable file. - - Real executable is stored in '*.app/Contents/MacOS/'. - - Having path to '*.app' gives ability to read it's plist info and - use "CFBundleExecutable" key from plist to know what is "executable." - - Plist is stored in '*.app/Contents/Info.plist'. - - This is because some '*.app' directories don't have same permissions - as real executable. - """ - # Try to find if there is `.app` file - if not os.path.exists(executable): - _executable = executable + ".app" - if os.path.exists(_executable): - executable = _executable - - # Try to find real executable if executable has `Contents` subfolder - contents_dir = os.path.join(executable, "Contents") - if os.path.exists(contents_dir): - executable_filename = None - # Load plist file and check for bundle executable - plist_filepath = os.path.join(contents_dir, "Info.plist") - if os.path.exists(plist_filepath): - import plistlib - - if hasattr(plistlib, "load"): - with open(plist_filepath, "rb") as stream: - parsed_plist = plistlib.load(stream) - else: - parsed_plist = plistlib.readPlist(plist_filepath) - executable_filename = parsed_plist.get("CFBundleExecutable") - - if executable_filename: - executable = os.path.join( - contents_dir, "MacOS", executable_filename - ) - - return executable - - def as_args(self): - return [self.executable_path] - - def _realpath(self): - """Check if path is valid executable path.""" - # Check for executable in PATH - result = find_executable(self.executable_path) - if result is not None: - return result - - # This is not 100% validation but it is better than remove ability to - # launch .bat, .sh or extentionless files - if os.path.exists(self.executable_path): - return self.executable_path - return None - - def exists(self): - if not self.executable_path: - return False - return bool(self._realpath()) - - -class UndefinedApplicationExecutable(ApplicationExecutable): - """Some applications do not require executable path from settings. - - In that case this class is used to "fake" existing executable. - """ - def __init__(self): - pass - - def __str__(self): - return self.__class__.__name__ - - def __repr__(self): - return "<{}>".format(self.__class__.__name__) - - def as_args(self): - return [] - - def exists(self): - return True - - -@six.add_metaclass(ABCMeta) -class LaunchHook: - """Abstract base class of launch hook.""" - # Order of prelaunch hook, will be executed as last if set to None. - order = None - # List of host implementations, skipped if empty. - hosts = set() - # Set of application groups - app_groups = set() - # Set of specific application names - app_names = set() - # Set of platform availability - platforms = set() - # Set of launch types for which is available - # - if empty then is available for all launch types - # - by default has 'local' which is most common reason for launc hooks - launch_types = {LaunchTypes.local} - - def __init__(self, launch_context): - """Constructor of launch hook. - - Always should be called - """ - self.log = Logger.get_logger(self.__class__.__name__) - - self.launch_context = launch_context - - is_valid = self.class_validation(launch_context) - if is_valid: - is_valid = self.validate() - - self.is_valid = is_valid - - @classmethod - def class_validation(cls, launch_context): - """Validation of class attributes by launch context. - - Args: - launch_context (ApplicationLaunchContext): Context of launching - application. - - Returns: - bool: Is launch hook valid for the context by class attributes. - """ - if cls.platforms: - low_platforms = tuple( - _platform.lower() - for _platform in cls.platforms - ) - if platform.system().lower() not in low_platforms: - return False - - if cls.hosts: - if launch_context.host_name not in cls.hosts: - return False - - if cls.app_groups: - if launch_context.app_group.name not in cls.app_groups: - return False - - if cls.app_names: - if launch_context.app_name not in cls.app_names: - return False - - if cls.launch_types: - if launch_context.launch_type not in cls.launch_types: - return False - - return True - - @property - def data(self): - return self.launch_context.data - - @property - def application(self): - return getattr(self.launch_context, "application", None) - - @property - def manager(self): - return getattr(self.application, "manager", None) - - @property - def host_name(self): - return getattr(self.application, "host_name", None) - - @property - def app_group(self): - return getattr(self.application, "group", None) - - @property - def app_name(self): - return getattr(self.application, "full_name", None) - - @property - def addons_manager(self): - return getattr(self.launch_context, "addons_manager", None) - - @property - def modules_manager(self): - """ - Deprecated: - Use 'addons_wrapper' instead. - """ - return self.addons_manager - - def validate(self): - """Optional validation of launch hook on initialization. - - Returns: - bool: Hook is valid (True) or invalid (False). - """ - # QUESTION Not sure if this method has any usable potential. - # - maybe result can be based on settings - return True - - @abstractmethod - def execute(self, *args, **kwargs): - """Abstract execute method where logic of hook is.""" - pass - - -class PreLaunchHook(LaunchHook): - """Abstract class of prelaunch hook. - - This launch hook will be processed before application is launched. - - If any exception will happen during processing the application won't be - launched. - """ - - -class PostLaunchHook(LaunchHook): - """Abstract class of postlaunch hook. - - This launch hook will be processed after application is launched. - - Nothing will happen if any exception will happen during processing. And - processing of other postlaunch hooks won't stop either. - """ - - -class ApplicationLaunchContext: - """Context of launching application. - - Main purpose of context is to prepare launch arguments and keyword - arguments for new process. Most important part of keyword arguments - preparations are environment variables. - - During the whole process is possible to use `data` attribute to store - object usable in multiple places. - - Launch arguments are strings in list. It is possible to "chain" argument - when order of them matters. That is possible to do with adding list where - order is right and should not change. - NOTE: This is recommendation, not requirement. - e.g.: `["nuke.exe", "--NukeX"]` -> In this case any part of process may - insert argument between `nuke.exe` and `--NukeX`. To keep them together - it is better to wrap them in another list: `[["nuke.exe", "--NukeX"]]`. - - Notes: - It is possible to use launch context only to prepare environment - variables. In that case `executable` may be None and can be used - 'run_prelaunch_hooks' method to run prelaunch hooks which prepare - them. - - Args: - application (Application): Application definition. - executable (ApplicationExecutable): Object with path to executable. - env_group (Optional[str]): Environment variable group. If not set - 'DEFAULT_ENV_SUBGROUP' is used. - launch_type (Optional[str]): Launch type. If not set 'local' is used. - **data (dict): Any additional data. Data may be used during - preparation to store objects usable in multiple places. - """ - - def __init__( - self, - application, - executable, - env_group=None, - launch_type=None, - **data - ): - from ayon_core.addon import AddonsManager - - # Application object - self.application = application - - self.addons_manager = AddonsManager() - - # Logger - logger_name = "{}-{}".format(self.__class__.__name__, - self.application.full_name) - self.log = Logger.get_logger(logger_name) - - self.executable = executable - - if launch_type is None: - launch_type = LaunchTypes.local - self.launch_type = launch_type - - if env_group is None: - env_group = DEFAULT_ENV_SUBGROUP - - self.env_group = env_group - - self.data = dict(data) - - launch_args = [] - if executable is not None: - launch_args = executable.as_args() - # subprocess.Popen launch arguments (first argument in constructor) - self.launch_args = launch_args - self.launch_args.extend(application.arguments) - if self.data.get("app_args"): - self.launch_args.extend(self.data.pop("app_args")) - - # Handle launch environemtns - src_env = self.data.pop("env", None) - if src_env is not None and not isinstance(src_env, dict): - self.log.warning(( - "Passed `env` kwarg has invalid type: {}. Expected: `dict`." - " Using `os.environ` instead." - ).format(str(type(src_env)))) - src_env = None - - if src_env is None: - src_env = os.environ - - ignored_env = {"QT_API", } - env = { - key: str(value) - for key, value in src_env.items() - if key not in ignored_env - } - # subprocess.Popen keyword arguments - self.kwargs = {"env": env} - - if platform.system().lower() == "windows": - # Detach new process from currently running process on Windows - flags = ( - subprocess.CREATE_NEW_PROCESS_GROUP - | subprocess.DETACHED_PROCESS - ) - self.kwargs["creationflags"] = flags - - if not sys.stdout: - self.kwargs["stdout"] = subprocess.DEVNULL - self.kwargs["stderr"] = subprocess.DEVNULL - - self.prelaunch_hooks = None - self.postlaunch_hooks = None - - self.process = None - self._prelaunch_hooks_executed = False - - @property - def env(self): - if ( - "env" not in self.kwargs - or self.kwargs["env"] is None - ): - self.kwargs["env"] = {} - return self.kwargs["env"] - - @env.setter - def env(self, value): - if not isinstance(value, dict): - raise ValueError( - "'env' attribute expect 'dict' object. Got: {}".format( - str(type(value)) - ) - ) - self.kwargs["env"] = value - - @property - def modules_manager(self): - """ - Deprecated: - Use 'addons_manager' instead. - - """ - return self.addons_manager - - def _collect_addons_launch_hook_paths(self): - """Helper to collect application launch hooks from addons. - - Module have to have implemented 'get_launch_hook_paths' method which - can expect application as argument or nothing. - - Returns: - List[str]: Paths to launch hook directories. - """ - - expected_types = (list, tuple, set) - - output = [] - for module in self.addons_manager.get_enabled_addons(): - # Skip module if does not have implemented 'get_launch_hook_paths' - func = getattr(module, "get_launch_hook_paths", None) - if func is None: - continue - - func = module.get_launch_hook_paths - if hasattr(inspect, "signature"): - sig = inspect.signature(func) - expect_args = len(sig.parameters) > 0 - else: - expect_args = len(inspect.getargspec(func)[0]) > 0 - - # Pass application argument if method expect it. - try: - if expect_args: - hook_paths = func(self.application) - else: - hook_paths = func() - except Exception: - self.log.warning( - "Failed to call 'get_launch_hook_paths'", - exc_info=True - ) - continue - - if not hook_paths: - continue - - # Convert string to list - if isinstance(hook_paths, six.string_types): - hook_paths = [hook_paths] - - # Skip invalid types - if not isinstance(hook_paths, expected_types): - self.log.warning(( - "Result of `get_launch_hook_paths`" - " has invalid type {}. Expected {}" - ).format(type(hook_paths), expected_types)) - continue - - output.extend(hook_paths) - return output - - def paths_to_launch_hooks(self): - """Directory paths where to look for launch hooks.""" - # This method has potential to be part of application manager (maybe). - paths = [] - - # TODO load additional studio paths from settings - global_hooks_dir = os.path.join(AYON_CORE_ROOT, "hooks") - - hooks_dirs = [ - global_hooks_dir - ] - if self.host_name: - # If host requires launch hooks and is module then launch hooks - # should be collected using 'collect_launch_hook_paths' - # - module have to implement 'get_launch_hook_paths' - host_module = self.addons_manager.get_host_addon(self.host_name) - if not host_module: - hooks_dirs.append(os.path.join( - AYON_CORE_ROOT, "hosts", self.host_name, "hooks" - )) - - for path in hooks_dirs: - if ( - os.path.exists(path) - and os.path.isdir(path) - and path not in paths - ): - paths.append(path) - - # Load modules paths - paths.extend(self._collect_addons_launch_hook_paths()) - - return paths - - def discover_launch_hooks(self, force=False): - """Load and prepare launch hooks.""" - if ( - self.prelaunch_hooks is not None - or self.postlaunch_hooks is not None - ): - if not force: - self.log.info("Launch hooks were already discovered.") - return - - self.prelaunch_hooks.clear() - self.postlaunch_hooks.clear() - - self.log.debug("Discovery of launch hooks started.") - - paths = self.paths_to_launch_hooks() - self.log.debug("Paths searched for launch hooks:\n{}".format( - "\n".join("- {}".format(path) for path in paths) - )) - - all_classes = { - "pre": [], - "post": [] - } - for path in paths: - if not os.path.exists(path): - self.log.info( - "Path to launch hooks does not exist: \"{}\"".format(path) - ) - continue - - modules, _crashed = modules_from_path(path) - for _filepath, module in modules: - all_classes["pre"].extend( - classes_from_module(PreLaunchHook, module) - ) - all_classes["post"].extend( - classes_from_module(PostLaunchHook, module) - ) - - for launch_type, classes in all_classes.items(): - hooks_with_order = [] - hooks_without_order = [] - for klass in classes: - try: - hook = klass(self) - if not hook.is_valid: - self.log.debug( - "Skipped hook invalid for current launch context: " - "{}".format(klass.__name__) - ) - continue - - if inspect.isabstract(hook): - self.log.debug("Skipped abstract hook: {}".format( - klass.__name__ - )) - continue - - # Separate hooks by pre/post class - if hook.order is None: - hooks_without_order.append(hook) - else: - hooks_with_order.append(hook) - - except Exception: - self.log.warning( - "Initialization of hook failed: " - "{}".format(klass.__name__), - exc_info=True - ) - - # Sort hooks with order by order - ordered_hooks = list(sorted( - hooks_with_order, key=lambda obj: obj.order - )) - # Extend ordered hooks with hooks without defined order - ordered_hooks.extend(hooks_without_order) - - if launch_type == "pre": - self.prelaunch_hooks = ordered_hooks - else: - self.postlaunch_hooks = ordered_hooks - - self.log.debug("Found {} prelaunch and {} postlaunch hooks.".format( - len(self.prelaunch_hooks), len(self.postlaunch_hooks) - )) - - @property - def app_name(self): - return self.application.name - - @property - def host_name(self): - return self.application.host_name - - @property - def app_group(self): - return self.application.group - - @property - def manager(self): - return self.application.manager - - def _run_process(self): - # Windows and MacOS have easier process start - low_platform = platform.system().lower() - if low_platform in ("windows", "darwin"): - return subprocess.Popen(self.launch_args, **self.kwargs) - - # Linux uses mid process - # - it is possible that the mid process executable is not - # available for this version of AYON in that case use standard - # launch - launch_args = get_linux_launcher_args() - if launch_args is None: - return subprocess.Popen(self.launch_args, **self.kwargs) - - # Prepare data that will be passed to midprocess - # - store arguments to a json and pass path to json as last argument - # - pass environments to set - app_env = self.kwargs.pop("env", {}) - json_data = { - "args": self.launch_args, - "env": app_env - } - if app_env: - # Filter environments of subprocess - self.kwargs["env"] = { - key: value - for key, value in os.environ.items() - if key in app_env - } - - # Create temp file - json_temp = tempfile.NamedTemporaryFile( - mode="w", prefix="op_app_args", suffix=".json", delete=False - ) - json_temp.close() - json_temp_filpath = json_temp.name - with open(json_temp_filpath, "w") as stream: - json.dump(json_data, stream) - - launch_args.append(json_temp_filpath) - - # Create mid-process which will launch application - process = subprocess.Popen(launch_args, **self.kwargs) - # Wait until the process finishes - # - This is important! The process would stay in "open" state. - process.wait() - # Remove the temp file - os.remove(json_temp_filpath) - # Return process which is already terminated - return process - - def run_prelaunch_hooks(self): - """Run prelaunch hooks. - - This method will be executed only once, any future calls will skip - the processing. - """ - - if self._prelaunch_hooks_executed: - self.log.warning("Prelaunch hooks were already executed.") - return - # Discover launch hooks - self.discover_launch_hooks() - - # Execute prelaunch hooks - for prelaunch_hook in self.prelaunch_hooks: - self.log.debug("Executing prelaunch hook: {}".format( - str(prelaunch_hook.__class__.__name__) - )) - prelaunch_hook.execute() - self._prelaunch_hooks_executed = True - - def launch(self): - """Collect data for new process and then create it. - - This method must not be executed more than once. - - Returns: - subprocess.Popen: Created process as Popen object. - """ - if self.process is not None: - self.log.warning("Application was already launched.") - return - - if not self._prelaunch_hooks_executed: - self.run_prelaunch_hooks() - - self.log.debug("All prelaunch hook executed. Starting new process.") - - # Prepare subprocess args - args_len_str = "" - if isinstance(self.launch_args, str): - args = self.launch_args - else: - args = self.clear_launch_args(self.launch_args) - args_len_str = " ({})".format(len(args)) - self.log.info( - "Launching \"{}\" with args{}: {}".format( - self.application.full_name, args_len_str, args - ) - ) - self.launch_args = args - - # Run process - self.process = self._run_process() - - # Process post launch hooks - for postlaunch_hook in self.postlaunch_hooks: - self.log.debug("Executing postlaunch hook: {}".format( - str(postlaunch_hook.__class__.__name__) - )) - - # TODO how to handle errors? - # - store to variable to let them accessible? - try: - postlaunch_hook.execute() - - except Exception: - self.log.warning( - "After launch procedures were not successful.", - exc_info=True - ) - - self.log.debug("Launch of {} finished.".format( - self.application.full_name - )) - - return self.process - - @staticmethod - def clear_launch_args(args): - """Collect launch arguments to final order. - - Launch argument should be list that may contain another lists this - function will upack inner lists and keep ordering. - - ``` - # source - [ [ arg1, [ arg2, arg3 ] ], arg4, [arg5, arg6]] - # result - [ arg1, arg2, arg3, arg4, arg5, arg6] - - Args: - args (list): Source arguments in list may contain inner lists. - - Return: - list: Unpacked arguments. - """ - if isinstance(args, str): - return args - all_cleared = False - while not all_cleared: - all_cleared = True - new_args = [] - for arg in args: - if isinstance(arg, (list, tuple, set)): - all_cleared = False - for _arg in arg: - new_args.append(_arg) - else: - new_args.append(arg) - args = new_args - - return args - - -class MissingRequiredKey(KeyError): - pass - - -class EnvironmentPrepData(dict): - """Helper dictionary for storin temp data during environment prep. - - Args: - data (dict): Data must contain required keys. - """ - required_keys = ( - "project_entity", "folder_entity", "task_entity", "app", "anatomy" - ) - - def __init__(self, data): - for key in self.required_keys: - if key not in data: - raise MissingRequiredKey(key) - - if not data.get("log"): - data["log"] = get_logger() - - if data.get("env") is None: - data["env"] = os.environ.copy() - - project_name = data["project_entity"]["name"] - if "project_settings" not in data: - data["project_settings"] = get_project_settings(project_name) - - super(EnvironmentPrepData, self).__init__(data) - - -def get_app_environments_for_context( - project_name, - folder_path, - task_name, - app_name, - env_group=None, - launch_type=None, - env=None, - addons_manager=None -): - """Prepare environment variables by context. - Args: - project_name (str): Name of project. - folder_path (str): Folder path. - task_name (str): Name of task. - app_name (str): Name of application that is launched and can be found - by ApplicationManager. - env_group (Optional[str]): Name of environment group. If not passed - default group is used. - launch_type (Optional[str]): Type for which prelaunch hooks are - executed. - env (Optional[dict[str, str]]): Initial environment variables. - `os.environ` is used when not passed. - addons_manager (Optional[AddonsManager]): Initialized modules - manager. - - Returns: - dict: Environments for passed context and application. - """ - - # Prepare app object which can be obtained only from ApplicationManager - app_manager = ApplicationManager() - context = app_manager.create_launch_context( - app_name, - project_name=project_name, - folder_path=folder_path, - task_name=task_name, - env_group=env_group, - launch_type=launch_type, - env=env, - addons_manager=addons_manager, - modules_manager=addons_manager, - ) - context.run_prelaunch_hooks() - return context.env - - -def _merge_env(env, current_env): - """Modified function(merge) from acre module.""" - import acre - - result = current_env.copy() - for key, value in env.items(): - # Keep missing keys by not filling `missing` kwarg - value = acre.lib.partial_format(value, data=current_env) - result[key] = value - return result - - -def _add_python_version_paths(app, env, logger, addons_manager): - """Add vendor packages specific for a Python version.""" - - for addon in addons_manager.get_enabled_addons(): - addon.modify_application_launch_arguments(app, env) - - # Skip adding if host name is not set - if not app.host_name: - return - - # Add Python 2/3 modules - python_vendor_dir = os.path.join( - AYON_CORE_ROOT, - "vendor", - "python" - ) - if app.use_python_2: - pythonpath = os.path.join(python_vendor_dir, "python_2") - else: - pythonpath = os.path.join(python_vendor_dir, "python_3") - - if not os.path.exists(pythonpath): - return - - logger.debug("Adding Python version specific paths to PYTHONPATH") - python_paths = [pythonpath] - - # Load PYTHONPATH from current launch context - python_path = env.get("PYTHONPATH") - if python_path: - python_paths.append(python_path) - - # Set new PYTHONPATH to launch context environments - env["PYTHONPATH"] = os.pathsep.join(python_paths) - - -def prepare_app_environments( - data, env_group=None, implementation_envs=True, addons_manager=None -): - """Modify launch environments based on launched app and context. - - Args: - data (EnvironmentPrepData): Dictionary where result and intermediate - result will be stored. - """ - import acre - - app = data["app"] - log = data["log"] - source_env = data["env"].copy() - - if addons_manager is None: - from ayon_core.addon import AddonsManager - - addons_manager = AddonsManager() - - _add_python_version_paths(app, source_env, log, addons_manager) - - # Use environments from local settings - filtered_local_envs = {} - # NOTE Overrides for environment variables are not implemented in AYON. - # project_settings = data["project_settings"] - # whitelist_envs = project_settings["general"].get("local_env_white_list") - # if whitelist_envs: - # local_settings = get_local_settings() - # local_envs = local_settings.get("environments") or {} - # filtered_local_envs = { - # key: value - # for key, value in local_envs.items() - # if key in whitelist_envs - # } - - # Apply local environment variables for already existing values - for key, value in filtered_local_envs.items(): - if key in source_env: - source_env[key] = value - - # `app_and_tool_labels` has debug purpose - app_and_tool_labels = [app.full_name] - # Environments for application - environments = [ - app.group.environment, - app.environment - ] - - folder_entity = data.get("folder_entity") - # Add tools environments - groups_by_name = {} - tool_by_group_name = collections.defaultdict(dict) - if folder_entity: - # Make sure each tool group can be added only once - for key in folder_entity["attrib"].get("tools") or []: - tool = app.manager.tools.get(key) - if not tool or not tool.is_valid_for_app(app): - continue - groups_by_name[tool.group.name] = tool.group - tool_by_group_name[tool.group.name][tool.name] = tool - - for group_name in sorted(groups_by_name.keys()): - group = groups_by_name[group_name] - environments.append(group.environment) - for tool_name in sorted(tool_by_group_name[group_name].keys()): - tool = tool_by_group_name[group_name][tool_name] - environments.append(tool.environment) - app_and_tool_labels.append(tool.full_name) - - log.debug( - "Will add environments for apps and tools: {}".format( - ", ".join(app_and_tool_labels) - ) - ) - - env_values = {} - for _env_values in environments: - if not _env_values: - continue - - # Choose right platform - tool_env = parse_environments(_env_values, env_group) - - # Apply local environment variables - # - must happen between all values because they may be used during - # merge - for key, value in filtered_local_envs.items(): - if key in tool_env: - tool_env[key] = value - - # Merge dictionaries - env_values = _merge_env(tool_env, env_values) - - merged_env = _merge_env(env_values, source_env) - - loaded_env = acre.compute(merged_env, cleanup=False) - - final_env = None - # Add host specific environments - if app.host_name and implementation_envs: - host_addon = addons_manager.get_host_addon(app.host_name) - if not host_addon: - module = __import__("ayon_core.hosts", fromlist=[app.host_name]) - host_module = getattr(module, app.host_name, None) - add_implementation_envs = None - if host_addon: - add_implementation_envs = getattr( - host_addon, "add_implementation_envs", None - ) - if add_implementation_envs: - # Function may only modify passed dict without returning value - final_env = add_implementation_envs(loaded_env, app) - - if final_env is None: - final_env = loaded_env - - keys_to_remove = set(source_env.keys()) - set(final_env.keys()) - - # Update env - data["env"].update(final_env) - for key in keys_to_remove: - data["env"].pop(key, None) - - -def apply_project_environments_value( - project_name, env, project_settings=None, env_group=None -): - """Apply project specific environments on passed environments. - - The environments are applied on passed `env` argument value so it is not - required to apply changes back. - - Args: - project_name (str): Name of project for which environments should be - received. - env (dict): Environment values on which project specific environments - will be applied. - project_settings (dict): Project settings for passed project name. - Optional if project settings are already prepared. - - Returns: - dict: Passed env values with applied project environments. - - Raises: - KeyError: If project settings do not contain keys for project specific - environments. - """ - import acre - - if project_settings is None: - project_settings = get_project_settings(project_name) - - env_value = project_settings["core"]["project_environments"] - if env_value: - env_value = json.loads(env_value) - parsed_value = parse_environments(env_value, env_group) - env.update(acre.compute( - _merge_env(parsed_value, env), - cleanup=False - )) - return env - - -def prepare_context_environments(data, env_group=None, addons_manager=None): - """Modify launch environments with context data for launched host. - - Args: - data (EnvironmentPrepData): Dictionary where result and intermediate - result will be stored. - """ - - from ayon_core.pipeline.template_data import get_template_data - - # Context environments - log = data["log"] - - project_entity = data["project_entity"] - folder_entity = data["folder_entity"] - task_entity = data["task_entity"] - if not project_entity: - log.info( - "Skipping context environments preparation." - " Launch context does not contain required data." - ) - return - - # Load project specific environments - project_name = project_entity["name"] - project_settings = get_project_settings(project_name) - data["project_settings"] = project_settings - - app = data["app"] - context_env = { - "AYON_PROJECT_NAME": project_entity["name"], - "AYON_APP_NAME": app.full_name - } - if folder_entity: - folder_path = folder_entity["path"] - context_env["AYON_FOLDER_PATH"] = folder_path - - if task_entity: - context_env["AYON_TASK_NAME"] = task_entity["name"] - - log.debug( - "Context environments set:\n{}".format( - json.dumps(context_env, indent=4) - ) - ) - data["env"].update(context_env) - - # Apply project specific environments on current env value - # - apply them once the context environments are set - apply_project_environments_value( - project_name, data["env"], project_settings, env_group - ) - - if not app.is_host: - return - - data["env"]["AYON_HOST_NAME"] = app.host_name - - if not folder_entity or not task_entity: - # QUESTION replace with log.info and skip workfile discovery? - # - technically it should be possible to launch host without context - raise ApplicationLaunchFailed( - "Host launch require folder and task context." - ) - - workdir_data = get_template_data( - project_entity, - folder_entity, - task_entity, - app.host_name, - project_settings - ) - data["workdir_data"] = workdir_data - - anatomy = data["anatomy"] - - task_type = workdir_data["task"]["type"] - # Temp solution how to pass task type to `_prepare_last_workfile` - data["task_type"] = task_type - - try: - from ayon_core.pipeline.workfile import get_workdir_with_workdir_data - - workdir = get_workdir_with_workdir_data( - workdir_data, - anatomy.project_name, - anatomy, - project_settings=project_settings - ) - - except Exception as exc: - raise ApplicationLaunchFailed( - "Error in anatomy.format: {}".format(str(exc)) - ) - - if not os.path.exists(workdir): - log.debug( - "Creating workdir folder: \"{}\"".format(workdir) - ) - try: - os.makedirs(workdir) - except Exception as exc: - raise ApplicationLaunchFailed( - "Couldn't create workdir because: {}".format(str(exc)) - ) - - data["env"]["AYON_WORKDIR"] = workdir - - _prepare_last_workfile(data, workdir, addons_manager) - - -def _prepare_last_workfile(data, workdir, addons_manager): - """last workfile workflow preparation. - - Function check if should care about last workfile workflow and tries - to find the last workfile. Both information are stored to `data` and - environments. - - Last workfile is filled always (with version 1) even if any workfile - exists yet. - - Args: - data (EnvironmentPrepData): Dictionary where result and intermediate - result will be stored. - workdir (str): Path to folder where workfiles should be stored. - """ - - from ayon_core.addon import AddonsManager - from ayon_core.pipeline import HOST_WORKFILE_EXTENSIONS - from ayon_core.pipeline.workfile import ( - should_use_last_workfile_on_launch, - should_open_workfiles_tool_on_launch, - ) - - if not addons_manager: - addons_manager = AddonsManager() - - log = data["log"] - - _workdir_data = data.get("workdir_data") - if not _workdir_data: - log.info( - "Skipping last workfile preparation." - " Key `workdir_data` not filled." - ) - return - - app = data["app"] - workdir_data = copy.deepcopy(_workdir_data) - project_name = data["project_name"] - task_name = data["task_name"] - task_type = data["task_type"] - - start_last_workfile = data.get("start_last_workfile") - if start_last_workfile is None: - start_last_workfile = should_use_last_workfile_on_launch( - project_name, app.host_name, task_name, task_type - ) - else: - log.info("Opening of last workfile was disabled by user") - - data["start_last_workfile"] = start_last_workfile - - workfile_startup = should_open_workfiles_tool_on_launch( - project_name, app.host_name, task_name, task_type - ) - data["workfile_startup"] = workfile_startup - - # Store boolean as "0"(False) or "1"(True) - data["env"]["AVALON_OPEN_LAST_WORKFILE"] = ( - str(int(bool(start_last_workfile))) - ) - data["env"]["AYON_WORKFILE_TOOL_ON_START"] = ( - str(int(bool(workfile_startup))) - ) - - _sub_msg = "" if start_last_workfile else " not" - log.debug( - "Last workfile should{} be opened on start.".format(_sub_msg) - ) - - # Last workfile path - last_workfile_path = data.get("last_workfile_path") or "" - if not last_workfile_path: - host_addon = addons_manager.get_host_addon(app.host_name) - if host_addon: - extensions = host_addon.get_workfile_extensions() - else: - extensions = HOST_WORKFILE_EXTENSIONS.get(app.host_name) - - if extensions: - from ayon_core.pipeline.workfile import ( - get_workfile_template_key, - get_last_workfile - ) - - anatomy = data["anatomy"] - project_settings = data["project_settings"] - task_type = workdir_data["task"]["type"] - template_key = get_workfile_template_key( - project_name, - task_type, - app.host_name, - project_settings=project_settings - ) - # Find last workfile - file_template = anatomy.get_template_item( - "work", template_key, "file" - ).template - - workdir_data.update({ - "version": 1, - "user": get_ayon_username(), - "ext": extensions[0] - }) - - last_workfile_path = get_last_workfile( - workdir, file_template, workdir_data, extensions, True - ) - - if os.path.exists(last_workfile_path): - log.debug(( - "Workfiles for launch context does not exists" - " yet but path will be set." - )) - log.debug( - "Setting last workfile path: {}".format(last_workfile_path) - ) - - data["env"]["AYON_LAST_WORKFILE"] = last_workfile_path - data["last_workfile_path"] = last_workfile_path From 2a833834d16c36087910034a696a1467ad8e7e4d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:34:35 +0100 Subject: [PATCH 11/27] modified launcher to be able to handle applications actions --- .../tools/launcher/models/actions.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/client/ayon_core/tools/launcher/models/actions.py b/client/ayon_core/tools/launcher/models/actions.py index 97943e6ad7..be78e89389 100644 --- a/client/ayon_core/tools/launcher/models/actions.py +++ b/client/ayon_core/tools/launcher/models/actions.py @@ -2,6 +2,7 @@ import os from ayon_core import resources from ayon_core.lib import Logger, AYONSettingsRegistry +from ayon_core.addon import AddonsManager from ayon_core.pipeline.actions import ( discover_launcher_actions, LauncherAction, @@ -116,7 +117,7 @@ class ApplicationAction(LauncherAction): """Process the full Application action""" from ayon_core.lib import ( - ApplictionExecutableNotFound, + ApplicationExecutableNotFound, ApplicationLaunchFailed, ) @@ -131,7 +132,7 @@ class ApplicationAction(LauncherAction): **self.data ) - except ApplictionExecutableNotFound as exc: + except ApplicationExecutableNotFound as exc: details = exc.details msg = exc.msg log_msg = str(msg) @@ -279,6 +280,8 @@ class ActionsModel: self._launcher_tool_reg = AYONSettingsRegistry("launcher_tool") + self._addons_manager = None + @property def log(self): if self._log is None: @@ -419,6 +422,11 @@ class ActionsModel: } ) + def _get_addons_manager(self): + if self._addons_manager is None: + self._addons_manager = AddonsManager() + return self._addons_manager + def _get_no_last_workfile_reg_data(self): try: no_workfile_reg_data = self._launcher_tool_reg.get_item( @@ -519,19 +527,16 @@ class ActionsModel: return action_items def _get_applications_action_classes(self): - from ayon_core.lib.applications import ( - CUSTOM_LAUNCH_APP_GROUPS, - ApplicationManager, - ) - actions = [] - manager = ApplicationManager() + addons_manager = self._get_addons_manager() + applications_addon = addons_manager.get_enabled_addon("applications") + if applications_addon is None: + return actions + + manager = applications_addon.get_applications_manager() for full_name, application in manager.applications.items(): - if ( - application.group.name in CUSTOM_LAUNCH_APP_GROUPS - or not application.enabled - ): + if not application.enabled: continue action = type( From 4cc8f8fb7f7a1e420421f26a8b65c9fbaaf68152 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 27 Mar 2024 18:40:04 +0100 Subject: [PATCH 12/27] modified cli functions --- client/ayon_core/cli.py | 4 +++ client/ayon_core/cli_commands.py | 59 ++++++++++++++------------------ 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/client/ayon_core/cli.py b/client/ayon_core/cli.py index 2759b4fccf..4e7dc6e65b 100644 --- a/client/ayon_core/cli.py +++ b/client/ayon_core/cli.py @@ -96,6 +96,10 @@ def extractenvironments(output_json_path, project, asset, task, app, envgroup): environments will be extracted. Context options are "project", "asset", "task", "app" + + Deprecated: + This function is deprecated and will be removed in future. Please use + 'addon applications extractenvironments ...' instead. """ Commands.extractenvironments( output_json_path, project, asset, task, app, envgroup diff --git a/client/ayon_core/cli_commands.py b/client/ayon_core/cli_commands.py index fa90571462..a69c6324ac 100644 --- a/client/ayon_core/cli_commands.py +++ b/client/ayon_core/cli_commands.py @@ -2,7 +2,7 @@ """Implementation of AYON commands.""" import os import sys -import json +import warnings class Commands: @@ -57,10 +57,7 @@ class Commands: """ from ayon_core.lib import Logger - from ayon_core.lib.applications import ( - get_app_environments_for_context, - LaunchTypes, - ) + from ayon_core.addon import AddonsManager from ayon_core.pipeline import ( install_ayon_plugins, @@ -68,7 +65,6 @@ class Commands: ) # Register target and host - import pyblish.api import pyblish.util if not isinstance(path, str): @@ -99,15 +95,13 @@ class Commands: for plugin_path in publish_paths: pyblish.api.register_plugin_path(plugin_path) - app_full_name = os.getenv("AYON_APP_NAME") - if app_full_name: + applications_addon = manager.get_enabled_addon("applications") + if applications_addon is not None: context = get_global_context() - env = get_app_environments_for_context( + env = applications_addon.get_farm_publish_environment_variables( context["project_name"], context["folder_path"], context["task_name"], - app_full_name, - launch_type=LaunchTypes.farm_publish, ) os.environ.update(env) @@ -149,36 +143,35 @@ class Commands: log.info("Publish finished.") @staticmethod - def extractenvironments(output_json_path, project, asset, task, app, - env_group): + def extractenvironments( + output_json_path, project, asset, task, app, env_group + ): """Produces json file with environment based on project and app. Called by Deadline plugin to propagate environment into render jobs. """ - from ayon_core.lib.applications import ( - get_app_environments_for_context, - LaunchTypes, + from ayon_core.addon import AddonsManager + + warnings.warn( + ( + "Command 'extractenvironments' is deprecated and will be" + " removed in future. Please use " + "'addon applications extractenvironments ...' instead." + ), + DeprecationWarning ) - - if all((project, asset, task, app)): - env = get_app_environments_for_context( - project, - asset, - task, - app, - env_group=env_group, - launch_type=LaunchTypes.farm_render + addons_manager = AddonsManager() + applications_addon = addons_manager.get_enabled_addon("applications") + if applications_addon is None: + raise RuntimeError( + "Applications addon is not available or enabled." ) - else: - env = os.environ.copy() - output_dir = os.path.dirname(output_json_path) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - with open(output_json_path, "w") as file_stream: - json.dump(env, file_stream, indent=4) + # Please ignore the fact this is using private method + applications_addon._cli_extract_environments( + output_json_path, project, asset, task, app, env_group + ) @staticmethod def contextselection(output_path, project_name, folder_path, strict): From 6a569c9e212abeb26c069f76a962db1c4b901719 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:04:49 +0100 Subject: [PATCH 13/27] use applications addon arguments to call extractenvironments --- .../custom/plugins/GlobalJobPreLoad.py | 56 +++++++++++++------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 8df96b425e..5f40470f96 100644 --- a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -463,19 +463,13 @@ def inject_ayon_environment(deadlinePlugin): export_url = os.path.join(tempfile.gettempdir(), temp_file_name) print(">>> Temporary path: {}".format(export_url)) - args = [ - "--headless", - "extractenvironments", - export_url - ] - add_kwargs = { "envgroup": "farm", } # Support backwards compatible keys for key, env_keys in ( ("project", ["AYON_PROJECT_NAME", "AVALON_PROJECT"]), - ("asset", ["AYON_FOLDER_PATH", "AVALON_ASSET"]), + ("folder", ["AYON_FOLDER_PATH", "AVALON_ASSET"]), ("task", ["AYON_TASK_NAME", "AVALON_TASK"]), ("app", ["AYON_APP_NAME", "AVALON_APP_NAME"]), ): @@ -486,18 +480,37 @@ def inject_ayon_environment(deadlinePlugin): break add_kwargs[key] = value - if job.GetJobEnvironmentKeyValue("IS_TEST"): - args.append("--automatic-tests") - - if all(add_kwargs.values()): - for key, value in add_kwargs.items(): - args.extend(["--{}".format(key), value]) - else: + if not all(add_kwargs.values()): raise RuntimeError(( "Missing required env vars: AYON_PROJECT_NAME," " AYON_FOLDER_PATH, AYON_TASK_NAME, AYON_APP_NAME" )) + # Use applications addon arguments + # TODO validate if applications addon should be used + args = [ + "--headless", + "addon", + "applications", + "extractenvironments", + export_url + ] + # Backwards compatibility for older versions + legacy_args = [ + "--headless", + "extractenvironments", + export_url + ] + if job.GetJobEnvironmentKeyValue("IS_TEST"): + args.append("--automatic-tests") + + for key, value in add_kwargs.items(): + args.extend(["--{}".format(key), value]) + # Legacy arguments expect '--asset' instead of '--folder' + if key == "folder": + key = "asset" + legacy_args.extend(["--{}".format(key), value]) + environment = { "AYON_SERVER_URL": ayon_server_url, "AYON_API_KEY": ayon_api_key, @@ -516,9 +529,18 @@ def inject_ayon_environment(deadlinePlugin): ) if process_exitcode != 0: - raise RuntimeError( - "Failed to run Ayon process to extract environments." + print( + "Failed to run AYON process to extract environments. Trying" + " to use legacy arguments." ) + legacy_args_str = subprocess.list2cmdline(legacy_args) + process_exitcode = deadlinePlugin.RunProcess( + exe, legacy_args_str, os.path.dirname(exe), -1 + ) + if process_exitcode != 0: + raise RuntimeError( + "Failed to run AYON process to extract environments." + ) print(">>> Loading file ...") with open(export_url) as fp: @@ -634,4 +656,4 @@ def __main__(deadlinePlugin): if ayon_publish_job == "1": inject_render_job_id(deadlinePlugin) if ayon_render_job == "1" or ayon_remote_job == "1": - inject_ayon_environment(deadlinePlugin) + inject_ayon_environment(deadlinePlugin) \ No newline at end of file From 9d910e69e4ea8480403a13bcd65e0b83146290da Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:04:59 +0100 Subject: [PATCH 14/27] bump minor version of global job pre load --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 5f40470f96..53e29f215f 100644 --- a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -13,7 +13,7 @@ from Deadline.Scripting import ( FileUtils, DirectoryUtils, ) -__version__ = "1.0.1" +__version__ = "1.1.0" VERSION_REGEX = re.compile( r"(?P0|[1-9]\d*)" r"\.(?P0|[1-9]\d*)" From fbebf98d9777a7f110c0bf3f39f01aba6bcc41e7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:20:21 +0100 Subject: [PATCH 15/27] add new line at the end of file --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index 53e29f215f..ac04407f5b 100644 --- a/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/client/ayon_core/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -656,4 +656,4 @@ def __main__(deadlinePlugin): if ayon_publish_job == "1": inject_render_job_id(deadlinePlugin) if ayon_render_job == "1" or ayon_remote_job == "1": - inject_ayon_environment(deadlinePlugin) \ No newline at end of file + inject_ayon_environment(deadlinePlugin) From 4af3e1cc3be50d617572d2fed2d87c53e9bbe175 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:20:41 +0100 Subject: [PATCH 16/27] global collect host name is not dependent on application addon --- .../plugins/publish/collect_host_name.py | 31 +++---------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/client/ayon_core/plugins/publish/collect_host_name.py b/client/ayon_core/plugins/publish/collect_host_name.py index e76579bbd2..ea4ec7ad41 100644 --- a/client/ayon_core/plugins/publish/collect_host_name.py +++ b/client/ayon_core/plugins/publish/collect_host_name.py @@ -1,14 +1,13 @@ """ Requires: None + Provides: - context -> host (str) + context -> hostName (str) """ import os import pyblish.api -from ayon_core.lib import ApplicationManager - class CollectHostName(pyblish.api.ContextPlugin): """Collect avalon host name to context.""" @@ -18,30 +17,8 @@ class CollectHostName(pyblish.api.ContextPlugin): def process(self, context): host_name = context.data.get("hostName") - app_name = context.data.get("appName") - app_label = context.data.get("appLabel") - # Don't override value if is already set - if host_name and app_name and app_label: + if host_name: return # Use AYON_HOST_NAME to get host name if available - if not host_name: - host_name = os.environ.get("AYON_HOST_NAME") - - # Use AYON_APP_NAME to get full app name - if not app_name: - app_name = os.environ.get("AYON_APP_NAME") - - # Fill missing values based on app full name - if (not host_name or not app_label) and app_name: - app_manager = ApplicationManager() - app = app_manager.applications.get(app_name) - if app: - if not host_name: - host_name = app.host_name - if not app_label: - app_label = app.full_label - - context.data["hostName"] = host_name - context.data["appName"] = app_name - context.data["appLabel"] = app_label + context.data["hostName"] = os.environ.get("AYON_HOST_NAME") From 0c6344777f6f6fa0920c92abb0dea7597a8865c0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:23:25 +0100 Subject: [PATCH 17/27] added CollectAppName to applications addon --- .../addons/ayon_applications/__init__.py | 7 +-- .../addons/ayon_applications/addon.py | 12 ++++- .../addons/ayon_applications/constants.py | 4 ++ .../plugins/publish/collect_app_name.py | 48 +++++++++++++++++++ 4 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py diff --git a/client/ayon_core/addons/ayon_applications/__init__.py b/client/ayon_core/addons/ayon_applications/__init__.py index 87ad705cda..b4a50279ab 100644 --- a/client/ayon_core/addons/ayon_applications/__init__.py +++ b/client/ayon_core/addons/ayon_applications/__init__.py @@ -1,5 +1,5 @@ -from .addon import ApplicationsAddon from .constants import ( + APPLICATIONS_ADDON_ROOT, DEFAULT_ENV_SUBGROUP, PLATFORM_NAMES, ) @@ -27,11 +27,10 @@ from .manager import ( ApplicationManager, ApplicationLaunchContext, ) +from .addon import ApplicationsAddon __all__ = ( - "ApplicationsAddon", - "DEFAULT_ENV_SUBGROUP", "PLATFORM_NAMES", @@ -54,4 +53,6 @@ __all__ = ( "ApplicationManager", "ApplicationLaunchContext", + + "ApplicationsAddon", ) diff --git a/client/ayon_core/addons/ayon_applications/addon.py b/client/ayon_core/addons/ayon_applications/addon.py index 9c4a5a392e..1db221fdf6 100644 --- a/client/ayon_core/addons/ayon_applications/addon.py +++ b/client/ayon_core/addons/ayon_applications/addon.py @@ -1,13 +1,14 @@ import os import json -from ayon_core.addon import AYONAddon, click_wrap +from ayon_core.addon import AYONAddon, IPluginPaths, click_wrap +from .constants import APPLICATIONS_ADDON_ROOT from .defs import LaunchTypes from .manager import ApplicationManager -class ApplicationsAddon(AYONAddon): +class ApplicationsAddon(AYONAddon, IPluginPaths): name = "applications" def get_app_environments_for_context( @@ -98,6 +99,13 @@ class ApplicationsAddon(AYONAddon): """ return ApplicationManager(settings) + def get_plugin_paths(self): + return { + "publish": [ + os.path.join(APPLICATIONS_ADDON_ROOT, "plugins", "publish") + ] + } + # --- CLI --- def cli(self, addon_click_group): main_group = click_wrap.group( diff --git a/client/ayon_core/addons/ayon_applications/constants.py b/client/ayon_core/addons/ayon_applications/constants.py index 03112ee422..92c8f4f254 100644 --- a/client/ayon_core/addons/ayon_applications/constants.py +++ b/client/ayon_core/addons/ayon_applications/constants.py @@ -1,2 +1,6 @@ +import os + +APPLICATIONS_ADDON_ROOT = os.path.dirname(os.path.abspath(__file__)) + PLATFORM_NAMES = {"windows", "linux", "darwin"} DEFAULT_ENV_SUBGROUP = "standard" diff --git a/client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py b/client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py new file mode 100644 index 0000000000..f54a551cda --- /dev/null +++ b/client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py @@ -0,0 +1,48 @@ +""" +Run after global plugin 'CollectHostName' in ayon_core. + +Requires: + None + +Provides: + context -> hostName (str) + context -> appName (str) + context -> appLabel (str) +""" +import os +import pyblish.api + +from ayon_applications import ApplicationManager + + +class CollectAppName(pyblish.api.ContextPlugin): + """Collect avalon host name to context.""" + + label = "Collect App Name" + order = pyblish.api.CollectorOrder - 0.499999 + + def process(self, context): + host_name = context.data.get("hostName") + app_name = context.data.get("appName") + app_label = context.data.get("appLabel") + # Don't override value if is already set + if host_name and app_name and app_label: + return + + # Use AYON_APP_NAME to get full app name + if not app_name: + app_name = os.environ.get("AYON_APP_NAME") + + # Fill missing values based on app full name + if (not host_name or not app_label) and app_name: + app_manager = ApplicationManager() + app = app_manager.applications.get(app_name) + if app: + if not host_name: + host_name = app.host_name + if not app_label: + app_label = app.full_label + + context.data["hostName"] = host_name + context.data["appName"] = app_name + context.data["appLabel"] = app_label From 33702d262c904fe95bf72a8c2ac7798c526a7247 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:23:37 +0100 Subject: [PATCH 18/27] fix import in job queue module --- client/ayon_core/modules/job_queue/addon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/modules/job_queue/addon.py b/client/ayon_core/modules/job_queue/addon.py index 32d06d0040..0fa54eb2f0 100644 --- a/client/ayon_core/modules/job_queue/addon.py +++ b/client/ayon_core/modules/job_queue/addon.py @@ -168,7 +168,7 @@ class JobQueueAddon(AYONAddon): @classmethod def start_worker(cls, app_name, server_url=None): import requests - from ayon_core.lib import ApplicationManager + from ayon_applications import ApplicationManager if not server_url: server_url = cls.get_server_url_from_settings() From e7021fa424bf59e5219f3d6ca42d1a1b214fc05e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:23:53 +0100 Subject: [PATCH 19/27] fix import in launcher model --- client/ayon_core/tools/launcher/models/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/tools/launcher/models/actions.py b/client/ayon_core/tools/launcher/models/actions.py index be78e89389..88c24b79db 100644 --- a/client/ayon_core/tools/launcher/models/actions.py +++ b/client/ayon_core/tools/launcher/models/actions.py @@ -116,7 +116,7 @@ class ApplicationAction(LauncherAction): def process(self, session, **kwargs): """Process the full Application action""" - from ayon_core.lib import ( + from ayon_applications import ( ApplicationExecutableNotFound, ApplicationLaunchFailed, ) From c32a43f012d94b66bfc7d47dff6d8a0105467143 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:24:46 +0100 Subject: [PATCH 20/27] removed open djv load plugin --- client/ayon_core/plugins/load/open_djv.py | 64 ----------------------- 1 file changed, 64 deletions(-) delete mode 100644 client/ayon_core/plugins/load/open_djv.py diff --git a/client/ayon_core/plugins/load/open_djv.py b/client/ayon_core/plugins/load/open_djv.py deleted file mode 100644 index 30023ac1f5..0000000000 --- a/client/ayon_core/plugins/load/open_djv.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -from ayon_core.lib import ApplicationManager -from ayon_core.pipeline import load - - -def existing_djv_path(): - app_manager = ApplicationManager() - djv_list = [] - - for app_name, app in app_manager.applications.items(): - if 'djv' in app_name and app.find_executable(): - djv_list.append(app_name) - - return djv_list - - -class OpenInDJV(load.LoaderPlugin): - """Open Image Sequence with system default""" - - djv_list = existing_djv_path() - product_types = {"*"} if djv_list else [] - representations = ["*"] - extensions = { - "cin", "dpx", "avi", "dv", "gif", "flv", "mkv", "mov", "mpg", "mpeg", - "mp4", "m4v", "mxf", "iff", "z", "ifl", "jpeg", "jpg", "jfif", "lut", - "1dl", "exr", "pic", "png", "ppm", "pnm", "pgm", "pbm", "rla", "rpf", - "sgi", "rgba", "rgb", "bw", "tga", "tiff", "tif", "img", "h264", - } - - label = "Open in DJV" - order = -10 - icon = "play-circle" - color = "orange" - - def load(self, context, name, namespace, data): - import clique - - path = self.filepath_from_context(context) - directory = os.path.dirname(path) - - pattern = clique.PATTERNS["frames"] - files = os.listdir(directory) - collections, remainder = clique.assemble( - files, - patterns=[pattern], - minimum_items=1 - ) - - if not remainder: - sequence = collections[0] - first_image = list(sequence)[0] - else: - first_image = path - filepath = os.path.normpath(os.path.join(directory, first_image)) - - self.log.info("Opening : {}".format(filepath)) - - last_djv_version = sorted(self.djv_list)[-1] - - app_manager = ApplicationManager() - djv = app_manager.applications.get(last_djv_version) - djv.arguments.append(filepath) - - app_manager.launch(last_djv_version) From 8902e18838742501b562f6d8ba1eba0cc4ba6c0d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 11:31:31 +0100 Subject: [PATCH 21/27] modified royalrender to support both options of arguments --- client/ayon_core/modules/royalrender/lib.py | 34 +++++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/modules/royalrender/lib.py b/client/ayon_core/modules/royalrender/lib.py index 5392803710..82bc96e759 100644 --- a/client/ayon_core/modules/royalrender/lib.py +++ b/client/ayon_core/modules/royalrender/lib.py @@ -308,31 +308,45 @@ class BaseCreateRoyalRenderJob(pyblish.api.InstancePlugin, export_url = os.path.join(tempfile.gettempdir(), temp_file_name) print(">>> Temporary path: {}".format(export_url)) - args = [ - "--headless", - "extractenvironments", - export_url - ] - anatomy_data = instance.context.data["anatomyData"] + addons_manager = instance.context.data["ayonAddonsManager"] + applications_addon = addons_manager.get_enabled_addon("applications") + + folder_key = "folder" + if applications_addon is None: + # Use 'asset' when applications addon command is not used + folder_key = "asset" add_kwargs = { "project": anatomy_data["project"]["name"], - "asset": instance.context.data["folderPath"], + folder_key: instance.context.data["folderPath"], "task": anatomy_data["task"]["name"], "app": instance.context.data.get("appName"), "envgroup": "farm" } - if os.getenv('IS_TEST'): - args.append("--automatic-tests") - if not all(add_kwargs.values()): raise RuntimeError(( "Missing required env vars: AYON_PROJECT_NAME, AYON_FOLDER_PATH," " AYON_TASK_NAME, AYON_APP_NAME" )) + args = ["--headless"] + # Use applications addon to extract environments + # NOTE this is for backwards compatibility, the global command + # will be removed in future and only applications addon command + # should be used. + if applications_addon is not None: + args.extend(["addon", "applications"]) + + args.extend([ + "extractenvironments", + export_url + ]) + + if os.getenv('IS_TEST'): + args.append("--automatic-tests") + for key, value in add_kwargs.items(): args.extend([f"--{key}", value]) self.log.debug("Executing: {}".format(" ".join(args))) From 4e31241c172ddbcc050844c6cb144e1319264504 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 12:00:57 +0100 Subject: [PATCH 22/27] moved the addon into subfolder by addon name --- client/ayon_core/addon/base.py | 73 ++++++++++++------- .../ayon_applications/__init__.py | 0 .../ayon_applications/addon.py | 0 .../ayon_applications/constants.py | 0 .../ayon_applications/defs.py | 0 .../ayon_applications/exceptions.py | 0 .../ayon_applications/hooks.py | 0 .../ayon_applications/manager.py | 0 .../plugins/publish/collect_app_name.py | 0 .../ayon_applications/utils.py | 0 10 files changed, 46 insertions(+), 27 deletions(-) rename client/ayon_core/addons/{ => applications}/ayon_applications/__init__.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/addon.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/constants.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/defs.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/exceptions.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/hooks.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/manager.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/plugins/publish/collect_app_name.py (100%) rename client/ayon_core/addons/{ => applications}/ayon_applications/utils.py (100%) diff --git a/client/ayon_core/addon/base.py b/client/ayon_core/addon/base.py index da02691275..cc27f2ce37 100644 --- a/client/ayon_core/addon/base.py +++ b/client/ayon_core/addon/base.py @@ -336,45 +336,64 @@ def _load_ayon_addons(openpype_modules, modules_key, log): return addons_to_skip_in_core -def _load_ayon_addons_dir(openpype_modules, modules_key, log): +def _load_ayon_core_addons_dir( + ignore_addon_names, openpype_modules, modules_key, log +): addons_dir = os.path.join(AYON_CORE_ROOT, "addons") if not os.path.exists(addons_dir): return - while addons_dir in sys.path: - sys.path.remove(addons_dir) - sys.path.insert(0, addons_dir) - imported_modules = [] - for name in os.listdir(addons_dir): - fullpath = os.path.join(addons_dir, name) - basename = os.path.splitext(name)[0] - try: - module = __import__(basename, fromlist=("",)) - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - inspect.isclass(attr) - and issubclass(attr, AYONAddon) - ): - new_import_str = "{}.{}".format(modules_key, basename) - sys.modules[new_import_str] = module - setattr(openpype_modules, basename, module) - imported_modules.append(module) - break - except Exception: - log.error( - "Failed to import addon '{}'.".format(fullpath), - exc_info=True - ) + filtered_paths = [] + for name in os.listdir(addons_dir): + if name in ignore_addon_names: + continue + path = os.path.join(addons_dir, name) + if os.path.isdir(path): + filtered_paths.append(path) + + for path in filtered_paths: + while path in sys.path: + sys.path.remove(path) + sys.path.insert(0, path) + + for name in os.listdir(path): + fullpath = os.path.join(path, name) + if os.path.isfile(fullpath): + basename, ext = os.path.splitext(name) + if ext != ".py": + continue + else: + basename = name + try: + module = __import__(basename, fromlist=("",)) + for attr_name in dir(module): + attr = getattr(module, attr_name) + if ( + inspect.isclass(attr) + and issubclass(attr, AYONAddon) + ): + new_import_str = "{}.{}".format(modules_key, basename) + sys.modules[new_import_str] = module + setattr(openpype_modules, basename, module) + imported_modules.append(module) + break + + except Exception: + log.error( + "Failed to import addon '{}'.".format(fullpath), + exc_info=True + ) return imported_modules def _load_addons_in_core( ignore_addon_names, openpype_modules, modules_key, log ): - _load_ayon_addons_dir(openpype_modules, modules_key, log) + _load_ayon_core_addons_dir( + ignore_addon_names, openpype_modules, modules_key, log + ) # Add current directory at first place # - has small differences in import logic hosts_dir = os.path.join(AYON_CORE_ROOT, "hosts") diff --git a/client/ayon_core/addons/ayon_applications/__init__.py b/client/ayon_core/addons/applications/ayon_applications/__init__.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/__init__.py rename to client/ayon_core/addons/applications/ayon_applications/__init__.py diff --git a/client/ayon_core/addons/ayon_applications/addon.py b/client/ayon_core/addons/applications/ayon_applications/addon.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/addon.py rename to client/ayon_core/addons/applications/ayon_applications/addon.py diff --git a/client/ayon_core/addons/ayon_applications/constants.py b/client/ayon_core/addons/applications/ayon_applications/constants.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/constants.py rename to client/ayon_core/addons/applications/ayon_applications/constants.py diff --git a/client/ayon_core/addons/ayon_applications/defs.py b/client/ayon_core/addons/applications/ayon_applications/defs.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/defs.py rename to client/ayon_core/addons/applications/ayon_applications/defs.py diff --git a/client/ayon_core/addons/ayon_applications/exceptions.py b/client/ayon_core/addons/applications/ayon_applications/exceptions.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/exceptions.py rename to client/ayon_core/addons/applications/ayon_applications/exceptions.py diff --git a/client/ayon_core/addons/ayon_applications/hooks.py b/client/ayon_core/addons/applications/ayon_applications/hooks.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/hooks.py rename to client/ayon_core/addons/applications/ayon_applications/hooks.py diff --git a/client/ayon_core/addons/ayon_applications/manager.py b/client/ayon_core/addons/applications/ayon_applications/manager.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/manager.py rename to client/ayon_core/addons/applications/ayon_applications/manager.py diff --git a/client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py b/client/ayon_core/addons/applications/ayon_applications/plugins/publish/collect_app_name.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/plugins/publish/collect_app_name.py rename to client/ayon_core/addons/applications/ayon_applications/plugins/publish/collect_app_name.py diff --git a/client/ayon_core/addons/ayon_applications/utils.py b/client/ayon_core/addons/applications/ayon_applications/utils.py similarity index 100% rename from client/ayon_core/addons/ayon_applications/utils.py rename to client/ayon_core/addons/applications/ayon_applications/utils.py From 1bacea2d6562f0543d3571ec6c1b5c0ca74f5c35 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 28 Mar 2024 12:01:51 +0100 Subject: [PATCH 23/27] added comment --- client/ayon_core/addon/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/ayon_core/addon/base.py b/client/ayon_core/addon/base.py index cc27f2ce37..6ef838652e 100644 --- a/client/ayon_core/addon/base.py +++ b/client/ayon_core/addon/base.py @@ -345,6 +345,8 @@ def _load_ayon_core_addons_dir( imported_modules = [] + # Make sure that addons which already have client code are not loaded + # from core again, with older code filtered_paths = [] for name in os.listdir(addons_dir): if name in ignore_addon_names: From 43ffe79808ecfca02f6301d1a01fbe9a9a9d03a4 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Thu, 28 Mar 2024 16:19:08 +0100 Subject: [PATCH 24/27] Fix import refactor + update method arguments --- .../ayon_core/hosts/houdini/plugins/load/load_filepath.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/hosts/houdini/plugins/load/load_filepath.py b/client/ayon_core/hosts/houdini/plugins/load/load_filepath.py index 515ffa6027..f107190f96 100644 --- a/client/ayon_core/hosts/houdini/plugins/load/load_filepath.py +++ b/client/ayon_core/hosts/houdini/plugins/load/load_filepath.py @@ -2,7 +2,7 @@ import os import re from ayon_core.pipeline import load -from openpype.hosts.houdini.api import pipeline +from ayon_core.hosts.houdini.api import pipeline import hou @@ -103,8 +103,8 @@ class FilePathLoader(load.LoaderPlugin): parm) node.setParmTemplateGroup(parm_template_group) - def switch(self, container, representation): - self.update(container, representation) + def switch(self, container, context): + self.update(container, context) def remove(self, container): From 95f828ba02914905c1204b7c2bc396c82cd67b11 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Mon, 1 Apr 2024 12:26:40 +0100 Subject: [PATCH 25/27] Use is_active for proper optional functionality --- client/ayon_core/plugins/publish/integrate_hero_version.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/client/ayon_core/plugins/publish/integrate_hero_version.py b/client/ayon_core/plugins/publish/integrate_hero_version.py index 7969457697..8c36719b77 100644 --- a/client/ayon_core/plugins/publish/integrate_hero_version.py +++ b/client/ayon_core/plugins/publish/integrate_hero_version.py @@ -90,6 +90,9 @@ class IntegrateHeroVersion( # *but all other plugins must be sucessfully completed def process(self, instance): + if not self.is_active(instance.data): + return + self.log.debug( "--- Integration of Hero version for product `{}` begins.".format( instance.data["productName"] From ec72877f2193ff78656cadb47b6aa0fb844917a0 Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Tue, 2 Apr 2024 13:04:05 +0200 Subject: [PATCH 26/27] make code-spell and ruff happy about houdini.api.lib --- client/ayon_core/hosts/houdini/api/lib.py | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/hosts/houdini/api/lib.py b/client/ayon_core/hosts/houdini/api/lib.py index a72118c276..99cd205616 100644 --- a/client/ayon_core/hosts/houdini/api/lib.py +++ b/client/ayon_core/hosts/houdini/api/lib.py @@ -948,7 +948,7 @@ def self_publish(): Firstly, it gets the node and its dependencies. Then, it deactivates all other ROPs - And finaly, it triggers the publishing action. + And finally, it triggers the publishing action. """ result, comment = hou.ui.readInput( @@ -1076,4 +1076,4 @@ def prompt_reset_context(): if options["instances"]: update_content_on_context_change() - dialog.deleteLater() \ No newline at end of file + dialog.deleteLater() diff --git a/pyproject.toml b/pyproject.toml index ee124ddc2d..bfdc3cca8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ line-ending = "auto" [tool.codespell] # Ignore words that are not in the dictionary. -ignore-words-list = "ayon,ynput" +ignore-words-list = "ayon,ynput,parms,parm,hda" skip = "./.*,./package/*,*/vendor/*,*/unreal/integration/*,*/aftereffects/api/extension/js/libs/*" count = true quiet-level = 3 From 8b07bcedbc634bd45aa69d15acfd443cc7d1331e Mon Sep 17 00:00:00 2001 From: MustafaJafar Date: Tue, 2 Apr 2024 13:04:42 +0200 Subject: [PATCH 27/27] ignore inputs when rendering --- client/ayon_core/hosts/houdini/api/lib.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/hosts/houdini/api/lib.py b/client/ayon_core/hosts/houdini/api/lib.py index 99cd205616..2755b646fa 100644 --- a/client/ayon_core/hosts/houdini/api/lib.py +++ b/client/ayon_core/hosts/houdini/api/lib.py @@ -243,7 +243,10 @@ def render_rop(ropnode): try: ropnode.render(verbose=verbose, # Allow Deadline to capture completion percentage - output_progress=verbose) + output_progress=verbose, + # Render only this node + # (do not render any of its dependencies) + ignore_inputs=True) except hou.Error as exc: # The hou.Error is not inherited from a Python Exception class, # so we explicitly capture the houdini error, otherwise pyblish