From e2565dc2d205e45e939d2ecef05933928520632a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 10 Feb 2023 17:12:19 +0100 Subject: [PATCH 01/57] Nuke: adding solution for originalBasename frame temlate formating https://github.com/ynput/OpenPype/pull/4452#issuecomment-1426020567 --- openpype/hosts/nuke/plugins/load/load_clip.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/nuke/plugins/load/load_clip.py b/openpype/hosts/nuke/plugins/load/load_clip.py index 565d777811..f9364172ea 100644 --- a/openpype/hosts/nuke/plugins/load/load_clip.py +++ b/openpype/hosts/nuke/plugins/load/load_clip.py @@ -220,8 +220,21 @@ class LoadClip(plugin.NukeLoader): dict: altered representation data """ representation = deepcopy(representation) - frame = representation["context"]["frame"] - representation["context"]["frame"] = "#" * len(str(frame)) + context = representation["context"] + template = representation["data"]["template"] + frame = context["frame"] + hashed_frame = "#" * len(str(frame)) + + if ( + "{originalBasename}" in template + and "frame" in context + ): + origin_basename = context["originalBasename"] + context["originalBasename"] = origin_basename.replace( + frame, hashed_frame + ) + + representation["context"]["frame"] = hashed_frame return representation def update(self, container, representation): From 8233b1ad8229ebc8388d453f08c3cdeec3196a9f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 10 Feb 2023 17:51:29 +0100 Subject: [PATCH 02/57] publishing files with fixed versionData to fit originalBasename tempate --- .../publish/collect_otio_subset_resources.py | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index e72c12d9a9..537aef683c 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -14,16 +14,41 @@ from openpype.pipeline.editorial import ( range_from_frames, make_sequence_collection ) - +from openpype.pipeline.publish import ( + get_publish_template_name +) class CollectOtioSubsetResources(pyblish.api.InstancePlugin): """Get Resources for a subset version""" label = "Collect OTIO Subset Resources" - order = pyblish.api.CollectorOrder - 0.077 + order = pyblish.api.CollectorOrder + 0.0021 families = ["clip"] hosts = ["resolve", "hiero", "flame"] + def get_template_name(self, instance): + """Return anatomy template name to use for integration""" + + # Anatomy data is pre-filled by Collectors + context = instance.context + project_name = context.data["projectName"] + + # Task can be optional in anatomy data + host_name = context.data["hostName"] + family = instance.data["family"] + anatomy_data = instance.context.data["anatomyData"] + task_info = anatomy_data.get("task") or {} + + return get_publish_template_name( + project_name, + host_name, + family, + task_name=task_info.get("name"), + task_type=task_info.get("type"), + project_settings=context.data["project_settings"], + logger=self.log + ) + def process(self, instance): if "audio" in instance.data["family"]: @@ -35,6 +60,13 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): if not instance.data.get("versionData"): instance.data["versionData"] = {} + template_name = self.get_template_name(instance) + anatomy = instance.context.data["anatomy"] + publish_template_category = anatomy.templates[template_name] + template = os.path.normpath(publish_template_category["path"]) + self.log.debug( + ">> template: {}".format(template)) + handle_start = instance.data["handleStart"] handle_end = instance.data["handleEnd"] @@ -84,6 +116,10 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): frame_start = instance.data["frameStart"] frame_end = frame_start + (media_out - media_in) + if "{originalDirname}" in template: + frame_start = media_in + frame_end = media_out + # add to version data start and end range data # for loader plugins to be correctly displayed and loaded instance.data["versionData"].update({ @@ -153,7 +189,6 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): repre = self._create_representation( frame_start, frame_end, collection=collection) - instance.data["originalBasename"] = collection.format("{head}") else: _trim = False dirname, filename = os.path.split(media_ref.target_url) @@ -168,8 +203,6 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): repre = self._create_representation( frame_start, frame_end, file=filename, trim=_trim) - instance.data["originalBasename"] = os.path.splitext(filename)[0] - instance.data["originalDirname"] = self.staging_dir if repre: From a1ab32b32961a7785e97fdd74024fd7fb7f261a6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 10 Feb 2023 17:53:36 +0100 Subject: [PATCH 03/57] polishing fixes --- .../publish/collect_otio_subset_resources.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index 537aef683c..5daa1b40fe 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -26,28 +26,6 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): families = ["clip"] hosts = ["resolve", "hiero", "flame"] - def get_template_name(self, instance): - """Return anatomy template name to use for integration""" - - # Anatomy data is pre-filled by Collectors - context = instance.context - project_name = context.data["projectName"] - - # Task can be optional in anatomy data - host_name = context.data["hostName"] - family = instance.data["family"] - anatomy_data = instance.context.data["anatomyData"] - task_info = anatomy_data.get("task") or {} - - return get_publish_template_name( - project_name, - host_name, - family, - task_name=task_info.get("name"), - task_type=task_info.get("type"), - project_settings=context.data["project_settings"], - logger=self.log - ) def process(self, instance): @@ -116,6 +94,7 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): frame_start = instance.data["frameStart"] frame_end = frame_start + (media_out - media_in) + # Fit start /end frame to media in /out if "{originalDirname}" in template: frame_start = media_in frame_end = media_out @@ -258,3 +237,26 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): if kwargs.get("trim") is True: representation_data["tags"] = ["trim"] return representation_data + + def get_template_name(self, instance): + """Return anatomy template name to use for integration""" + + # Anatomy data is pre-filled by Collectors + context = instance.context + project_name = context.data["projectName"] + + # Task can be optional in anatomy data + host_name = context.data["hostName"] + family = instance.data["family"] + anatomy_data = instance.context.data["anatomyData"] + task_info = anatomy_data.get("task") or {} + + return get_publish_template_name( + project_name, + host_name, + family, + task_name=task_info.get("name"), + task_type=task_info.get("type"), + project_settings=context.data["project_settings"], + logger=self.log + ) \ No newline at end of file From 6fbf7be560cc68704a2ed2933955cac10a80176e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 10 Feb 2023 17:54:54 +0100 Subject: [PATCH 04/57] wrong template key name --- openpype/plugins/publish/collect_otio_subset_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index 5daa1b40fe..dab52986da 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -95,7 +95,7 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): frame_end = frame_start + (media_out - media_in) # Fit start /end frame to media in /out - if "{originalDirname}" in template: + if "{originalBasename}" in template: frame_start = media_in frame_end = media_out From 6653df6369afbe53905af5eccacd29a2faed72d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Tue, 14 Feb 2023 12:18:00 +0100 Subject: [PATCH 05/57] Update openpype/hosts/nuke/plugins/load/load_clip.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/hosts/nuke/plugins/load/load_clip.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openpype/hosts/nuke/plugins/load/load_clip.py b/openpype/hosts/nuke/plugins/load/load_clip.py index f9364172ea..8f9b463037 100644 --- a/openpype/hosts/nuke/plugins/load/load_clip.py +++ b/openpype/hosts/nuke/plugins/load/load_clip.py @@ -222,13 +222,12 @@ class LoadClip(plugin.NukeLoader): representation = deepcopy(representation) context = representation["context"] template = representation["data"]["template"] - frame = context["frame"] - hashed_frame = "#" * len(str(frame)) - if ( "{originalBasename}" in template and "frame" in context ): + frame = context["frame"] + hashed_frame = "#" * len(str(frame)) origin_basename = context["originalBasename"] context["originalBasename"] = origin_basename.replace( frame, hashed_frame From 1abe920b5f18e32e51dbdaa8ac60b6dff175e22f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 14 Feb 2023 14:27:42 +0100 Subject: [PATCH 06/57] anatomy data from instance rather then context --- .../plugins/publish/collect_otio_subset_resources.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openpype/plugins/publish/collect_otio_subset_resources.py b/openpype/plugins/publish/collect_otio_subset_resources.py index dab52986da..f659791d95 100644 --- a/openpype/plugins/publish/collect_otio_subset_resources.py +++ b/openpype/plugins/publish/collect_otio_subset_resources.py @@ -22,7 +22,7 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): """Get Resources for a subset version""" label = "Collect OTIO Subset Resources" - order = pyblish.api.CollectorOrder + 0.0021 + order = pyblish.api.CollectorOrder + 0.491 families = ["clip"] hosts = ["resolve", "hiero", "flame"] @@ -50,9 +50,9 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): # get basic variables otio_clip = instance.data["otioClip"] - otio_avalable_range = otio_clip.available_range() - media_fps = otio_avalable_range.start_time.rate - available_duration = otio_avalable_range.duration.value + otio_available_range = otio_clip.available_range() + media_fps = otio_available_range.start_time.rate + available_duration = otio_available_range.duration.value # get available range trimmed with processed retimes retimed_attributes = get_media_range_with_retimes( @@ -248,7 +248,7 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): # Task can be optional in anatomy data host_name = context.data["hostName"] family = instance.data["family"] - anatomy_data = instance.context.data["anatomyData"] + anatomy_data = instance.data["anatomyData"] task_info = anatomy_data.get("task") or {} return get_publish_template_name( @@ -259,4 +259,4 @@ class CollectOtioSubsetResources(pyblish.api.InstancePlugin): task_type=task_info.get("type"), project_settings=context.data["project_settings"], logger=self.log - ) \ No newline at end of file + ) From 24715ff2de8e8c673670643ebb9263f3bd7219e3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 22 Feb 2023 12:34:45 +0100 Subject: [PATCH 07/57] global: template should not have frame or other optional elements in tempate, since they are already part of the `originalBasename` --- openpype/settings/defaults/project_anatomy/templates.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/settings/defaults/project_anatomy/templates.json b/openpype/settings/defaults/project_anatomy/templates.json index 32230e0625..02c0e35377 100644 --- a/openpype/settings/defaults/project_anatomy/templates.json +++ b/openpype/settings/defaults/project_anatomy/templates.json @@ -55,7 +55,7 @@ }, "source": { "folder": "{root[work]}/{originalDirname}", - "file": "{originalBasename}<.{@frame}><_{udim}>.{ext}", + "file": "{originalBasename}.{ext}", "path": "{@folder}/{@file}" }, "__dynamic_keys_labels__": { @@ -66,4 +66,4 @@ "source": "source" } } -} \ No newline at end of file +} From d81debdc1050a008597515b1655dfe341566eba5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 16:18:14 +0100 Subject: [PATCH 08/57] implemented methods to get active site type from sync server module --- .../modules/sync_server/sync_server_module.py | 101 +++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index ba0abe7d3b..1c2e820031 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -3,7 +3,6 @@ import sys import time from datetime import datetime import threading -import platform import copy import signal from collections import deque, defaultdict @@ -25,7 +24,11 @@ from openpype.lib import Logger, get_local_site_id from openpype.pipeline import AvalonMongoDB, Anatomy from openpype.settings.lib import ( get_default_anatomy_settings, - get_anatomy_settings + get_anatomy_settings, + get_local_settings, +) +from openpype.settings.constants import ( + DEFAULT_PROJECT_KEY ) from .providers.local_drive import LocalDriveHandler @@ -639,6 +642,100 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return get_local_site_id() return active_site + def get_active_site_type(self, project_name, local_settings=None): + """Active site which is defined by artist. + + Unlike 'get_active_site' is this method also checking local settings + where might be different active site set by user. The output is limited + to "studio" and "local". + + This method is used by Anatomy when is decided which + + Todos: + Check if sync server is enabled for the project. + - To be able to do that the sync settings MUST NOT be cached for + all projects at once. The sync settings preparation for all + projects is reasonable only in sync server loop. + + Args: + project_name (str): Name of project where to look for active site. + local_settings (Optional[dict[str, Any]]): Prepared local settings. + + Returns: + Literal["studio", "local"]: Active site. + """ + + if not self.enabled: + return "studio" + + if local_settings is None: + local_settings = get_local_settings() + + local_project_settings = local_settings.get("projects") + project_settings = get_project_settings(project_name) + sync_server_settings = project_settings["global"]["sync_server"] + if not sync_server_settings["enabled"]: + return "studio" + + project_active_site = sync_server_settings["config"]["active_site"] + if not local_project_settings: + return project_active_site + + project_locals = local_project_settings.get(project_name) or {} + default_locals = local_project_settings.get(DEFAULT_PROJECT_KEY) or {} + active_site = ( + project_locals.get("active_site") + or default_locals.get("active_site") + ) + if active_site: + return active_site + return project_active_site + + def get_local_site_root_overrides( + self, project_name, site_name, local_settings=None + ): + """Get root overrides for project on a site. + + Implemented to be used in 'Anatomy' for other than 'studio' site. + + Args: + project_name (str): Project for which root overrides should be + received. + site_name (str): Name of site for which should be received roots. + local_settings (Optional[dict[str, Any]]): Prepare local settigns + values. + + Returns: + Union[dict[str, Any], None]: Root overrides for this machine. + """ + + if local_settings is None: + local_settings = get_local_settings() + + if not local_settings: + return + + local_project_settings = local_settings.get("projects") or {} + + # Check for roots existence in local settings first + roots_project_locals = ( + local_project_settings + .get(project_name, {}) + ) + roots_default_locals = ( + local_project_settings + .get(DEFAULT_PROJECT_KEY, {}) + ) + + # Skip rest of processing if roots are not set + if not roots_project_locals and not roots_default_locals: + return + + # Combine roots from local settings + roots_locals = roots_default_locals.get(site_name) or {} + roots_locals.update(roots_project_locals.get(site_name) or {}) + return roots_locals + # remote sites def get_remote_sites(self, project_name): """ From ded3933a0c461f734a07c86e3495479fb64f4573 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 16:22:29 +0100 Subject: [PATCH 09/57] Base anatomy expect only root overrides --- openpype/pipeline/anatomy.py | 63 +++++++----------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index 49d86d69d6..0768167885 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -57,20 +57,13 @@ class BaseAnatomy(object): root_key_regex = re.compile(r"{(root?[^}]+)}") root_name_regex = re.compile(r"root\[([^]]+)\]") - def __init__(self, project_doc, local_settings, site_name): + def __init__(self, project_doc, root_overrides=None): project_name = project_doc["name"] self.project_name = project_name self.project_code = project_doc["data"]["code"] - if (site_name and - site_name not in ["studio", "local", get_local_site_id()]): - raise RuntimeError("Anatomy could be created only for default " - "local sites not for {}".format(site_name)) - - self._site_name = site_name - self._data = self._prepare_anatomy_data( - project_doc, local_settings, site_name + project_doc, root_overrides ) self._templates_obj = AnatomyTemplates(self) self._roots_obj = Roots(self) @@ -92,28 +85,18 @@ class BaseAnatomy(object): def items(self): return copy.deepcopy(self._data).items() - def _prepare_anatomy_data(self, project_doc, local_settings, site_name): + def _prepare_anatomy_data(self, project_doc, root_overrides): """Prepare anatomy data for further processing. Method added to replace `{task}` with `{task[name]}` in templates. """ - project_name = project_doc["name"] + anatomy_data = self._project_doc_to_anatomy_data(project_doc) - templates_data = anatomy_data.get("templates") - if templates_data: - # Replace `{task}` with `{task[name]}` in templates - value_queue = collections.deque() - value_queue.append(templates_data) - while value_queue: - item = value_queue.popleft() - if not isinstance(item, dict): - continue - - self._apply_local_settings_on_anatomy_data(anatomy_data, - local_settings, - project_name, - site_name) + self._apply_local_settings_on_anatomy_data( + anatomy_data, + root_overrides + ) return anatomy_data @@ -347,7 +330,7 @@ class BaseAnatomy(object): return output def _apply_local_settings_on_anatomy_data( - self, anatomy_data, local_settings, project_name, site_name + self, anatomy_data, root_overrides ): """Apply local settings on anatomy data. @@ -366,40 +349,18 @@ class BaseAnatomy(object): Args: anatomy_data (dict): Data for anatomy. - local_settings (dict): Data of local settings. - project_name (str): Name of project for which anatomy data are. + root_overrides (dict): Data of local settings. """ - if not local_settings: - return - local_project_settings = local_settings.get("projects") or {} - - # Check for roots existence in local settings first - roots_project_locals = ( - local_project_settings - .get(project_name, {}) - ) - roots_default_locals = ( - local_project_settings - .get(DEFAULT_PROJECT_KEY, {}) - ) - - # Skip rest of processing if roots are not set - if not roots_project_locals and not roots_default_locals: - return - - # Combine roots from local settings - roots_locals = roots_default_locals.get(site_name) or {} - roots_locals.update(roots_project_locals.get(site_name) or {}) # Skip processing if roots for current active site are not available in # local settings - if not roots_locals: + if not root_overrides: return current_platform = platform.system().lower() root_data = anatomy_data["roots"] - for root_name, path in roots_locals.items(): + for root_name, path in root_overrides.items(): if root_name not in root_data: continue anatomy_data["roots"][root_name][current_platform] = ( From c55104afdb868a4b5dea6ec69c5a164250cc51be Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 16:22:53 +0100 Subject: [PATCH 10/57] Use CacheItem for keeping track about caches --- openpype/pipeline/anatomy.py | 57 ++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index 0768167885..a5cb97b60f 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -368,9 +368,62 @@ class BaseAnatomy(object): ) +class CacheItem: + """Helper to cache data. + + Helper does not handle refresh of data and does not mark data as outdated. + Who uses the object should check of outdated state on his own will. + """ + + default_lifetime = 10 + + def __init__(self, lifetime=None): + self._data = None + self._cached = None + self._lifetime = lifetime or self.default_lifetime + + @property + def data(self): + """Cached data/object. + + Returns: + Any: Whatever was cached. + """ + + return self._data + + @property + def is_outdated(self): + """Item has outdated cache. + + Lifetime of cache item expired or was not yet set. + + Returns: + bool: Item is outdated. + """ + + if self._cached is None: + return True + return (time.time() - self._cached) > self._lifetime + + def update_data(self, data): + """Update cache of data. + + Args: + data (Any): Data to cache. + """ + + self._data = data + self._cached = time.time() + + class Anatomy(BaseAnatomy): - _project_cache = {} - _site_cache = {} + _sync_server_addon_cache = CacheItem() + _project_cache = collections.defaultdict(CacheItem) + _default_site_id_cache = collections.defaultdict(CacheItem) + _root_overrides_cache = collections.defaultdict( + lambda: collections.defaultdict(CacheItem) + ) def __init__(self, project_name=None, site_name=None): if not project_name: From 72eac0b31edaed55347f0f047f28eec894417055 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 16:23:28 +0100 Subject: [PATCH 11/57] change how site resolving and it's root overrides work --- openpype/pipeline/anatomy.py | 170 +++++++++++++++++++++++------------ 1 file changed, 114 insertions(+), 56 deletions(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index a5cb97b60f..7f5be18b12 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -9,7 +9,6 @@ import six import time from openpype.settings.lib import ( - get_project_settings, get_local_settings, ) from openpype.settings.constants import ( @@ -25,6 +24,7 @@ from openpype.lib.path_templates import ( ) from openpype.lib.log import Logger from openpype.lib import get_local_site_id +from openpype.modules import ModulesManager log = Logger.get_logger(__name__) @@ -436,73 +436,131 @@ class Anatomy(BaseAnatomy): )) project_doc = self.get_project_doc_from_cache(project_name) - local_settings = get_local_settings() - if not site_name: - site_name = self.get_site_name_from_cache( - project_name, local_settings - ) + root_overrides = self._get_site_root_overrides(project_name, site_name) - super(Anatomy, self).__init__( - project_doc, - local_settings, - site_name - ) + super(Anatomy, self).__init__(project_doc, root_overrides) @classmethod def get_project_doc_from_cache(cls, project_name): - project_cache = cls._project_cache.get(project_name) - if project_cache is not None: - if time.time() - project_cache["start"] > 10: - cls._project_cache.pop(project_name) - project_cache = None - - if project_cache is None: - project_cache = { - "project_doc": get_project(project_name), - "start": time.time() - } - cls._project_cache[project_name] = project_cache - - return copy.deepcopy( - cls._project_cache[project_name]["project_doc"] - ) + project_cache = cls._project_cache[project_name] + if project_cache.is_outdated: + project_cache.update_data(get_project(project_name)) + return copy.deepcopy(project_cache.data) @classmethod - def get_site_name_from_cache(cls, project_name, local_settings): - site_cache = cls._site_cache.get(project_name) - if site_cache is not None: - if time.time() - site_cache["start"] > 10: - cls._site_cache.pop(project_name) - site_cache = None + def get_sync_server_addon(cls): + if cls._sync_server_addon_cache.is_outdated: + manager = ModulesManager() + cls._sync_server_addon_cache.update_data( + manager.enabled_modules.get("sync_server") + ) + return cls._sync_server_addon_cache.data - if site_cache: - return site_cache["site_name"] + @classmethod + def _get_studio_roots_overrides(cls, project_name, local_settings=None): + """This would return 'studio' site override by local settings. - local_project_settings = local_settings.get("projects") + Notes: + This logic handles local overrides of studio site which may be + available even when sync server is not enabled. + Handling of 'studio' and 'local' site was separated as preparation + for AYON development where that will be received from + separated sources. + + Args: + project_name (str): Name of project. + local_settings (Optional[dict[str, Any]]): Prepared local settings. + + Returns: + Union[Dict[str, str], None]): Local root overrides. + """ + + if local_settings is None: + local_settings = get_local_settings() + + local_project_settings = local_settings.get("projects") or {} if not local_project_settings: + return None + + # Check for roots existence in local settings first + roots_project_locals = ( + local_project_settings + .get(project_name, {}) + ) + roots_default_locals = ( + local_project_settings + .get(DEFAULT_PROJECT_KEY, {}) + ) + + # Skip rest of processing if roots are not set + if not roots_project_locals and not roots_default_locals: return - project_locals = local_project_settings.get(project_name) or {} - default_locals = local_project_settings.get(DEFAULT_PROJECT_KEY) or {} - active_site = ( - project_locals.get("active_site") - or default_locals.get("active_site") - ) - if not active_site: - project_settings = get_project_settings(project_name) - active_site = ( - project_settings - ["global"] - ["sync_server"] - ["config"] - ["active_site"] - ) + # Combine roots from local settings + roots_locals = roots_default_locals.get("studio") or {} + roots_locals.update_data(roots_project_locals.get("studio") or {}) + return roots_locals - cls._site_cache[project_name] = { - "site_name": active_site, - "start": time.time() - } - return active_site + + @classmethod + def _get_site_root_overrides(cls, project_name, site_name): + """Get root overrides for site. + + Args: + project_name (str): Project name for which root overrides should be + received. + site_name (Union[str, None]): Name of site for which root overrides + should be returned. + """ + + # Local settings may be used more than once or may not be used at all + # - to avoid slowdowns 'get_local_settings' is not called until it's + # really needed + local_settings = None + local_site_id = get_local_site_id() + + # First check if sync server is available and enabled + sync_server = cls.get_sync_server_addon() + if sync_server is None or not sync_server.enabled: + # QUESTION is ok to force 'studio' when site sync is not enabled? + site_name = "studio" + + elif not site_name: + # Use sync server to receive active site name + project_cache = cls._default_site_id_cache[project_name] + if project_cache.is_outdated: + local_settings = get_local_settings() + project_cache.update_data( + sync_server.get_active_site_type( + project_name, local_settings + ) + ) + site_name = project_cache.data + + elif site_name not in ("studio", "local"): + # Validate that site name is valid + if site_name != local_site_id: + raise RuntimeError(( + "Anatomy could be created only for" + " default local sites not for {}" + ).format(site_name)) + site_name = "local" + + site_cache = cls._root_overrides_cache[project_name][site_name] + if site_cache.is_outdated: + if site_name == "studio": + # Handle studio root overrides without sync server + # - studio root overrides can be done even without sync server + roots_overrides = cls._get_studio_roots_overrides( + project_name, local_settings + ) + else: + # Ask sync server to get roots overrides + roots_overrides = sync_server.get_local_site_root_overrides( + project_name, site_name, local_settings + ) + site_cache.update_data(roots_overrides) + return site_cache.data class AnatomyTemplateUnsolved(TemplateUnsolved): From 43179444397f57cfbeca04803caf503a3635e1a5 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 16:32:59 +0100 Subject: [PATCH 12/57] fix modules access --- openpype/pipeline/anatomy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index 7f5be18b12..bbc696d7f1 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -452,7 +452,7 @@ class Anatomy(BaseAnatomy): if cls._sync_server_addon_cache.is_outdated: manager = ModulesManager() cls._sync_server_addon_cache.update_data( - manager.enabled_modules.get("sync_server") + manager.get_enabled_module("sync_server") ) return cls._sync_server_addon_cache.data From f4c47d41d493050adc40324ab0c32c77efc217ae Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 18:12:34 +0100 Subject: [PATCH 13/57] move validation of site name to sync server --- openpype/modules/sync_server/sync_server_module.py | 10 ++++++++++ openpype/pipeline/anatomy.py | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 1c2e820031..c3e28bc4f2 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -709,6 +709,16 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Union[dict[str, Any], None]: Root overrides for this machine. """ + # Validate that site name is valid + if site_name not in ("studio", "local"): + # Considure local site id as 'local' + if site_name != get_local_site_id(): + raise RuntimeError(( + "Anatomy could be created only for" + " default local sites not for {}" + ).format(site_name)) + site_name = "local" + if local_settings is None: local_settings = get_local_settings() diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index bbc696d7f1..bb365c916e 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -23,7 +23,6 @@ from openpype.lib.path_templates import ( FormatObject, ) from openpype.lib.log import Logger -from openpype.lib import get_local_site_id from openpype.modules import ModulesManager log = Logger.get_logger(__name__) @@ -517,7 +516,6 @@ class Anatomy(BaseAnatomy): # - to avoid slowdowns 'get_local_settings' is not called until it's # really needed local_settings = None - local_site_id = get_local_site_id() # First check if sync server is available and enabled sync_server = cls.get_sync_server_addon() @@ -537,15 +535,6 @@ class Anatomy(BaseAnatomy): ) site_name = project_cache.data - elif site_name not in ("studio", "local"): - # Validate that site name is valid - if site_name != local_site_id: - raise RuntimeError(( - "Anatomy could be created only for" - " default local sites not for {}" - ).format(site_name)) - site_name = "local" - site_cache = cls._root_overrides_cache[project_name][site_name] if site_cache.is_outdated: if site_name == "studio": From 5de35b48e57ac958eadb6cfd387a146e962bc90d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 18:13:14 +0100 Subject: [PATCH 14/57] change exception type and message --- openpype/modules/sync_server/sync_server_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index c3e28bc4f2..7ea8f62d03 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -713,9 +713,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if site_name not in ("studio", "local"): # Considure local site id as 'local' if site_name != get_local_site_id(): - raise RuntimeError(( - "Anatomy could be created only for" - " default local sites not for {}" + raise ValueError(( + "Root overrides are available only for" + " default sites not for \"{}\"" ).format(site_name)) site_name = "local" From 1b5d7a1d385b275d84fcc2bb4ee8d3a71b569f3e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 18:13:37 +0100 Subject: [PATCH 15/57] rename method --- openpype/modules/sync_server/sync_server_module.py | 2 +- openpype/pipeline/anatomy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 7ea8f62d03..28863c091a 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -691,7 +691,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return active_site return project_active_site - def get_local_site_root_overrides( + def get_site_root_overrides( self, project_name, site_name, local_settings=None ): """Get root overrides for project on a site. diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index bb365c916e..d1bda2cf18 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -545,7 +545,7 @@ class Anatomy(BaseAnatomy): ) else: # Ask sync server to get roots overrides - roots_overrides = sync_server.get_local_site_root_overrides( + roots_overrides = sync_server.get_site_root_overrides( project_name, site_name, local_settings ) site_cache.update_data(roots_overrides) From 3408c25f7957805ac5b250c154ef2da464e0662e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 18:24:35 +0100 Subject: [PATCH 16/57] remove doubled empty line --- openpype/pipeline/anatomy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index d1bda2cf18..ac915b61f0 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -500,7 +500,6 @@ class Anatomy(BaseAnatomy): roots_locals.update_data(roots_project_locals.get("studio") or {}) return roots_locals - @classmethod def _get_site_root_overrides(cls, project_name, site_name): """Get root overrides for site. From afec68dcba636ccc345ab080bf049c37ec692068 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 27 Feb 2023 10:56:11 +0100 Subject: [PATCH 17/57] fix dict update --- openpype/pipeline/anatomy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/anatomy.py b/openpype/pipeline/anatomy.py index ac915b61f0..683960f3d8 100644 --- a/openpype/pipeline/anatomy.py +++ b/openpype/pipeline/anatomy.py @@ -497,7 +497,7 @@ class Anatomy(BaseAnatomy): # Combine roots from local settings roots_locals = roots_default_locals.get("studio") or {} - roots_locals.update_data(roots_project_locals.get("studio") or {}) + roots_locals.update(roots_project_locals.get("studio") or {}) return roots_locals @classmethod From 3c215350931b4b65d5eab9a51cdc804ed128478a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 27 Feb 2023 11:34:30 +0100 Subject: [PATCH 18/57] removed unnecessary try catch --- openpype/modules/sync_server/sync_server.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/openpype/modules/sync_server/sync_server.py b/openpype/modules/sync_server/sync_server.py index aef3623efa..5b873a37cf 100644 --- a/openpype/modules/sync_server/sync_server.py +++ b/openpype/modules/sync_server/sync_server.py @@ -50,13 +50,10 @@ async def upload(module, project_name, file, representation, provider_name, presets=preset) file_path = file.get("path", "") - try: - local_file_path, remote_file_path = resolve_paths( - module, file_path, project_name, - remote_site_name, remote_handler - ) - except Exception as exp: - print(exp) + local_file_path, remote_file_path = resolve_paths( + module, file_path, project_name, + remote_site_name, remote_handler + ) target_folder = os.path.dirname(remote_file_path) folder_id = remote_handler.create_folder(target_folder) From fa879c9beb1982cb02f12d36925d28c81943589a Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 27 Feb 2023 17:06:14 +0100 Subject: [PATCH 19/57] fix UI definitions for publish plugins (#4534) --- openpype/tools/publisher/control.py | 3 ++ openpype/tools/publisher/widgets/widgets.py | 36 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/openpype/tools/publisher/control.py b/openpype/tools/publisher/control.py index 023a20ca5e..49e7eeb4f7 100644 --- a/openpype/tools/publisher/control.py +++ b/openpype/tools/publisher/control.py @@ -18,6 +18,7 @@ from openpype.client import ( ) from openpype.lib.events import EventSystem from openpype.lib.attribute_definitions import ( + UIDef, serialize_attr_defs, deserialize_attr_defs, ) @@ -1938,6 +1939,8 @@ class PublisherController(BasePublisherController): plugin_values = all_plugin_values[plugin_name] for attr_def in attr_defs: + if isinstance(attr_def, UIDef): + continue if attr_def.key not in plugin_values: plugin_values[attr_def.key] = [] attr_values = plugin_values[attr_def.key] diff --git a/openpype/tools/publisher/widgets/widgets.py b/openpype/tools/publisher/widgets/widgets.py index 8da3886419..86475460aa 100644 --- a/openpype/tools/publisher/widgets/widgets.py +++ b/openpype/tools/publisher/widgets/widgets.py @@ -9,7 +9,7 @@ import collections from qtpy import QtWidgets, QtCore, QtGui import qtawesome -from openpype.lib.attribute_definitions import UnknownDef +from openpype.lib.attribute_definitions import UnknownDef, UIDef from openpype.tools.attribute_defs import create_widget_for_attr_def from openpype.tools import resources from openpype.tools.flickcharm import FlickCharm @@ -1442,7 +1442,16 @@ class PublishPluginAttrsWidget(QtWidgets.QWidget): ) content_widget = QtWidgets.QWidget(self._scroll_area) - content_layout = QtWidgets.QFormLayout(content_widget) + attr_def_widget = QtWidgets.QWidget(content_widget) + attr_def_layout = QtWidgets.QGridLayout(attr_def_widget) + attr_def_layout.setColumnStretch(0, 0) + attr_def_layout.setColumnStretch(1, 1) + + content_layout = QtWidgets.QVBoxLayout(content_widget) + content_layout.addWidget(attr_def_widget, 0) + content_layout.addStretch(1) + + row = 0 for plugin_name, attr_defs, all_plugin_values in result: plugin_values = all_plugin_values[plugin_name] @@ -1459,8 +1468,29 @@ class PublishPluginAttrsWidget(QtWidgets.QWidget): hidden_widget = True if not hidden_widget: + expand_cols = 2 + if attr_def.is_value_def and attr_def.is_label_horizontal: + expand_cols = 1 + + col_num = 2 - expand_cols label = attr_def.label or attr_def.key - content_layout.addRow(label, widget) + if label: + label_widget = QtWidgets.QLabel(label, content_widget) + tooltip = attr_def.tooltip + if tooltip: + label_widget.setToolTip(tooltip) + attr_def_layout.addWidget( + label_widget, row, 0, 1, expand_cols + ) + if not attr_def.is_label_horizontal: + row += 1 + attr_def_layout.addWidget( + widget, row, col_num, 1, expand_cols + ) + row += 1 + + if isinstance(attr_def, UIDef): + continue widget.value_changed.connect(self._input_value_changed) From 2839e775e43eaddaa28d58bb11471c450198b4dd Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Mon, 27 Feb 2023 18:55:23 +0100 Subject: [PATCH 20/57] Deadline: Hint to use Python 3 (#4518) * add shebank for python 3 --- .../deadline/repository/custom/plugins/GlobalJobPreLoad.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py index e4fc64269a..20a58c9131 100644 --- a/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py +++ b/openpype/modules/deadline/repository/custom/plugins/GlobalJobPreLoad.py @@ -1,3 +1,4 @@ +# /usr/bin/env python3 # -*- coding: utf-8 -*- import os import tempfile @@ -341,7 +342,7 @@ def inject_openpype_environment(deadlinePlugin): "app": job.GetJobEnvironmentKeyValue("AVALON_APP_NAME"), "envgroup": "farm" } - + if job.GetJobEnvironmentKeyValue('IS_TEST'): args.append("--automatic-tests") From 86f0383f183251cb2b725f06ad33ba887e2305ef Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:13:44 +0100 Subject: [PATCH 21/57] Publisher: Prevent access to create tab after publish start (#4528) make sure it is not possible to go to create tab if the tab is not enabled --- openpype/tools/publisher/widgets/overview_widget.py | 3 +++ openpype/tools/publisher/window.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/tools/publisher/widgets/overview_widget.py b/openpype/tools/publisher/widgets/overview_widget.py index 022de2dc34..8706daeda6 100644 --- a/openpype/tools/publisher/widgets/overview_widget.py +++ b/openpype/tools/publisher/widgets/overview_widget.py @@ -146,6 +146,7 @@ class OverviewWidget(QtWidgets.QFrame): self._subset_list_view = subset_list_view self._subset_views_layout = subset_views_layout + self._create_btn = create_btn self._delete_btn = delete_btn self._subset_attributes_widget = subset_attributes_widget @@ -388,11 +389,13 @@ class OverviewWidget(QtWidgets.QFrame): def _on_publish_start(self): """Publish started.""" + self._create_btn.setEnabled(False) self._subset_attributes_wrap.setEnabled(False) def _on_publish_reset(self): """Context in controller has been refreshed.""" + self._create_btn.setEnabled(True) self._subset_attributes_wrap.setEnabled(True) self._subset_content_widget.setEnabled(self._controller.host_is_valid) diff --git a/openpype/tools/publisher/window.py b/openpype/tools/publisher/window.py index 6f7ffdb8ea..74977d65d8 100644 --- a/openpype/tools/publisher/window.py +++ b/openpype/tools/publisher/window.py @@ -561,7 +561,8 @@ class PublisherWindow(QtWidgets.QDialog): return self._tabs_widget.is_current_tab(identifier) def _go_to_create_tab(self): - self._set_current_tab("create") + if self._create_tab.isEnabled(): + self._set_current_tab("create") def _go_to_publish_tab(self): self._set_current_tab("publish") From f83fefd8a91f90133216f819a251594a72fe920c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 20 Feb 2023 18:35:21 +0100 Subject: [PATCH 22/57] Added support for extensions filtering --- openpype/pipeline/load/plugins.py | 82 +++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/openpype/pipeline/load/plugins.py b/openpype/pipeline/load/plugins.py index 9b891a4da3..8372743a23 100644 --- a/openpype/pipeline/load/plugins.py +++ b/openpype/pipeline/load/plugins.py @@ -21,16 +21,18 @@ class LoaderPlugin(list): Arguments: context (dict): avalon-core:context-1.0 - name (str, optional): Use pre-defined name - namespace (str, optional): Use pre-defined namespace .. versionadded:: 4.0 This class was introduced """ - families = list() - representations = list() + families = [] + representations = [] + # Extensions filtering was not available until 20/2/2023 + # - filtering by extensions is not enabled if is set to 'None' + # which is to keep backwards compatibility + extensions = None order = 0 is_multiple_contexts_compatible = False enabled = True @@ -82,20 +84,72 @@ class LoaderPlugin(list): print(" - setting `{}`: `{}`".format(option, value)) setattr(cls, option, value) + @classmethod + def has_valid_extension(cls, repre_doc): + """Has representation document valid extension for loader. + + Args: + repre_doc (dict[str, Any]): Representation document. + + Returns: + bool: Representation has valid extension + """ + + # Empty list of extensions is considered as all representations are + # invalid -> use '["*"]' to support all extensions + valid_extensions = cls.extensions + if not valid_extensions: + return False + + # Get representation main file extension from 'context' + repre_context = repre_doc.get("context") or {} + ext = repre_context.get("ext") + if not ext: + # Legacy way how to get extensions + path = repre_doc["data"].get("path") + if not path: + cls.log.info( + "Representation doesn't have known source of extension" + " information." + ) + return False + + cls.log.info("Using legacy source of extension on representation.") + ext = os.path.splitext(path)[-1] + while ext.startswith("."): + ext = ext[1:] + + # If representation does not have extension then can't be valid + if not ext: + return False + + if "*" in valid_extensions: + return True + + valid_extensions_low = {ext.lower() for ext in valid_extensions} + return ext.lower() in valid_extensions_low + + @classmethod def is_compatible_loader(cls, context): """Return whether a loader is compatible with a context. + On override make sure it is overriden as class or static method. + This checks the version's families and the representation for the given - Loader. + loader plugin. + + Args: + context (dict[str, Any]): Documents of context for which should + be loader used. Returns: - bool + bool: Is loader compatible for context. """ plugin_repre_names = cls.get_representations() plugin_families = cls.families - if not plugin_repre_names or not plugin_families: + if not plugin_repre_names or not plugin_families or not cls.extensions: return False repre_doc = context.get("representation") @@ -109,11 +163,19 @@ class LoaderPlugin(list): ): return False - maj_version, _ = schema.get_schema_version(context["subset"]["schema"]) + if ( + cls.extensions is not None + and not cls.has_valid_extension(repre_doc) + ): + return False + + verison_doc = context["version"] + subset_doc = context["subset"] + maj_version, _ = schema.get_schema_version(subset_doc["schema"]) if maj_version < 3: - families = context["version"]["data"].get("families", []) + families = verison_doc["data"].get("families", []) else: - families = context["subset"]["data"]["families"] + families = subset_doc["families"] plugin_families = set(plugin_families) return ( From 1b7af3e2c6299df7d6aee1f9c3e5f3e12e60db3f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 23 Feb 2023 18:53:32 +0100 Subject: [PATCH 23/57] fix filtering --- openpype/pipeline/load/plugins.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openpype/pipeline/load/plugins.py b/openpype/pipeline/load/plugins.py index 8372743a23..4ce7d47fd9 100644 --- a/openpype/pipeline/load/plugins.py +++ b/openpype/pipeline/load/plugins.py @@ -149,7 +149,7 @@ class LoaderPlugin(list): plugin_repre_names = cls.get_representations() plugin_families = cls.families - if not plugin_repre_names or not plugin_families or not cls.extensions: + if not plugin_repre_names or not plugin_families: return False repre_doc = context.get("representation") @@ -173,14 +173,21 @@ class LoaderPlugin(list): subset_doc = context["subset"] maj_version, _ = schema.get_schema_version(subset_doc["schema"]) if maj_version < 3: - families = verison_doc["data"].get("families", []) + families = verison_doc["data"].get("families") else: - families = subset_doc["families"] + families = context["subset"]["data"].get("families") + if families is None: + family = context["subset"]["data"].get("family") + if family: + families = [family] plugin_families = set(plugin_families) return ( "*" in plugin_families - or any(family in plugin_families for family in families) + or any( + family in plugin_families + for family in (families or []) + ) ) @classmethod From 7a8aa123ff8bc159cbc7b4a0c7acedb01847b9e9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 23 Feb 2023 21:48:32 +0100 Subject: [PATCH 24/57] cleanup changes based on comments --- openpype/pipeline/load/plugins.py | 57 ++++++++++++------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/openpype/pipeline/load/plugins.py b/openpype/pipeline/load/plugins.py index 4ce7d47fd9..f5b6e858b4 100644 --- a/openpype/pipeline/load/plugins.py +++ b/openpype/pipeline/load/plugins.py @@ -29,10 +29,7 @@ class LoaderPlugin(list): families = [] representations = [] - # Extensions filtering was not available until 20/2/2023 - # - filtering by extensions is not enabled if is set to 'None' - # which is to keep backwards compatibility - extensions = None + extensions = {"*"} order = 0 is_multiple_contexts_compatible = False enabled = True @@ -95,11 +92,8 @@ class LoaderPlugin(list): bool: Representation has valid extension """ - # Empty list of extensions is considered as all representations are - # invalid -> use '["*"]' to support all extensions - valid_extensions = cls.extensions - if not valid_extensions: - return False + if "*" in cls.extensions: + return True # Get representation main file extension from 'context' repre_context = repre_doc.get("context") or {} @@ -114,22 +108,16 @@ class LoaderPlugin(list): ) return False - cls.log.info("Using legacy source of extension on representation.") - ext = os.path.splitext(path)[-1] - while ext.startswith("."): - ext = ext[1:] + cls.log.info("Using legacy source of extension from path.") + ext = os.path.splitext(path)[-1].lstrip(".") # If representation does not have extension then can't be valid if not ext: return False - if "*" in valid_extensions: - return True - - valid_extensions_low = {ext.lower() for ext in valid_extensions} + valid_extensions_low = {ext.lower() for ext in cls.extensions} return ext.lower() in valid_extensions_low - @classmethod def is_compatible_loader(cls, context): """Return whether a loader is compatible with a context. @@ -149,7 +137,11 @@ class LoaderPlugin(list): plugin_repre_names = cls.get_representations() plugin_families = cls.families - if not plugin_repre_names or not plugin_families: + if ( + not plugin_repre_names + or not plugin_families + or not cls.extensions + ): return False repre_doc = context.get("representation") @@ -163,32 +155,27 @@ class LoaderPlugin(list): ): return False - if ( - cls.extensions is not None - and not cls.has_valid_extension(repre_doc) - ): + if not cls.has_valid_extension(repre_doc): return False - verison_doc = context["version"] + plugin_families = set(plugin_families) + if "*" in plugin_families: + return True + subset_doc = context["subset"] maj_version, _ = schema.get_schema_version(subset_doc["schema"]) if maj_version < 3: - families = verison_doc["data"].get("families") + families = context["version"]["data"].get("families") else: - families = context["subset"]["data"].get("families") + families = subset_doc["data"].get("families") if families is None: - family = context["subset"]["data"].get("family") + family = subset_doc["data"].get("family") if family: families = [family] - plugin_families = set(plugin_families) - return ( - "*" in plugin_families - or any( - family in plugin_families - for family in (families or []) - ) - ) + if not families: + return False + return any(family in plugin_families for family in families) @classmethod def get_representations(cls): From 8180076c08efa2e1ad3e1ece939b1b40899b3ef2 Mon Sep 17 00:00:00 2001 From: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> Date: Fri, 24 Feb 2023 10:20:40 +0100 Subject: [PATCH 25/57] safer data access Co-authored-by: Roy Nieterau --- openpype/pipeline/load/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/load/plugins.py b/openpype/pipeline/load/plugins.py index f5b6e858b4..d35281e823 100644 --- a/openpype/pipeline/load/plugins.py +++ b/openpype/pipeline/load/plugins.py @@ -100,7 +100,7 @@ class LoaderPlugin(list): ext = repre_context.get("ext") if not ext: # Legacy way how to get extensions - path = repre_doc["data"].get("path") + path = repre_doc.get("data", {}).get("path") if not path: cls.log.info( "Representation doesn't have known source of extension" From 9e432f7c5c7fa19dec06123d6e0867c9735be147 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 24 Feb 2023 10:55:30 +0100 Subject: [PATCH 26/57] add data and context to repre document fields --- openpype/tools/loader/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/tools/loader/widgets.py b/openpype/tools/loader/widgets.py index 0c5c9391cf..538925d141 100644 --- a/openpype/tools/loader/widgets.py +++ b/openpype/tools/loader/widgets.py @@ -339,7 +339,7 @@ class SubsetWidget(QtWidgets.QWidget): repre_docs = get_representations( project_name, version_ids=version_ids, - fields=["name", "parent"] + fields=["name", "parent", "data", "context"] ) repre_docs_by_version_id = { From 8a34313b97b7d6dc370aac485f188ffdb705a3f9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 24 Feb 2023 11:01:24 +0100 Subject: [PATCH 27/57] added 'data' and 'context' to representations widget too --- openpype/tools/loader/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/tools/loader/widgets.py b/openpype/tools/loader/widgets.py index 538925d141..98ac9c871f 100644 --- a/openpype/tools/loader/widgets.py +++ b/openpype/tools/loader/widgets.py @@ -1264,7 +1264,7 @@ class RepresentationWidget(QtWidgets.QWidget): repre_docs = list(get_representations( project_name, representation_ids=repre_ids, - fields=["name", "parent"] + fields=["name", "parent", "data", "context"] )) version_ids = [ From 1bd5f04880611a4461ead821dc2a14cf60d466b3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 11:16:25 +0100 Subject: [PATCH 28/57] nuke: extension to loaders --- openpype/hosts/nuke/plugins/load/load_clip.py | 28 ++++++++++--------- .../hosts/nuke/plugins/load/load_image.py | 10 +++++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/openpype/hosts/nuke/plugins/load/load_clip.py b/openpype/hosts/nuke/plugins/load/load_clip.py index 8f9b463037..18ecaf4d2b 100644 --- a/openpype/hosts/nuke/plugins/load/load_clip.py +++ b/openpype/hosts/nuke/plugins/load/load_clip.py @@ -21,6 +21,10 @@ from openpype.hosts.nuke.api import ( viewer_update_and_undo_stop, colorspace_exists_on_node ) +from openpype.lib.transcoding import ( + VIDEO_EXTENSIONS, + IMAGE_EXTENSIONS +) from openpype.hosts.nuke.api import plugin @@ -38,13 +42,11 @@ class LoadClip(plugin.NukeLoader): "prerender", "review" ] - representations = [ - "exr", - "dpx", - "mov", - "review", - "mp4" - ] + representations = ["*"] + + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) + ) label = "Load Clip" order = -20 @@ -81,17 +83,17 @@ class LoadClip(plugin.NukeLoader): @classmethod def get_representations(cls): - return ( - cls.representations - + cls._representations - + plugin.get_review_presets_config() - ) + return cls._representations or cls.representations def load(self, context, name, namespace, options): + """Load asset via database + """ representation = context["representation"] - # reste container id so it is always unique for each instance + # reset container id so it is always unique for each instance self.reset_container_id() + self.log.warning(self.extensions) + is_sequence = len(representation["files"]) > 1 if is_sequence: diff --git a/openpype/hosts/nuke/plugins/load/load_image.py b/openpype/hosts/nuke/plugins/load/load_image.py index 49dc12f588..f82ee4db88 100644 --- a/openpype/hosts/nuke/plugins/load/load_image.py +++ b/openpype/hosts/nuke/plugins/load/load_image.py @@ -19,6 +19,9 @@ from openpype.hosts.nuke.api import ( update_container, viewer_update_and_undo_stop ) +from openpype.lib.transcoding import ( + IMAGE_EXTENSIONS +) class LoadImage(load.LoaderPlugin): @@ -33,7 +36,10 @@ class LoadImage(load.LoaderPlugin): "review", "image" ] - representations = ["exr", "dpx", "jpg", "jpeg", "png", "psd", "tiff"] + representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS + ) label = "Load Image" order = -10 @@ -58,7 +64,7 @@ class LoadImage(load.LoaderPlugin): @classmethod def get_representations(cls): - return cls.representations + cls._representations + return cls._representations or cls.representations def load(self, context, name, namespace, options): self.log.info("__ options: `{}`".format(options)) From f432fb29de1cc9e01ea59a2482a9430086c987fa Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 11:26:46 +0100 Subject: [PATCH 29/57] nuke: adding extension to plugins --- openpype/hosts/nuke/plugins/load/actions.py | 1 + openpype/hosts/nuke/plugins/load/load_backdrop.py | 3 ++- openpype/hosts/nuke/plugins/load/load_camera_abc.py | 3 ++- openpype/hosts/nuke/plugins/load/load_clip.py | 1 - openpype/hosts/nuke/plugins/load/load_effects.py | 3 ++- openpype/hosts/nuke/plugins/load/load_effects_ip.py | 3 ++- openpype/hosts/nuke/plugins/load/load_gizmo.py | 3 ++- openpype/hosts/nuke/plugins/load/load_gizmo_ip.py | 3 ++- openpype/hosts/nuke/plugins/load/load_matchmove.py | 4 +++- openpype/hosts/nuke/plugins/load/load_model.py | 3 ++- openpype/hosts/nuke/plugins/load/load_script_precomp.py | 3 ++- 11 files changed, 20 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/nuke/plugins/load/actions.py b/openpype/hosts/nuke/plugins/load/actions.py index 69f56c7305..e562c74c58 100644 --- a/openpype/hosts/nuke/plugins/load/actions.py +++ b/openpype/hosts/nuke/plugins/load/actions.py @@ -17,6 +17,7 @@ class SetFrameRangeLoader(load.LoaderPlugin): "yeticache", "pointcache"] representations = ["*"] + extension = {"*"} label = "Set frame range" order = 11 diff --git a/openpype/hosts/nuke/plugins/load/load_backdrop.py b/openpype/hosts/nuke/plugins/load/load_backdrop.py index d1fb763500..f227aa161a 100644 --- a/openpype/hosts/nuke/plugins/load/load_backdrop.py +++ b/openpype/hosts/nuke/plugins/load/load_backdrop.py @@ -25,8 +25,9 @@ from openpype.hosts.nuke.api import containerise, update_container class LoadBackdropNodes(load.LoaderPlugin): """Loading Published Backdrop nodes (workfile, nukenodes)""" - representations = ["nk"] families = ["workfile", "nukenodes"] + representations = ["*"] + extension = {"nk"} label = "Import Nuke Nodes" order = 0 diff --git a/openpype/hosts/nuke/plugins/load/load_camera_abc.py b/openpype/hosts/nuke/plugins/load/load_camera_abc.py index 9fef7424c8..11cc63d25c 100644 --- a/openpype/hosts/nuke/plugins/load/load_camera_abc.py +++ b/openpype/hosts/nuke/plugins/load/load_camera_abc.py @@ -25,7 +25,8 @@ class AlembicCameraLoader(load.LoaderPlugin): """ families = ["camera"] - representations = ["abc"] + representations = ["*"] + extension = {"abc"} label = "Load Alembic Camera" icon = "camera" diff --git a/openpype/hosts/nuke/plugins/load/load_clip.py b/openpype/hosts/nuke/plugins/load/load_clip.py index 18ecaf4d2b..d170276add 100644 --- a/openpype/hosts/nuke/plugins/load/load_clip.py +++ b/openpype/hosts/nuke/plugins/load/load_clip.py @@ -43,7 +43,6 @@ class LoadClip(plugin.NukeLoader): "review" ] representations = ["*"] - extensions = set( ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) ) diff --git a/openpype/hosts/nuke/plugins/load/load_effects.py b/openpype/hosts/nuke/plugins/load/load_effects.py index cef4b0a5fc..d49f87a094 100644 --- a/openpype/hosts/nuke/plugins/load/load_effects.py +++ b/openpype/hosts/nuke/plugins/load/load_effects.py @@ -22,8 +22,9 @@ from openpype.hosts.nuke.api import ( class LoadEffects(load.LoaderPlugin): """Loading colorspace soft effect exported from nukestudio""" - representations = ["effectJson"] families = ["effect"] + representations = ["*"] + extension = {"json"} label = "Load Effects - nodes" order = 0 diff --git a/openpype/hosts/nuke/plugins/load/load_effects_ip.py b/openpype/hosts/nuke/plugins/load/load_effects_ip.py index 9bd40be816..bfe32c1ed9 100644 --- a/openpype/hosts/nuke/plugins/load/load_effects_ip.py +++ b/openpype/hosts/nuke/plugins/load/load_effects_ip.py @@ -23,8 +23,9 @@ from openpype.hosts.nuke.api import ( class LoadEffectsInputProcess(load.LoaderPlugin): """Loading colorspace soft effect exported from nukestudio""" - representations = ["effectJson"] families = ["effect"] + representations = ["*"] + extension = {"json"} label = "Load Effects - Input Process" order = 0 diff --git a/openpype/hosts/nuke/plugins/load/load_gizmo.py b/openpype/hosts/nuke/plugins/load/load_gizmo.py index 9a18eeef5c..2aa7c49723 100644 --- a/openpype/hosts/nuke/plugins/load/load_gizmo.py +++ b/openpype/hosts/nuke/plugins/load/load_gizmo.py @@ -24,8 +24,9 @@ from openpype.hosts.nuke.api import ( class LoadGizmo(load.LoaderPlugin): """Loading nuke Gizmo""" - representations = ["gizmo"] families = ["gizmo"] + representations = ["*"] + extension = {"gizmo"} label = "Load Gizmo" order = 0 diff --git a/openpype/hosts/nuke/plugins/load/load_gizmo_ip.py b/openpype/hosts/nuke/plugins/load/load_gizmo_ip.py index 2890dbfd2c..2514a28299 100644 --- a/openpype/hosts/nuke/plugins/load/load_gizmo_ip.py +++ b/openpype/hosts/nuke/plugins/load/load_gizmo_ip.py @@ -26,8 +26,9 @@ from openpype.hosts.nuke.api import ( class LoadGizmoInputProcess(load.LoaderPlugin): """Loading colorspace soft effect exported from nukestudio""" - representations = ["gizmo"] families = ["gizmo"] + representations = ["*"] + extension = {"gizmo"} label = "Load Gizmo - Input Process" order = 0 diff --git a/openpype/hosts/nuke/plugins/load/load_matchmove.py b/openpype/hosts/nuke/plugins/load/load_matchmove.py index f5a90706c7..a7d124d472 100644 --- a/openpype/hosts/nuke/plugins/load/load_matchmove.py +++ b/openpype/hosts/nuke/plugins/load/load_matchmove.py @@ -8,7 +8,9 @@ class MatchmoveLoader(load.LoaderPlugin): """ families = ["matchmove"] - representations = ["py"] + representations = ["*"] + extension = {"py"} + defaults = ["Camera", "Object"] label = "Run matchmove script" diff --git a/openpype/hosts/nuke/plugins/load/load_model.py b/openpype/hosts/nuke/plugins/load/load_model.py index ad985e83c6..f968da8475 100644 --- a/openpype/hosts/nuke/plugins/load/load_model.py +++ b/openpype/hosts/nuke/plugins/load/load_model.py @@ -23,7 +23,8 @@ class AlembicModelLoader(load.LoaderPlugin): """ families = ["model", "pointcache", "animation"] - representations = ["abc"] + representations = ["*"] + extension = {"abc"} label = "Load Alembic" icon = "cube" diff --git a/openpype/hosts/nuke/plugins/load/load_script_precomp.py b/openpype/hosts/nuke/plugins/load/load_script_precomp.py index f0972f85d2..90581c2f22 100644 --- a/openpype/hosts/nuke/plugins/load/load_script_precomp.py +++ b/openpype/hosts/nuke/plugins/load/load_script_precomp.py @@ -20,8 +20,9 @@ from openpype.hosts.nuke.api import ( class LinkAsGroup(load.LoaderPlugin): """Copy the published file to be pasted at the desired location""" - representations = ["nk"] families = ["workfile", "nukenodes"] + representations = ["*"] + extension = {"nk"} label = "Load Precomp" order = 0 From 1b479f7eb7919c114efa7537f4a9922eb9e01a4b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 11:52:26 +0100 Subject: [PATCH 30/57] resolve: adding extensions to loader --- openpype/hosts/resolve/plugins/load/load_clip.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/resolve/plugins/load/load_clip.py b/openpype/hosts/resolve/plugins/load/load_clip.py index a0c78c182f..d30a7ea272 100644 --- a/openpype/hosts/resolve/plugins/load/load_clip.py +++ b/openpype/hosts/resolve/plugins/load/load_clip.py @@ -14,7 +14,10 @@ from openpype.hosts.resolve.api.pipeline import ( containerise, update_container, ) - +from openpype.lib.transcoding import ( + VIDEO_EXTENSIONS, + IMAGE_EXTENSIONS +) class LoadClip(plugin.TimelineItemLoader): """Load a subset to timeline as clip @@ -24,7 +27,11 @@ class LoadClip(plugin.TimelineItemLoader): """ families = ["render2d", "source", "plate", "render", "review"] - representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264", "mov"] + + representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) + ) label = "Load as clip" order = -10 From 339417c5055cdb8f77ee603a32a9caa630ab695d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 15:14:31 +0100 Subject: [PATCH 31/57] hiero: adding extension to load plugins also removing representation from settings and ignoring settings for representation on plugin --- .../hosts/hiero/plugins/load/load_clip.py | 41 ++++++++++++++++++- .../hosts/hiero/plugins/load/load_effects.py | 3 +- .../defaults/project_settings/hiero.json | 10 ----- .../projects_schema/schema_project_hiero.json | 8 +--- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/openpype/hosts/hiero/plugins/load/load_clip.py b/openpype/hosts/hiero/plugins/load/load_clip.py index 2a7d1af41e..77844d2448 100644 --- a/openpype/hosts/hiero/plugins/load/load_clip.py +++ b/openpype/hosts/hiero/plugins/load/load_clip.py @@ -6,6 +6,10 @@ from openpype.pipeline import ( legacy_io, get_representation_path, ) +from openpype.lib.transcoding import ( + VIDEO_EXTENSIONS, + IMAGE_EXTENSIONS +) import openpype.hosts.hiero.api as phiero @@ -17,7 +21,10 @@ class LoadClip(phiero.SequenceLoader): """ families = ["render2d", "source", "plate", "render", "review"] - representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264"] + representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) + ) label = "Load as clip" order = -10 @@ -34,6 +41,38 @@ class LoadClip(phiero.SequenceLoader): clip_name_template = "{asset}_{subset}_{representation}" + def apply_settings(cls, project_settings, system_settings): + + plugin_type_settings = ( + project_settings + .get("hiero", {}) + .get("load", {}) + ) + + if not plugin_type_settings: + return + + plugin_name = cls.__name__ + + plugin_settings = None + # Look for plugin settings in host specific settings + if plugin_name in plugin_type_settings: + plugin_settings = plugin_type_settings[plugin_name] + + if not plugin_settings: + return + + print(">>> We have preset for {}".format(plugin_name)) + for option, value in plugin_settings.items(): + if option == "enabled" and value is False: + print(" - is disabled by preset") + elif option == "representations": + continue + else: + print(" - setting `{}`: `{}`".format(option, value)) + setattr(cls, option, value) + + def load(self, context, name, namespace, options): # add clip name template to options options.update({ diff --git a/openpype/hosts/hiero/plugins/load/load_effects.py b/openpype/hosts/hiero/plugins/load/load_effects.py index a3fcd63b5b..b61cca9731 100644 --- a/openpype/hosts/hiero/plugins/load/load_effects.py +++ b/openpype/hosts/hiero/plugins/load/load_effects.py @@ -19,8 +19,9 @@ from openpype.lib import Logger class LoadEffects(load.LoaderPlugin): """Loading colorspace soft effect exported from nukestudio""" - representations = ["effectJson"] families = ["effect"] + representations = ["*"] + extension = {"json"} label = "Load Effects" order = 0 diff --git a/openpype/settings/defaults/project_settings/hiero.json b/openpype/settings/defaults/project_settings/hiero.json index 0412967eaa..100c1f5b47 100644 --- a/openpype/settings/defaults/project_settings/hiero.json +++ b/openpype/settings/defaults/project_settings/hiero.json @@ -60,16 +60,6 @@ "render", "review" ], - "representations": [ - "exr", - "dpx", - "jpg", - "jpeg", - "png", - "h264", - "mov", - "mp4" - ], "clip_name_template": "{asset}_{subset}_{representation}" } }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_hiero.json b/openpype/settings/entities/schemas/projects_schema/schema_project_hiero.json index 03bfb56ad1..f44f92438c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_hiero.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_hiero.json @@ -266,12 +266,6 @@ "label": "Families", "object_type": "text" }, - { - "type": "list", - "key": "representations", - "label": "Representations", - "object_type": "text" - }, { "type": "text", "key": "clip_name_template", @@ -334,4 +328,4 @@ "name": "schema_scriptsmenu" } ] -} \ No newline at end of file +} From 6d43fa21902f0b9cfa2beee43bf9e9d8ff362eb0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 15:19:50 +0100 Subject: [PATCH 32/57] flame: adding extensions to loaders also removing representations from settings and ignoring them in plugin settings loader --- openpype/hosts/flame/api/plugin.py | 31 +++++++++++++++++++ .../hosts/flame/plugins/load/load_clip.py | 9 +++++- .../flame/plugins/load/load_clip_batch.py | 10 ++++-- .../defaults/project_settings/flame.json | 22 ------------- .../projects_schema/schema_project_flame.json | 12 ------- 5 files changed, 47 insertions(+), 37 deletions(-) diff --git a/openpype/hosts/flame/api/plugin.py b/openpype/hosts/flame/api/plugin.py index b1db612671..983d7486b3 100644 --- a/openpype/hosts/flame/api/plugin.py +++ b/openpype/hosts/flame/api/plugin.py @@ -702,6 +702,37 @@ class ClipLoader(LoaderPlugin): _mapping = None + def apply_settings(cls, project_settings, system_settings): + + plugin_type_settings = ( + project_settings + .get("flame", {}) + .get("load", {}) + ) + + if not plugin_type_settings: + return + + plugin_name = cls.__name__ + + plugin_settings = None + # Look for plugin settings in host specific settings + if plugin_name in plugin_type_settings: + plugin_settings = plugin_type_settings[plugin_name] + + if not plugin_settings: + return + + print(">>> We have preset for {}".format(plugin_name)) + for option, value in plugin_settings.items(): + if option == "enabled" and value is False: + print(" - is disabled by preset") + elif option == "representations": + continue + else: + print(" - setting `{}`: `{}`".format(option, value)) + setattr(cls, option, value) + def get_colorspace(self, context): """Get colorspace name diff --git a/openpype/hosts/flame/plugins/load/load_clip.py b/openpype/hosts/flame/plugins/load/load_clip.py index 6f47c23d57..25b31c94a3 100644 --- a/openpype/hosts/flame/plugins/load/load_clip.py +++ b/openpype/hosts/flame/plugins/load/load_clip.py @@ -4,6 +4,10 @@ import flame from pprint import pformat import openpype.hosts.flame.api as opfapi from openpype.lib import StringTemplate +from openpype.lib.transcoding import ( + VIDEO_EXTENSIONS, + IMAGE_EXTENSIONS +) class LoadClip(opfapi.ClipLoader): @@ -14,7 +18,10 @@ class LoadClip(opfapi.ClipLoader): """ families = ["render2d", "source", "plate", "render", "review"] - representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264"] + representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) + ) label = "Load as clip" order = -10 diff --git a/openpype/hosts/flame/plugins/load/load_clip_batch.py b/openpype/hosts/flame/plugins/load/load_clip_batch.py index 5975c6e42f..86bc0f8f1e 100644 --- a/openpype/hosts/flame/plugins/load/load_clip_batch.py +++ b/openpype/hosts/flame/plugins/load/load_clip_batch.py @@ -4,7 +4,10 @@ import flame from pprint import pformat import openpype.hosts.flame.api as opfapi from openpype.lib import StringTemplate - +from openpype.lib.transcoding import ( + VIDEO_EXTENSIONS, + IMAGE_EXTENSIONS +) class LoadClipBatch(opfapi.ClipLoader): """Load a subset to timeline as clip @@ -14,7 +17,10 @@ class LoadClipBatch(opfapi.ClipLoader): """ families = ["render2d", "source", "plate", "render", "review"] - representations = ["exr", "dpx", "jpg", "jpeg", "png", "h264"] + representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) + ) label = "Load as clip to current batch" order = -10 diff --git a/openpype/settings/defaults/project_settings/flame.json b/openpype/settings/defaults/project_settings/flame.json index 3190bdb3bf..5a13d81384 100644 --- a/openpype/settings/defaults/project_settings/flame.json +++ b/openpype/settings/defaults/project_settings/flame.json @@ -114,17 +114,6 @@ "render", "review" ], - "representations": [ - "exr", - "dpx", - "jpg", - "jpeg", - "png", - "h264", - "mov", - "mp4", - "exr16fpdwaa" - ], "reel_group_name": "OpenPype_Reels", "reel_name": "Loaded", "clip_name_template": "{asset}_{subset}<_{output}>", @@ -143,17 +132,6 @@ "render", "review" ], - "representations": [ - "exr", - "dpx", - "jpg", - "jpeg", - "png", - "h264", - "mov", - "mp4", - "exr16fpdwaa" - ], "reel_name": "OP_LoadedReel", "clip_name_template": "{batch}_{asset}_{subset}<_{output}>", "layer_rename_template": "{asset}_{subset}<_{output}>", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json index 0f20c0efbe..aab8f21d15 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_flame.json @@ -494,12 +494,6 @@ "label": "Families", "object_type": "text" }, - { - "type": "list", - "key": "representations", - "label": "Representations", - "object_type": "text" - }, { "type": "separator" }, @@ -552,12 +546,6 @@ "label": "Families", "object_type": "text" }, - { - "type": "list", - "key": "representations", - "label": "Representations", - "object_type": "text" - }, { "type": "separator" }, From a2d7ab24f60609096b58fbb8e6b891e743a1892e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 15:24:22 +0100 Subject: [PATCH 33/57] fusion: adding extensions to loaders --- openpype/hosts/fusion/plugins/load/actions.py | 1 + openpype/hosts/fusion/plugins/load/load_alembic.py | 3 ++- openpype/hosts/fusion/plugins/load/load_fbx.py | 3 ++- openpype/hosts/fusion/plugins/load/load_sequence.py | 6 ++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/fusion/plugins/load/actions.py b/openpype/hosts/fusion/plugins/load/actions.py index 819c9272fd..3b14f022e5 100644 --- a/openpype/hosts/fusion/plugins/load/actions.py +++ b/openpype/hosts/fusion/plugins/load/actions.py @@ -15,6 +15,7 @@ class FusionSetFrameRangeLoader(load.LoaderPlugin): "pointcache", "render"] representations = ["*"] + extensions = {"*"} label = "Set frame range" order = 11 diff --git a/openpype/hosts/fusion/plugins/load/load_alembic.py b/openpype/hosts/fusion/plugins/load/load_alembic.py index f8b8c2cb0a..11bf59af12 100644 --- a/openpype/hosts/fusion/plugins/load/load_alembic.py +++ b/openpype/hosts/fusion/plugins/load/load_alembic.py @@ -13,7 +13,8 @@ class FusionLoadAlembicMesh(load.LoaderPlugin): """Load Alembic mesh into Fusion""" families = ["pointcache", "model"] - representations = ["abc"] + representations = ["*"] + extensions = {"abc"} label = "Load alembic mesh" order = -10 diff --git a/openpype/hosts/fusion/plugins/load/load_fbx.py b/openpype/hosts/fusion/plugins/load/load_fbx.py index 70fe82ffef..b8f501ae7e 100644 --- a/openpype/hosts/fusion/plugins/load/load_fbx.py +++ b/openpype/hosts/fusion/plugins/load/load_fbx.py @@ -14,7 +14,8 @@ class FusionLoadFBXMesh(load.LoaderPlugin): """Load FBX mesh into Fusion""" families = ["*"] - representations = ["fbx"] + representations = ["*"] + extensions = {"fbx"} label = "Load FBX mesh" order = -10 diff --git a/openpype/hosts/fusion/plugins/load/load_sequence.py b/openpype/hosts/fusion/plugins/load/load_sequence.py index 6f44c61d1b..465df757b1 100644 --- a/openpype/hosts/fusion/plugins/load/load_sequence.py +++ b/openpype/hosts/fusion/plugins/load/load_sequence.py @@ -12,6 +12,9 @@ from openpype.hosts.fusion.api import ( get_current_comp, comp_lock_and_undo_chunk ) +from openpype.lib.transcoding import ( + IMAGE_EXTENSIONS +) comp = get_current_comp() @@ -129,6 +132,9 @@ class FusionLoadSequence(load.LoaderPlugin): families = ["imagesequence", "review", "render", "plate"] representations = ["*"] + extensions = set( + ext.lstrip(".") for ext in IMAGE_EXTENSIONS + ) label = "Load sequence" order = -10 From 3a02a03efad89fe50850c270a55ee028e3f6ec91 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 24 Feb 2023 16:38:34 +0100 Subject: [PATCH 34/57] fusion: adding video extensions to sequence loader --- openpype/hosts/fusion/plugins/load/load_sequence.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/fusion/plugins/load/load_sequence.py b/openpype/hosts/fusion/plugins/load/load_sequence.py index 465df757b1..542c3c1a51 100644 --- a/openpype/hosts/fusion/plugins/load/load_sequence.py +++ b/openpype/hosts/fusion/plugins/load/load_sequence.py @@ -13,7 +13,8 @@ from openpype.hosts.fusion.api import ( comp_lock_and_undo_chunk ) from openpype.lib.transcoding import ( - IMAGE_EXTENSIONS + IMAGE_EXTENSIONS, + VIDEO_EXTENSIONS ) comp = get_current_comp() @@ -133,7 +134,7 @@ class FusionLoadSequence(load.LoaderPlugin): families = ["imagesequence", "review", "render", "plate"] representations = ["*"] extensions = set( - ext.lstrip(".") for ext in IMAGE_EXTENSIONS + ext.lstrip(".") for ext in IMAGE_EXTENSIONS.union(VIDEO_EXTENSIONS) ) label = "Load sequence" From 42cc7152334bb59485470a6bb3a56d8eb76f8dde Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Fri, 24 Feb 2023 16:50:11 +0100 Subject: [PATCH 35/57] changed info to debug --- openpype/pipeline/load/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/pipeline/load/plugins.py b/openpype/pipeline/load/plugins.py index d35281e823..e380d65bbe 100644 --- a/openpype/pipeline/load/plugins.py +++ b/openpype/pipeline/load/plugins.py @@ -108,7 +108,7 @@ class LoaderPlugin(list): ) return False - cls.log.info("Using legacy source of extension from path.") + cls.log.debug("Using legacy source of extension from path.") ext = os.path.splitext(path)[-1].lstrip(".") # If representation does not have extension then can't be valid From 956fbb780badbd2bcbf052f4646d3014b5fe9df8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 28 Feb 2023 11:24:02 +0100 Subject: [PATCH 36/57] change Harmony loader to use extensions --- openpype/hosts/harmony/plugins/load/load_imagesequence.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/harmony/plugins/load/load_imagesequence.py b/openpype/hosts/harmony/plugins/load/load_imagesequence.py index 1b64aff595..7c3185fd75 100644 --- a/openpype/hosts/harmony/plugins/load/load_imagesequence.py +++ b/openpype/hosts/harmony/plugins/load/load_imagesequence.py @@ -20,8 +20,9 @@ class ImageSequenceLoader(load.LoaderPlugin): Stores the imported asset in a container named after the asset. """ - families = ["shot", "render", "image", "plate", "reference"] - representations = ["jpeg", "png", "jpg"] + families = ["shot", "render", "image", "plate", "reference", "review"] + representations = ["*"] + extensions = set(["jpeg", "png", "jpg"]) def load(self, context, name=None, namespace=None, data=None): """Plugin entry point. From 0765602ce3aabfc6751662d1026a7638385d67a8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 28 Feb 2023 11:27:13 +0100 Subject: [PATCH 37/57] simpler set initialization --- openpype/hosts/harmony/plugins/load/load_imagesequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/hosts/harmony/plugins/load/load_imagesequence.py b/openpype/hosts/harmony/plugins/load/load_imagesequence.py index 7c3185fd75..b95d25f507 100644 --- a/openpype/hosts/harmony/plugins/load/load_imagesequence.py +++ b/openpype/hosts/harmony/plugins/load/load_imagesequence.py @@ -22,7 +22,7 @@ class ImageSequenceLoader(load.LoaderPlugin): families = ["shot", "render", "image", "plate", "reference", "review"] representations = ["*"] - extensions = set(["jpeg", "png", "jpg"]) + extensions = {"jpeg", "png", "jpg"} def load(self, context, name=None, namespace=None, data=None): """Plugin entry point. From 477d2e24143ef5caebe6801d094541ab012ffe39 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 27 Feb 2023 18:15:44 +0100 Subject: [PATCH 38/57] :bug: return AssetContainer to allow loading --- .../OpenPype/Private/AssetContainer.cpp | 114 ++++++++++++++++++ .../Private/AssetContainerFactory.cpp | 20 +++ .../Source/OpenPype/Public/AssetContainer.h | 37 ++++++ .../OpenPype/Public/AssetContainerFactory.h | 21 ++++ 4 files changed, 192 insertions(+) create mode 100644 openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainer.cpp create mode 100644 openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainerFactory.cpp create mode 100644 openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainer.h create mode 100644 openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainerFactory.h diff --git a/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainer.cpp b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainer.cpp new file mode 100644 index 0000000000..0bea9e3d78 --- /dev/null +++ b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainer.cpp @@ -0,0 +1,114 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#include "AssetContainer.h" +#include "AssetRegistry/AssetRegistryModule.h" +#include "Misc/PackageName.h" +#include "Engine.h" +#include "Containers/UnrealString.h" + +UAssetContainer::UAssetContainer(const FObjectInitializer& ObjectInitializer) +: UAssetUserData(ObjectInitializer) +{ + FAssetRegistryModule& AssetRegistryModule = FModuleManager::LoadModuleChecked("AssetRegistry"); + FString path = UAssetContainer::GetPathName(); + UE_LOG(LogTemp, Warning, TEXT("UAssetContainer %s"), *path); + FARFilter Filter; + Filter.PackagePaths.Add(FName(*path)); + + AssetRegistryModule.Get().OnAssetAdded().AddUObject(this, &UAssetContainer::OnAssetAdded); + AssetRegistryModule.Get().OnAssetRemoved().AddUObject(this, &UAssetContainer::OnAssetRemoved); + AssetRegistryModule.Get().OnAssetRenamed().AddUObject(this, &UAssetContainer::OnAssetRenamed); +} + +void UAssetContainer::OnAssetAdded(const FAssetData& AssetData) +{ + TArray split; + + // get directory of current container + FString selfFullPath = UAssetContainer::GetPathName(); + FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath); + + // get asset path and class + FString assetPath = AssetData.GetFullName(); + FString assetFName = AssetData.AssetClassPath.ToString(); + UE_LOG(LogTemp, Log, TEXT("asset name %s"), *assetFName); + // split path + assetPath.ParseIntoArray(split, TEXT(" "), true); + + FString assetDir = FPackageName::GetLongPackagePath(*split[1]); + + // take interest only in paths starting with path of current container + if (assetDir.StartsWith(*selfDir)) + { + // exclude self + if (assetFName != "AssetContainer") + { + assets.Add(assetPath); + assetsData.Add(AssetData); + UE_LOG(LogTemp, Log, TEXT("%s: asset added to %s"), *selfFullPath, *selfDir); + } + } +} + +void UAssetContainer::OnAssetRemoved(const FAssetData& AssetData) +{ + TArray split; + + // get directory of current container + FString selfFullPath = UAssetContainer::GetPathName(); + FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath); + + // get asset path and class + FString assetPath = AssetData.GetFullName(); + FString assetFName = AssetData.AssetClassPath.ToString(); + + // split path + assetPath.ParseIntoArray(split, TEXT(" "), true); + + FString assetDir = FPackageName::GetLongPackagePath(*split[1]); + + // take interest only in paths starting with path of current container + FString path = UAssetContainer::GetPathName(); + FString lpp = FPackageName::GetLongPackagePath(*path); + + if (assetDir.StartsWith(*selfDir)) + { + // exclude self + if (assetFName != "AssetContainer") + { + // UE_LOG(LogTemp, Warning, TEXT("%s: asset removed"), *lpp); + assets.Remove(assetPath); + assetsData.Remove(AssetData); + } + } +} + +void UAssetContainer::OnAssetRenamed(const FAssetData& AssetData, const FString& str) +{ + TArray split; + + // get directory of current container + FString selfFullPath = UAssetContainer::GetPathName(); + FString selfDir = FPackageName::GetLongPackagePath(*selfFullPath); + + // get asset path and class + FString assetPath = AssetData.GetFullName(); + FString assetFName = AssetData.AssetClassPath.ToString(); + + // split path + assetPath.ParseIntoArray(split, TEXT(" "), true); + + FString assetDir = FPackageName::GetLongPackagePath(*split[1]); + if (assetDir.StartsWith(*selfDir)) + { + // exclude self + if (assetFName != "AssetContainer") + { + + assets.Remove(str); + assets.Add(assetPath); + assetsData.Remove(AssetData); + // UE_LOG(LogTemp, Warning, TEXT("%s: asset renamed %s"), *lpp, *str); + } + } +} diff --git a/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainerFactory.cpp b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainerFactory.cpp new file mode 100644 index 0000000000..b943150bdd --- /dev/null +++ b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Private/AssetContainerFactory.cpp @@ -0,0 +1,20 @@ +#include "AssetContainerFactory.h" +#include "AssetContainer.h" + +UAssetContainerFactory::UAssetContainerFactory(const FObjectInitializer& ObjectInitializer) + : UFactory(ObjectInitializer) +{ + SupportedClass = UAssetContainer::StaticClass(); + bCreateNew = false; + bEditorImport = true; +} + +UObject* UAssetContainerFactory::FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) +{ + UAssetContainer* AssetContainer = NewObject(InParent, Class, Name, Flags); + return AssetContainer; +} + +bool UAssetContainerFactory::ShouldShowInNewMenu() const { + return false; +} diff --git a/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainer.h b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainer.h new file mode 100644 index 0000000000..9157569c08 --- /dev/null +++ b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainer.h @@ -0,0 +1,37 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "UObject/NoExportTypes.h" +#include "Engine/AssetUserData.h" +#include "AssetRegistry/AssetData.h" +#include "AssetContainer.generated.h" + +/** + * + */ +UCLASS(Blueprintable) +class OPENPYPE_API UAssetContainer : public UAssetUserData +{ + GENERATED_BODY() + +public: + + UAssetContainer(const FObjectInitializer& ObjectInitalizer); + // ~UAssetContainer(); + + UPROPERTY(EditAnywhere, BlueprintReadOnly, Category="Assets") + TArray assets; + + // There seems to be no reflection option to expose array of FAssetData + /* + UPROPERTY(Transient, BlueprintReadOnly, Category = "Python", meta=(DisplayName="Assets Data")) + TArray assetsData; + */ +private: + TArray assetsData; + void OnAssetAdded(const FAssetData& AssetData); + void OnAssetRemoved(const FAssetData& AssetData); + void OnAssetRenamed(const FAssetData& AssetData, const FString& str); +}; diff --git a/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainerFactory.h b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainerFactory.h new file mode 100644 index 0000000000..9095f8a3d7 --- /dev/null +++ b/openpype/hosts/unreal/integration/UE_5.0/OpenPype/Source/OpenPype/Public/AssetContainerFactory.h @@ -0,0 +1,21 @@ +// Fill out your copyright notice in the Description page of Project Settings. + +#pragma once + +#include "CoreMinimal.h" +#include "Factories/Factory.h" +#include "AssetContainerFactory.generated.h" + +/** + * + */ +UCLASS() +class OPENPYPE_API UAssetContainerFactory : public UFactory +{ + GENERATED_BODY() + +public: + UAssetContainerFactory(const FObjectInitializer& ObjectInitializer); + virtual UObject* FactoryCreateNew(UClass* Class, UObject* InParent, FName Name, EObjectFlags Flags, UObject* Context, FFeedbackContext* Warn) override; + virtual bool ShouldShowInNewMenu() const override; +}; From d68cc23d32aabb7773b40ff2896fb8ea6390cae7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 28 Feb 2023 14:37:41 +0100 Subject: [PATCH 39/57] hardcode Poetry to 1.3.2 temporarily New 1.4.0 breaks sphinxcontrib-applehelp. Change in poetry.lock would need new minor version. --- tools/create_env.ps1 | 2 +- tools/create_env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/create_env.ps1 b/tools/create_env.ps1 index f79bc2a076..7d1f6635d0 100644 --- a/tools/create_env.ps1 +++ b/tools/create_env.ps1 @@ -68,7 +68,7 @@ function Install-Poetry() { } $env:POETRY_HOME="$openpype_root\.poetry" - # $env:POETRY_VERSION="1.1.15" + $env:POETRY_VERSION="1.3.2" (Invoke-WebRequest -Uri https://install.python-poetry.org/ -UseBasicParsing).Content | & $($python) - } diff --git a/tools/create_env.sh b/tools/create_env.sh index fbae69e56d..6915d3f000 100755 --- a/tools/create_env.sh +++ b/tools/create_env.sh @@ -109,7 +109,7 @@ detect_python () { install_poetry () { echo -e "${BIGreen}>>>${RST} Installing Poetry ..." export POETRY_HOME="$openpype_root/.poetry" - # export POETRY_VERSION="1.1.15" + export POETRY_VERSION="1.3.2" command -v curl >/dev/null 2>&1 || { echo -e "${BIRed}!!!${RST}${BIYellow} Missing ${RST}${BIBlue}curl${BIYellow} command.${RST}"; return 1; } curl -sSL https://install.python-poetry.org/ | python - } From 9f35ed2a63ce38f0d7c6b1c29fec305116896a04 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 28 Feb 2023 16:33:18 +0100 Subject: [PATCH 40/57] Fix - store target_colorspace as new colorspace (#4544) When transcoding into new colorspace, representation must carry this information instead original color space. --- openpype/plugins/publish/extract_color_transcode.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openpype/plugins/publish/extract_color_transcode.py b/openpype/plugins/publish/extract_color_transcode.py index a6fa710425..58e0350a2e 100644 --- a/openpype/plugins/publish/extract_color_transcode.py +++ b/openpype/plugins/publish/extract_color_transcode.py @@ -129,11 +129,14 @@ class ExtractOIIOTranscode(publish.Extractor): colorspace_data.get("display")) # both could be already collected by DCC, - # but could be overwritten + # but could be overwritten when transcoding if view: new_repre["colorspaceData"]["view"] = view if display: new_repre["colorspaceData"]["display"] = display + if target_colorspace: + new_repre["colorspaceData"]["colorspace"] = \ + target_colorspace additional_command_args = (output_def["oiiotool_args"] ["additional_command_args"]) From b29b53af2024a4b849a4e96f3e77e24f5fba7be7 Mon Sep 17 00:00:00 2001 From: Ynbot Date: Wed, 1 Mar 2023 03:32:10 +0000 Subject: [PATCH 41/57] [Automated] Bump version --- openpype/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/version.py b/openpype/version.py index bd1ba5309d..4d6f3d43e4 100644 --- a/openpype/version.py +++ b/openpype/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring Pype version.""" -__version__ = "3.15.2-nightly.2" +__version__ = "3.15.2-nightly.3" From f4d22e6ac671245dab4c4f6831ea6f057ca088c6 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Fri, 24 Feb 2023 11:44:41 +0000 Subject: [PATCH 42/57] Options for VrayProxy output. --- .../maya/plugins/create/create_vrayproxy.py | 6 ++++ .../maya/plugins/publish/collect_vrayproxy.py | 8 ++++- .../plugins/publish/extract_pointcache.py | 2 +- .../maya/plugins/publish/extract_vrayproxy.py | 2 +- .../plugins/publish/validate_vrayproxy.py | 32 ++++++++++------- .../defaults/project_settings/maya.json | 16 +++++---- .../schemas/schema_maya_create.json | 34 ++++++++++++++++--- 7 files changed, 73 insertions(+), 27 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_vrayproxy.py b/openpype/hosts/maya/plugins/create/create_vrayproxy.py index 5c0365b495..d135073e82 100644 --- a/openpype/hosts/maya/plugins/create/create_vrayproxy.py +++ b/openpype/hosts/maya/plugins/create/create_vrayproxy.py @@ -9,6 +9,9 @@ class CreateVrayProxy(plugin.Creator): family = "vrayproxy" icon = "gears" + vrmesh = True + alembic = True + def __init__(self, *args, **kwargs): super(CreateVrayProxy, self).__init__(*args, **kwargs) @@ -18,3 +21,6 @@ class CreateVrayProxy(plugin.Creator): # Write vertex colors self.data["vertexColors"] = False + + self.data["vrmesh"] = self.vrmesh + self.data["alembic"] = self.alembic diff --git a/openpype/hosts/maya/plugins/publish/collect_vrayproxy.py b/openpype/hosts/maya/plugins/publish/collect_vrayproxy.py index 236797ca3c..24521a2f09 100644 --- a/openpype/hosts/maya/plugins/publish/collect_vrayproxy.py +++ b/openpype/hosts/maya/plugins/publish/collect_vrayproxy.py @@ -9,10 +9,16 @@ class CollectVrayProxy(pyblish.api.InstancePlugin): Add `pointcache` family for it. """ order = pyblish.api.CollectorOrder + 0.01 - label = 'Collect Vray Proxy' + label = "Collect Vray Proxy" families = ["vrayproxy"] def process(self, instance): """Collector entry point.""" if not instance.data.get('families'): instance.data["families"] = [] + + if instance.data.get("vrmesh"): + instance.data["families"].append("vrayproxy.vrmesh") + + if instance.data.get("alembic"): + instance.data["families"].append("vrayproxy.alembic") diff --git a/openpype/hosts/maya/plugins/publish/extract_pointcache.py b/openpype/hosts/maya/plugins/publish/extract_pointcache.py index a3b0560099..f44c13767c 100644 --- a/openpype/hosts/maya/plugins/publish/extract_pointcache.py +++ b/openpype/hosts/maya/plugins/publish/extract_pointcache.py @@ -23,7 +23,7 @@ class ExtractAlembic(publish.Extractor): label = "Extract Pointcache (Alembic)" hosts = ["maya"] - families = ["pointcache", "model", "vrayproxy"] + families = ["pointcache", "model", "vrayproxy.alembic"] targets = ["local", "remote"] def process(self, instance): diff --git a/openpype/hosts/maya/plugins/publish/extract_vrayproxy.py b/openpype/hosts/maya/plugins/publish/extract_vrayproxy.py index 38bf02245a..9b10d2737d 100644 --- a/openpype/hosts/maya/plugins/publish/extract_vrayproxy.py +++ b/openpype/hosts/maya/plugins/publish/extract_vrayproxy.py @@ -16,7 +16,7 @@ class ExtractVRayProxy(publish.Extractor): label = "VRay Proxy (.vrmesh)" hosts = ["maya"] - families = ["vrayproxy"] + families = ["vrayproxy.vrmesh"] def process(self, instance): diff --git a/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py b/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py index 3eceace76d..075eedc378 100644 --- a/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py +++ b/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py @@ -1,27 +1,33 @@ import pyblish.api +from openpype.pipeline import KnownPublishError + class ValidateVrayProxy(pyblish.api.InstancePlugin): order = pyblish.api.ValidatorOrder - label = 'VRay Proxy Settings' - hosts = ['maya'] - families = ['studio.vrayproxy'] + label = "VRay Proxy Settings" + hosts = ["maya"] + families = ["vrayproxy"] def process(self, instance): - - invalid = self.get_invalid(instance) - if invalid: - raise RuntimeError("'%s' has invalid settings for VRay Proxy " - "export!" % instance.name) - - @classmethod - def get_invalid(cls, instance): data = instance.data + self.log.info(data["family"]) + self.log.info(data["families"]) if not data["setMembers"]: - cls.log.error("'%s' is empty! This is a bug" % instance.name) + raise KnownPublishError( + "'%s' is empty! This is a bug" % instance.name + ) if data["animation"]: if data["frameEnd"] < data["frameStart"]: - cls.log.error("End frame is smaller than start frame") + raise KnownPublishError( + "End frame is smaller than start frame" + ) + + if not data["vrmesh"] and not data["alembic"]: + raise KnownPublishError( + "Both vrmesh and alembic are off. Needs at least one to" + " publish." + ) diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 2559448900..90334a6644 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -199,6 +199,14 @@ "maskColor_manager": false, "maskOperator": false }, + "CreateVrayProxy": { + "enabled": true, + "vrmesh": true, + "alembic": true, + "defaults": [ + "Main" + ] + }, "CreateMultiverseUsd": { "enabled": true, "defaults": [ @@ -268,12 +276,6 @@ "Anim" ] }, - "CreateVrayProxy": { - "enabled": true, - "defaults": [ - "Main" - ] - }, "CreateVRayScene": { "enabled": true, "defaults": [ @@ -676,7 +678,7 @@ "families": [ "pointcache", "model", - "vrayproxy" + "vrayproxy.alembic" ] }, "ExtractObj": { diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json index 77a39f692f..49503cce83 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json @@ -332,6 +332,36 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "CreateVrayProxy", + "label": "Create VRay Proxy", + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "boolean", + "key": "vrmesh", + "label": "VrMesh" + }, + { + "type": "boolean", + "key": "alembic", + "label": "Alembic" + }, + { + "type": "list", + "key": "defaults", + "label": "Default Subsets", + "object_type": "text" + } + ] + }, { "type": "schema_template", "name": "template_create_plugin", @@ -380,10 +410,6 @@ "key": "CreateSetDress", "label": "Create Set Dress" }, - { - "key": "CreateVrayProxy", - "label": "Create VRay Proxy" - }, { "key": "CreateVRayScene", "label": "Create VRay Scene" From 65e0273f1c93dee1bd359d740d3206b4c4472548 Mon Sep 17 00:00:00 2001 From: Toke Jepsen Date: Tue, 28 Feb 2023 07:22:09 +0000 Subject: [PATCH 43/57] Update openpype/hosts/maya/plugins/publish/validate_vrayproxy.py --- openpype/hosts/maya/plugins/publish/validate_vrayproxy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py b/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py index 075eedc378..a106b970b4 100644 --- a/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py +++ b/openpype/hosts/maya/plugins/publish/validate_vrayproxy.py @@ -12,8 +12,6 @@ class ValidateVrayProxy(pyblish.api.InstancePlugin): def process(self, instance): data = instance.data - self.log.info(data["family"]) - self.log.info(data["families"]) if not data["setMembers"]: raise KnownPublishError( From 4f94a4454ab254018c189c8aa53c21fb12b1392d Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Wed, 1 Mar 2023 16:02:12 +0000 Subject: [PATCH 44/57] Only run Maya specific code in Maya. --- .../plugins/publish/submit_publish_job.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 5325715e38..29f6f406df 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -933,15 +933,18 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): self.log.info(data.get("expectedFiles")) - additional_data = { - "renderProducts": instance.data["renderProducts"], - "colorspaceConfig": instance.data["colorspaceConfig"], - "display": instance.data["colorspaceDisplay"], - "view": instance.data["colorspaceView"], - "colorspaceTemplate": instance.data["colorspaceConfig"].replace( - str(context.data["anatomy"].roots["work"]), "{root[work]}" - ) - } + additional_data = {} + if pyblish.api.current_host() == "maya": + config = instance.data["colorspaceConfig"] + additional_data = { + "renderProducts": instance.data["renderProducts"], + "colorspaceConfig": instance.data["colorspaceConfig"], + "display": instance.data["colorspaceDisplay"], + "view": instance.data["colorspaceView"], + "colorspaceTemplate": config.replace( + str(context.data["anatomy"].roots["work"]), "{root[work]}" + ) + } if isinstance(data.get("expectedFiles")[0], dict): # we cannot attach AOVs to other subsets as we consider every From a3508b14122d6ab884a4303d636bdf37b35ca973 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 2 Mar 2023 08:18:47 +0000 Subject: [PATCH 45/57] Fix _get_representations --- openpype/modules/deadline/plugins/publish/submit_publish_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 29f6f406df..adfbcbded8 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -588,7 +588,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): self.log.debug("instances:{}".format(instances)) return instances - def _get_representations(self, instance, exp_files, additional_data): + def _get_representations(self, instance, exp_files): """Create representations for file sequences. This will return representations of expected files if they are not From 91685e3d1fd3b43677fc33a537c3d93a5e8920cb Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 2 Mar 2023 11:06:47 +0000 Subject: [PATCH 46/57] Move AOV code to host agnostic. --- .../deadline/plugins/publish/submit_publish_job.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index adfbcbded8..31df4746ba 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -933,8 +933,10 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): self.log.info(data.get("expectedFiles")) - additional_data = {} - if pyblish.api.current_host() == "maya": + if isinstance(data.get("expectedFiles")[0], dict): + # we cannot attach AOVs to other subsets as we consider every + # AOV subset of its own. + config = instance.data["colorspaceConfig"] additional_data = { "renderProducts": instance.data["renderProducts"], @@ -946,10 +948,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): ) } - if isinstance(data.get("expectedFiles")[0], dict): - # we cannot attach AOVs to other subsets as we consider every - # AOV subset of its own. - if len(data.get("attachTo")) > 0: assert len(data.get("expectedFiles")[0].keys()) == 1, ( "attaching multiple AOVs or renderable cameras to " From 185623ff702a3ddc58038a4368e69e5b3ce4cc94 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Fri, 24 Feb 2023 16:21:00 +0000 Subject: [PATCH 47/57] Set frame range with handles on review instance. --- openpype/hosts/maya/api/lib.py | 32 +++++++++++++------ .../maya/plugins/create/create_review.py | 6 ++-- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 4324d321dc..0d9733fcf7 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -4,7 +4,6 @@ import os import sys import platform import uuid -import math import re import json @@ -2064,13 +2063,8 @@ def set_scene_resolution(width, height, pixelAspect): cmds.setAttr("%s.pixelAspect" % control_node, pixelAspect) -def reset_frame_range(): - """Set frame range to current asset""" - - fps = convert_to_maya_fps( - float(legacy_io.Session.get("AVALON_FPS", 25)) - ) - set_scene_fps(fps) +def get_frame_range(): + """Get the current assets frame range and handles.""" # Set frame start/end project_name = legacy_io.active_project() @@ -2097,8 +2091,26 @@ def reset_frame_range(): if handle_end is None: handle_end = handles - frame_start -= int(handle_start) - frame_end += int(handle_end) + return { + "frameStart": frame_start, + "frameEnd": frame_end, + "handleStart": handle_start, + "handleEnd": handle_end + } + + +def reset_frame_range(): + """Set frame range to current asset""" + + fps = convert_to_maya_fps( + float(legacy_io.Session.get("AVALON_FPS", 25)) + ) + set_scene_fps(fps) + + frame_range = get_frame_range() + + frame_start = frame_range["frameStart"] - int(frame_range["handleStart"]) + frame_end = frame_range["frameEnd"] + int(frame_range["handleEnd"]) cmds.playbackOptions(minTime=frame_start) cmds.playbackOptions(maxTime=frame_end) diff --git a/openpype/hosts/maya/plugins/create/create_review.py b/openpype/hosts/maya/plugins/create/create_review.py index ba51ffa009..6e0bd2e4c3 100644 --- a/openpype/hosts/maya/plugins/create/create_review.py +++ b/openpype/hosts/maya/plugins/create/create_review.py @@ -28,13 +28,13 @@ class CreateReview(plugin.Creator): def __init__(self, *args, **kwargs): super(CreateReview, self).__init__(*args, **kwargs) + data = OrderedDict(**self.data) # get basic animation data : start / end / handles / steps - data = OrderedDict(**self.data) - animation_data = lib.collect_animation_data(fps=True) - for key, value in animation_data.items(): + for key, value in lib.get_frame_range().items(): data[key] = value + data["fps"] = lib.collect_animation_data(fps=True)["fps"] data["review_width"] = self.Width data["review_height"] = self.Height data["isolate"] = self.isolate From db0a3554b62afcfc03a8a33563e20a9a935bef22 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Fri, 24 Feb 2023 16:42:01 +0000 Subject: [PATCH 48/57] Validate frame range on instance to asset. - frame start - frame end - handle start - handle end --- .../plugins/publish/validate_frame_range.py | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/validate_frame_range.py b/openpype/hosts/maya/plugins/publish/validate_frame_range.py index d86925184e..dbf856a30a 100644 --- a/openpype/hosts/maya/plugins/publish/validate_frame_range.py +++ b/openpype/hosts/maya/plugins/publish/validate_frame_range.py @@ -57,6 +57,10 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): inst_start = int(instance.data.get("frameStartHandle")) inst_end = int(instance.data.get("frameEndHandle")) + inst_frame_start = int(instance.data.get("frameStart")) + inst_frame_end = int(instance.data.get("frameEnd")) + inst_handle_start = int(instance.data.get("handleStart")) + inst_handle_end = int(instance.data.get("handleEnd")) # basic sanity checks assert frame_start_handle <= frame_end_handle, ( @@ -69,7 +73,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): if [ef for ef in self.exclude_families if instance.data["family"] in ef]: return - if(inst_start != frame_start_handle): + if (inst_start != frame_start_handle): errors.append("Instance start frame [ {} ] doesn't " "match the one set on instance [ {} ]: " "{}/{}/{}/{} (handle/start/end/handle)".format( @@ -78,7 +82,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): handle_start, frame_start, frame_end, handle_end )) - if(inst_end != frame_end_handle): + if (inst_end != frame_end_handle): errors.append("Instance end frame [ {} ] doesn't " "match the one set on instance [ {} ]: " "{}/{}/{}/{} (handle/start/end/handle)".format( @@ -87,6 +91,19 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): handle_start, frame_start, frame_end, handle_end )) + checks = { + "frame start": (frame_start, inst_frame_start), + "frame end": (frame_end, inst_frame_end), + "handle start": (handle_start, inst_handle_start), + "handle end": (handle_end, inst_handle_end) + } + for label, values in checks.items(): + if values[0] != values[1]: + errors.append( + "{} on instance ({}) does not match with the asset " + "({}).".format(label.title(), values[1], values[0]) + ) + for e in errors: self.log.error(e) From d33aa1cc7df5eb7ae73320693b3a9b00183b3d70 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 28 Feb 2023 07:26:50 +0000 Subject: [PATCH 49/57] Better error reports. --- openpype/hosts/maya/plugins/publish/validate_frame_range.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/validate_frame_range.py b/openpype/hosts/maya/plugins/publish/validate_frame_range.py index dbf856a30a..59b06874b3 100644 --- a/openpype/hosts/maya/plugins/publish/validate_frame_range.py +++ b/openpype/hosts/maya/plugins/publish/validate_frame_range.py @@ -75,7 +75,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): return if (inst_start != frame_start_handle): errors.append("Instance start frame [ {} ] doesn't " - "match the one set on instance [ {} ]: " + "match the one set on asset [ {} ]: " "{}/{}/{}/{} (handle/start/end/handle)".format( inst_start, frame_start_handle, @@ -84,7 +84,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): if (inst_end != frame_end_handle): errors.append("Instance end frame [ {} ] doesn't " - "match the one set on instance [ {} ]: " + "match the one set on asset [ {} ]: " "{}/{}/{}/{} (handle/start/end/handle)".format( inst_end, frame_end_handle, From 7bbf5608711c865f3f50c82f1fbd34ddc679982a Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 28 Feb 2023 07:42:58 +0000 Subject: [PATCH 50/57] Backwards compatibility --- .../maya/plugins/create/create_review.py | 8 +++-- .../defaults/project_settings/maya.json | 13 +++++---- .../schemas/schema_maya_create.json | 29 ++++++++++++++++--- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/openpype/hosts/maya/plugins/create/create_review.py b/openpype/hosts/maya/plugins/create/create_review.py index 6e0bd2e4c3..f1b626c06b 100644 --- a/openpype/hosts/maya/plugins/create/create_review.py +++ b/openpype/hosts/maya/plugins/create/create_review.py @@ -25,13 +25,17 @@ class CreateReview(plugin.Creator): "depth peeling", "alpha cut" ] + useMayaTimeline = True def __init__(self, *args, **kwargs): super(CreateReview, self).__init__(*args, **kwargs) data = OrderedDict(**self.data) - # get basic animation data : start / end / handles / steps - for key, value in lib.get_frame_range().items(): + # Option for using Maya or asset frame range in settings. + frame_range = lib.get_frame_range() + if self.useMayaTimeline: + frame_range = lib.collect_animation_data(fps=True) + for key, value in frame_range.items(): data[key] = value data["fps"] = lib.collect_animation_data(fps=True)["fps"] diff --git a/openpype/settings/defaults/project_settings/maya.json b/openpype/settings/defaults/project_settings/maya.json index 90334a6644..dca0b95293 100644 --- a/openpype/settings/defaults/project_settings/maya.json +++ b/openpype/settings/defaults/project_settings/maya.json @@ -179,6 +179,13 @@ "Main" ] }, + "CreateReview": { + "enabled": true, + "defaults": [ + "Main" + ], + "useMayaTimeline": true + }, "CreateAss": { "enabled": true, "defaults": [ @@ -255,12 +262,6 @@ "Main" ] }, - "CreateReview": { - "enabled": true, - "defaults": [ - "Main" - ] - }, "CreateRig": { "enabled": true, "defaults": [ diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json index 49503cce83..1598f90643 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_maya_create.json @@ -240,6 +240,31 @@ } ] }, + { + "type": "dict", + "collapsible": true, + "key": "CreateReview", + "label": "Create Review", + "checkbox_key": "enabled", + "children": [ + { + "type": "boolean", + "key": "enabled", + "label": "Enabled" + }, + { + "type": "list", + "key": "defaults", + "label": "Default Subsets", + "object_type": "text" + }, + { + "type": "boolean", + "key": "useMayaTimeline", + "label": "Use Maya Timeline for Frame Range." + } + ] + }, { "type": "dict", "collapsible": true, @@ -398,10 +423,6 @@ "key": "CreateRenderSetup", "label": "Create Render Setup" }, - { - "key": "CreateReview", - "label": "Create Review" - }, { "key": "CreateRig", "label": "Create Rig" From db98f65b43517c1930ab3be11f56f0fa672e5c8d Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 23 Feb 2023 16:23:21 +0000 Subject: [PATCH 51/57] Fix publish pool and secondary. --- .../modules/deadline/plugins/publish/submit_publish_job.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 31df4746ba..53c09ad22f 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -284,6 +284,9 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): args.append("--automatic-tests") # Generate the payload for Deadline submission + secondary_pool = ( + self.deadline_pool_secondary or instance.data.get("secondaryPool") + ) payload = { "JobInfo": { "Plugin": self.deadline_plugin, @@ -297,8 +300,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "Priority": priority, "Group": self.deadline_group, - "Pool": instance.data.get("primaryPool"), - "SecondaryPool": instance.data.get("secondaryPool"), + "Pool": self.deadline_pool or instance.data.get("primaryPool"), + "SecondaryPool": secondary_pool, # ensure the outputdirectory with correct slashes "OutputDirectory0": output_dir.replace("\\", "/") }, From d827ffa8fbabd6fb02dc03e123e22d69c5b87c24 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 23 Feb 2023 16:23:38 +0000 Subject: [PATCH 52/57] Use publish pool for tile jobs. --- .../deadline/plugins/publish/submit_maya_deadline.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py index 22b5c02296..15025e47f2 100644 --- a/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_maya_deadline.py @@ -419,8 +419,13 @@ class MayaSubmitDeadline(abstract_submit_deadline.AbstractSubmitDeadline): assembly_job_info.Name += " - Tile Assembly Job" assembly_job_info.Frames = 1 assembly_job_info.MachineLimit = 1 - assembly_job_info.Priority = instance.data.get("tile_priority", - self.tile_priority) + assembly_job_info.Priority = instance.data.get( + "tile_priority", self.tile_priority + ) + + pool = instance.context.data["project_settings"]["deadline"] + pool = pool["publish"]["ProcessSubmittedJobOnFarm"]["deadline_pool"] + assembly_job_info.Pool = pool or instance.data.get("primaryPool", "") assembly_plugin_info = { "CleanupTiles": 1, From 9117a0d6329c5cf47aee68bf4fd2e57dac5fff6a Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 23 Feb 2023 16:39:10 +0000 Subject: [PATCH 53/57] Documentation --- website/docs/module_deadline.md | 34 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/website/docs/module_deadline.md b/website/docs/module_deadline.md index c96da91909..c4b179b98d 100644 --- a/website/docs/module_deadline.md +++ b/website/docs/module_deadline.md @@ -28,16 +28,16 @@ For [AWS Thinkbox Deadline](https://www.awsthinkbox.com/deadline) support you ne OpenPype integration for Deadline consists of two parts: - The `OpenPype` Deadline Plug-in -- A `GlobalJobPreLoad` Deadline Script (this gets triggered for each deadline job) +- A `GlobalJobPreLoad` Deadline Script (this gets triggered for each deadline job) The `GlobalJobPreLoad` handles populating render and publish jobs with proper environment variables using settings from the `OpenPype` Deadline Plug-in. -The `OpenPype` Deadline Plug-in must be configured to point to a valid OpenPype executable location. The executable need to be installed to +The `OpenPype` Deadline Plug-in must be configured to point to a valid OpenPype executable location. The executable need to be installed to destinations accessible by DL process. Check permissions (must be executable and accessible by Deadline process) - Enable `Tools > Super User Mode` in Deadline Monitor -- Go to `Tools > Configure Plugins...`, find `OpenPype` in the list on the left side, find location of OpenPype +- Go to `Tools > Configure Plugins...`, find `OpenPype` in the list on the left side, find location of OpenPype executable. It is recommended to use the `openpype_console` executable as it provides a bit more logging. - In case of multi OS farms, provide multiple locations, each Deadline Worker goes through the list and tries to find the first accessible @@ -45,12 +45,22 @@ executable. It is recommended to use the `openpype_console` executable as it pro ![Configure plugin](assets/deadline_configure_plugin.png) +### Pools + +The main pools can be configured at `project_settings/deadline/publish/CollectDeadlinePools/primary_pool`, which is applied to the rendering jobs. + +The dependent publishing job's pool uses `project_settings/deadline/publish/ProcessSubmittedJobOnFarm/deadline_pool`. If nothing is specified the pool will fallback to the main pool above. + +:::note maya tile rendering +The logic for publishing job pool assignment applies to tiling jobs. +::: + ## Troubleshooting #### Publishing jobs fail directly in DCCs - Double check that all previously described steps were finished -- Check that `deadlinewebservice` is running on DL server +- Check that `deadlinewebservice` is running on DL server - Check that user's machine has access to deadline server on configured port #### Jobs are failing on DL side @@ -61,40 +71,40 @@ Each publishing from OpenPype consists of 2 jobs, first one is rendering, second - Jobs are failing with `OpenPype executable was not found` error - Check if OpenPype is installed on the Worker handling this job and ensure `OpenPype` Deadline Plug-in is properly [configured](#configuration) + Check if OpenPype is installed on the Worker handling this job and ensure `OpenPype` Deadline Plug-in is properly [configured](#configuration) - Publishing job is failing with `ffmpeg not installed` error - + OpenPype executable has to have access to `ffmpeg` executable, check OpenPype `Setting > General` ![FFmpeg setting](assets/ffmpeg_path.png) - Both jobs finished successfully, but there is no review on Ftrack - Make sure that you correctly set published family to be send to Ftrack. + Make sure that you correctly set published family to be send to Ftrack. ![Ftrack Family](assets/ftrack/ftrack-collect-main.png) Example: I want send to Ftrack review of rendered images from Harmony : - `Host names`: "harmony" - - `Families`: "render" + - `Families`: "render" - `Add Ftrack Family` to "Enabled" - + Make sure that you actually configured to create review for published subset in `project_settings/ftrack/publish/CollectFtrackFamily` ![Ftrack Family](assets/deadline_review.png) - Example: I want to create review for all reviewable subsets in Harmony : + Example: I want to create review for all reviewable subsets in Harmony : - Add "harmony" as a new key an ".*" as a value. - Rendering jobs are stuck in 'Queued' state or failing Make sure that your Deadline is not limiting specific jobs to be run only on specific machines. (Eg. only some machines have installed particular application.) - + Check `project_settings/deadline` - + ![Deadline group](assets/deadline_group.png) Example: I have separated machines with "Harmony" installed into "harmony" group on Deadline. I want rendering jobs published from Harmony to run only on those machines. From 6f5ba9ce2a98aed1777df57731ae02409cc474a2 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Thu, 23 Feb 2023 16:42:02 +0000 Subject: [PATCH 54/57] Docs --- website/docs/module_deadline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/module_deadline.md b/website/docs/module_deadline.md index c4b179b98d..ab1016788d 100644 --- a/website/docs/module_deadline.md +++ b/website/docs/module_deadline.md @@ -49,7 +49,7 @@ executable. It is recommended to use the `openpype_console` executable as it pro The main pools can be configured at `project_settings/deadline/publish/CollectDeadlinePools/primary_pool`, which is applied to the rendering jobs. -The dependent publishing job's pool uses `project_settings/deadline/publish/ProcessSubmittedJobOnFarm/deadline_pool`. If nothing is specified the pool will fallback to the main pool above. +The dependent publishing job's pool uses `project_settings/deadline/publish/ProcessSubmittedJobOnFarm/deadline_pool`. If nothing is specified the pool will fallback to the primary pool above. :::note maya tile rendering The logic for publishing job pool assignment applies to tiling jobs. From b61f1ed1df5a5a6dcab162ffe320a4093b5efd12 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 28 Feb 2023 08:17:29 +0000 Subject: [PATCH 55/57] Validate against zero polygon mesh. --- .../plugins/publish/validate_mesh_empty.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 openpype/hosts/maya/plugins/publish/validate_mesh_empty.py diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_empty.py b/openpype/hosts/maya/plugins/publish/validate_mesh_empty.py new file mode 100644 index 0000000000..848d66c4ae --- /dev/null +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_empty.py @@ -0,0 +1,54 @@ +from maya import cmds + +import pyblish.api +import openpype.hosts.maya.api.action +from openpype.pipeline.publish import ( + RepairAction, + ValidateMeshOrder +) + + +class ValidateMeshEmpty(pyblish.api.InstancePlugin): + """Validate meshes have some vertices. + + Its possible to have meshes without any vertices. To replicate + this issue, delete all faces/polygons then all edges. + """ + + order = ValidateMeshOrder + hosts = ["maya"] + families = ["model"] + label = "Mesh Empty" + actions = [ + openpype.hosts.maya.api.action.SelectInvalidAction, RepairAction + ] + + @classmethod + def repair(cls, instance): + invalid = cls.get_invalid(instance) + for node in invalid: + cmds.delete(node) + + @classmethod + def get_invalid(cls, instance): + invalid = [] + + meshes = cmds.ls(instance, type="mesh", long=True) + for mesh in meshes: + num_vertices = cmds.polyEvaluate(mesh, vertex=True) + + if num_vertices == 0: + cls.log.warning( + "\"{}\" does not have any vertices.".format(mesh) + ) + invalid.append(mesh) + + return invalid + + def process(self, instance): + + invalid = self.get_invalid(instance) + if invalid: + raise RuntimeError( + "Meshes found in instance without any vertices: %s" % invalid + ) From 206bf55a4909e603e845d0ca5b3fc46896f1ca90 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 28 Feb 2023 08:18:16 +0000 Subject: [PATCH 56/57] Refactor `len_flattened` --- openpype/hosts/maya/api/lib.py | 31 +++++++++++++++++ .../plugins/publish/validate_mesh_has_uv.py | 32 +---------------- .../validate_mesh_vertices_have_edges.py | 34 +------------------ 3 files changed, 33 insertions(+), 64 deletions(-) diff --git a/openpype/hosts/maya/api/lib.py b/openpype/hosts/maya/api/lib.py index 0d9733fcf7..954576f02e 100644 --- a/openpype/hosts/maya/api/lib.py +++ b/openpype/hosts/maya/api/lib.py @@ -3574,3 +3574,34 @@ def get_color_management_output_transform(): if preferences["output_transform_enabled"]: colorspace = preferences["output_transform"] return colorspace + + +def len_flattened(components): + """Return the length of the list as if it was flattened. + + Maya will return consecutive components as a single entry + when requesting with `maya.cmds.ls` without the `flatten` + flag. Though enabling `flatten` on a large list (e.g. millions) + will result in a slow result. This command will return the amount + of entries in a non-flattened list by parsing the result with + regex. + + Args: + components (list): The non-flattened components. + + Returns: + int: The amount of entries. + + """ + assert isinstance(components, (list, tuple)) + n = 0 + + pattern = re.compile(r"\[(\d+):(\d+)\]") + for c in components: + match = pattern.search(c) + if match: + start, end = match.groups() + n += int(end) - int(start) + 1 + else: + n += 1 + return n diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py b/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py index 0eece1014e..1775bd84c6 100644 --- a/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py @@ -1,39 +1,9 @@ -import re - from maya import cmds import pyblish.api import openpype.hosts.maya.api.action from openpype.pipeline.publish import ValidateMeshOrder - - -def len_flattened(components): - """Return the length of the list as if it was flattened. - - Maya will return consecutive components as a single entry - when requesting with `maya.cmds.ls` without the `flatten` - flag. Though enabling `flatten` on a large list (e.g. millions) - will result in a slow result. This command will return the amount - of entries in a non-flattened list by parsing the result with - regex. - - Args: - components (list): The non-flattened components. - - Returns: - int: The amount of entries. - - """ - assert isinstance(components, (list, tuple)) - n = 0 - for c in components: - match = re.search("\[([0-9]+):([0-9]+)\]", c) - if match: - start, end = match.groups() - n += int(end) - int(start) + 1 - else: - n += 1 - return n +from openpype.hosts.maya.api.lib import len_flattened class ValidateMeshHasUVs(pyblish.api.InstancePlugin): diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py b/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py index 9ac7735501..51e1ddfc7f 100644 --- a/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py @@ -1,5 +1,3 @@ -import re - from maya import cmds import pyblish.api @@ -8,37 +6,7 @@ from openpype.pipeline.publish import ( RepairAction, ValidateMeshOrder, ) - - -def len_flattened(components): - """Return the length of the list as if it was flattened. - - Maya will return consecutive components as a single entry - when requesting with `maya.cmds.ls` without the `flatten` - flag. Though enabling `flatten` on a large list (e.g. millions) - will result in a slow result. This command will return the amount - of entries in a non-flattened list by parsing the result with - regex. - - Args: - components (list): The non-flattened components. - - Returns: - int: The amount of entries. - - """ - assert isinstance(components, (list, tuple)) - n = 0 - - pattern = re.compile(r"\[(\d+):(\d+)\]") - for c in components: - match = pattern.search(c) - if match: - start, end = match.groups() - n += int(end) - int(start) + 1 - else: - n += 1 - return n +from openpype.hosts.maya.api.lib import len_flattened class ValidateMeshVerticesHaveEdges(pyblish.api.InstancePlugin): From 63177ca7e935cd87e716229831eaf1f456825e66 Mon Sep 17 00:00:00 2001 From: Toke Stuart Jepsen Date: Tue, 28 Feb 2023 08:18:50 +0000 Subject: [PATCH 57/57] Only mesh empty validator should fail. --- .../plugins/publish/validate_mesh_has_uv.py | 9 +++++++++ .../publish/validate_mesh_non_zero_edge.py | 20 +++++++++++++++++-- .../validate_mesh_vertices_have_edges.py | 7 +++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py b/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py index 1775bd84c6..b7836b3e92 100644 --- a/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_has_uv.py @@ -27,6 +27,15 @@ class ValidateMeshHasUVs(pyblish.api.InstancePlugin): invalid = [] for node in cmds.ls(instance, type='mesh'): + num_vertices = cmds.polyEvaluate(node, vertex=True) + + if num_vertices == 0: + cls.log.warning( + "Skipping \"{}\", cause it does not have any " + "vertices.".format(node) + ) + continue + uv = cmds.polyEvaluate(node, uv=True) if uv == 0: diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_non_zero_edge.py b/openpype/hosts/maya/plugins/publish/validate_mesh_non_zero_edge.py index 78e844d201..b49ba85648 100644 --- a/openpype/hosts/maya/plugins/publish/validate_mesh_non_zero_edge.py +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_non_zero_edge.py @@ -28,7 +28,10 @@ class ValidateMeshNonZeroEdgeLength(pyblish.api.InstancePlugin): @classmethod def get_invalid(cls, instance): """Return the invalid edges. - Also see: http://help.autodesk.com/view/MAYAUL/2015/ENU/?guid=Mesh__Cleanup + + Also see: + + http://help.autodesk.com/view/MAYAUL/2015/ENU/?guid=Mesh__Cleanup """ @@ -36,8 +39,21 @@ class ValidateMeshNonZeroEdgeLength(pyblish.api.InstancePlugin): if not meshes: return list() + valid_meshes = [] + for mesh in meshes: + num_vertices = cmds.polyEvaluate(mesh, vertex=True) + + if num_vertices == 0: + cls.log.warning( + "Skipping \"{}\", cause it does not have any " + "vertices.".format(mesh) + ) + continue + + valid_meshes.append(mesh) + # Get all edges - edges = ['{0}.e[*]'.format(node) for node in meshes] + edges = ['{0}.e[*]'.format(node) for node in valid_meshes] # Filter by constraint on edge length invalid = lib.polyConstraint(edges, diff --git a/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py b/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py index 51e1ddfc7f..d885158004 100644 --- a/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py +++ b/openpype/hosts/maya/plugins/publish/validate_mesh_vertices_have_edges.py @@ -55,6 +55,13 @@ class ValidateMeshVerticesHaveEdges(pyblish.api.InstancePlugin): for mesh in meshes: num_vertices = cmds.polyEvaluate(mesh, vertex=True) + if num_vertices == 0: + cls.log.warning( + "Skipping \"{}\", cause it does not have any " + "vertices.".format(mesh) + ) + continue + # Vertices from all edges edges = "%s.e[*]" % mesh vertices = cmds.polyListComponentConversion(edges, toVertex=True)