From 2d8c41cc6e6e53ecc21372f75bfd590397a33a28 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Thu, 30 Jun 2022 10:39:05 +0200 Subject: [PATCH] moved render abstractions to openpype.pipeline.publish --- .../plugins/publish/collect_render.py | 6 +- .../plugins/publish/collect_farm_render.py | 8 +- openpype/lib/abstract_collect_render.py | 280 ++---------------- openpype/lib/abstract_expected_files.py | 71 ++--- .../deadline/abstract_submit_deadline.py | 2 +- openpype/pipeline/publish/__init__.py | 11 + .../publish/abstract_collect_render.py | 268 +++++++++++++++++ .../publish/abstract_expected_files.py | 53 ++++ 8 files changed, 386 insertions(+), 313 deletions(-) create mode 100644 openpype/pipeline/publish/abstract_collect_render.py create mode 100644 openpype/pipeline/publish/abstract_expected_files.py diff --git a/openpype/hosts/aftereffects/plugins/publish/collect_render.py b/openpype/hosts/aftereffects/plugins/publish/collect_render.py index 97b3175c57..bb199a61f7 100644 --- a/openpype/hosts/aftereffects/plugins/publish/collect_render.py +++ b/openpype/hosts/aftereffects/plugins/publish/collect_render.py @@ -6,8 +6,8 @@ import attr import pyblish.api from openpype.settings import get_project_settings -from openpype.lib import abstract_collect_render -from openpype.lib.abstract_collect_render import RenderInstance +from openpype.pipeline import publish +from openpype.pipeline.publish import RenderInstance from openpype.hosts.aftereffects.api import get_stub @@ -25,7 +25,7 @@ class AERenderInstance(RenderInstance): file_name = attr.ib(default=None) -class CollectAERender(abstract_collect_render.AbstractCollectRender): +class CollectAERender(publish.AbstractCollectRender): order = pyblish.api.CollectorOrder + 0.405 label = "Collect After Effects Render Layers" diff --git a/openpype/hosts/harmony/plugins/publish/collect_farm_render.py b/openpype/hosts/harmony/plugins/publish/collect_farm_render.py index 3e9e680efd..f6b26eb3e8 100644 --- a/openpype/hosts/harmony/plugins/publish/collect_farm_render.py +++ b/openpype/hosts/harmony/plugins/publish/collect_farm_render.py @@ -4,11 +4,10 @@ from pathlib import Path import attr -import openpype.lib -import openpype.lib.abstract_collect_render -from openpype.lib.abstract_collect_render import RenderInstance from openpype.lib import get_formatted_current_time from openpype.pipeline import legacy_io +from openpype.pipeline import publish +from openpype.pipeline.publish import RenderInstance import openpype.hosts.harmony.api as harmony @@ -20,8 +19,7 @@ class HarmonyRenderInstance(RenderInstance): leadingZeros = attr.ib(default=3) -class CollectFarmRender(openpype.lib.abstract_collect_render. - AbstractCollectRender): +class CollectFarmRender(publish.AbstractCollectRender): """Gather all publishable renders.""" # https://docs.toonboom.com/help/harmony-17/premium/reference/node/output/write-node-image-formats.html diff --git a/openpype/lib/abstract_collect_render.py b/openpype/lib/abstract_collect_render.py index 3d81f6d794..2cc1c23822 100644 --- a/openpype/lib/abstract_collect_render.py +++ b/openpype/lib/abstract_collect_render.py @@ -1,269 +1,33 @@ # -*- coding: utf-8 -*- -"""Collect render template. +"""Content was moved to 'openpype.pipeline.farm.abstract_collect_render'. -TODO: use @dataclass when times come. +Please change your imports as soon as possible. +File will be probably removed in OpenPype 3.14.* """ -from abc import abstractmethod -import attr -import six - -import pyblish.api - -from openpype.pipeline import legacy_io - -from .abstract_metaplugins import AbstractMetaContextPlugin +import warnings +from openpype.pipeline.publish import AbstractCollectRender, RenderInstance -@attr.s -class RenderInstance(object): - """Data collected by collectors. - - This data class later on passed to collected instances. - Those attributes are required later on. - - """ - - # metadata - version = attr.ib() # instance version - time = attr.ib() # time of instance creation (get_formatted_current_time) - source = attr.ib() # path to source scene file - label = attr.ib() # label to show in GUI - subset = attr.ib() # subset name - task = attr.ib() # task name - asset = attr.ib() # asset name (AVALON_ASSET) - attachTo = attr.ib() # subset name to attach render to - setMembers = attr.ib() # list of nodes/members producing render output - publish = attr.ib() # bool, True to publish instance - name = attr.ib() # instance name - - # format settings - resolutionWidth = attr.ib() # resolution width (1920) - resolutionHeight = attr.ib() # resolution height (1080) - pixelAspect = attr.ib() # pixel aspect (1.0) - - # time settings - frameStart = attr.ib() # start frame - frameEnd = attr.ib() # start end - frameStep = attr.ib() # frame step - - handleStart = attr.ib(default=None) # start frame - handleEnd = attr.ib(default=None) # start frame - - # for software (like Harmony) where frame range cannot be set by DB - # handles need to be propagated if exist - ignoreFrameHandleCheck = attr.ib(default=False) - - # -------------------- - # With default values - # metadata - renderer = attr.ib(default="") # renderer - can be used in Deadline - review = attr.ib(default=False) # generate review from instance (bool) - priority = attr.ib(default=50) # job priority on farm - - family = attr.ib(default="renderlayer") - families = attr.ib(default=["renderlayer"]) # list of families - - # format settings - multipartExr = attr.ib(default=False) # flag for multipart exrs - convertToScanline = attr.ib(default=False) # flag for exr conversion - - tileRendering = attr.ib(default=False) # bool: treat render as tiles - tilesX = attr.ib(default=0) # number of tiles in X - tilesY = attr.ib(default=0) # number of tiles in Y - - # submit_publish_job - toBeRenderedOn = attr.ib(default=None) - deadlineSubmissionJob = attr.ib(default=None) - anatomyData = attr.ib(default=None) - outputDir = attr.ib(default=None) - context = attr.ib(default=None) - - @frameStart.validator - def check_frame_start(self, _, value): - """Validate if frame start is not larger then end.""" - if value > self.frameEnd: - raise ValueError("frameStart must be smaller " - "or equal then frameEnd") - - @frameEnd.validator - def check_frame_end(self, _, value): - """Validate if frame end is not less then start.""" - if value < self.frameStart: - raise ValueError("frameEnd must be smaller " - "or equal then frameStart") - - @tilesX.validator - def check_tiles_x(self, _, value): - """Validate if tile x isn't less then 1.""" - if not self.tileRendering: - return - if value < 1: - raise ValueError("tile X size cannot be less then 1") - - if value == 1 and self.tilesY == 1: - raise ValueError("both tiles X a Y sizes are set to 1") - - @tilesY.validator - def check_tiles_y(self, _, value): - """Validate if tile y isn't less then 1.""" - if not self.tileRendering: - return - if value < 1: - raise ValueError("tile Y size cannot be less then 1") - - if value == 1 and self.tilesX == 1: - raise ValueError("both tiles X a Y sizes are set to 1") +class CollectRenderDeprecated(DeprecationWarning): + pass -@six.add_metaclass(AbstractMetaContextPlugin) -class AbstractCollectRender(pyblish.api.ContextPlugin): - """Gather all publishable render layers from renderSetup.""" +warnings.simplefilter("always", CollectRenderDeprecated) +warnings.warn( + ( + "Content of 'abstract_collect_render' was moved." + "\nUsing deprecated source of 'abstract_collect_render'. Content was" + " move to 'openpype.pipeline.farm.abstract_collect_render'." + " Please change your imports as soon as possible." + ), + category=CollectRenderDeprecated, + stacklevel=4 +) - order = pyblish.api.CollectorOrder + 0.01 - label = "Collect Render" - sync_workfile_version = False - def __init__(self, *args, **kwargs): - """Constructor.""" - super(AbstractCollectRender, self).__init__(*args, **kwargs) - self._file_path = None - self._asset = legacy_io.Session["AVALON_ASSET"] - self._context = None - - def process(self, context): - """Entry point to collector.""" - self._context = context - for instance in context: - # make sure workfile instance publishing is enabled - try: - if "workfile" in instance.data["families"]: - instance.data["publish"] = True - # TODO merge renderFarm and render.farm - if ("renderFarm" in instance.data["families"] or - "render.farm" in instance.data["families"]): - instance.data["remove"] = True - except KeyError: - # be tolerant if 'families' is missing. - pass - - self._file_path = context.data["currentFile"].replace("\\", "/") - - render_instances = self.get_instances(context) - for render_instance in render_instances: - exp_files = self.get_expected_files(render_instance) - assert exp_files, "no file names were generated, this is bug" - - # if we want to attach render to subset, check if we have AOV's - # in expectedFiles. If so, raise error as we cannot attach AOV - # (considered to be subset on its own) to another subset - if render_instance.attachTo: - assert isinstance(exp_files, list), ( - "attaching multiple AOVs or renderable cameras to " - "subset is not supported" - ) - - frame_start_render = int(render_instance.frameStart) - frame_end_render = int(render_instance.frameEnd) - if (render_instance.ignoreFrameHandleCheck or - int(context.data['frameStartHandle']) == frame_start_render - and int(context.data['frameEndHandle']) == frame_end_render): # noqa: W503, E501 - - handle_start = context.data['handleStart'] - handle_end = context.data['handleEnd'] - frame_start = context.data['frameStart'] - frame_end = context.data['frameEnd'] - frame_start_handle = context.data['frameStartHandle'] - frame_end_handle = context.data['frameEndHandle'] - else: - handle_start = 0 - handle_end = 0 - frame_start = frame_start_render - frame_end = frame_end_render - frame_start_handle = frame_start_render - frame_end_handle = frame_end_render - - data = { - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStart": frame_start, - "frameEnd": frame_end, - "frameStartHandle": frame_start_handle, - "frameEndHandle": frame_end_handle, - "byFrameStep": int(render_instance.frameStep), - - "author": context.data["user"], - # Add source to allow tracing back to the scene from - # which was submitted originally - "expectedFiles": exp_files, - } - if self.sync_workfile_version: - data["version"] = context.data["version"] - - # add additional data - data = self.add_additional_data(data) - render_instance_dict = attr.asdict(render_instance) - - instance = context.create_instance(render_instance.name) - instance.data["label"] = render_instance.label - instance.data.update(render_instance_dict) - instance.data.update(data) - - self.post_collecting_action() - - @abstractmethod - def get_instances(self, context): - """Get all renderable instances and their data. - - Args: - context (pyblish.api.Context): Context object. - - Returns: - list of :class:`RenderInstance`: All collected renderable instances - (like render layers, write nodes, etc.) - - """ - pass - - @abstractmethod - def get_expected_files(self, render_instance): - """Get list of expected files. - - Returns: - list: expected files. This can be either simple list of files with - their paths, or list of dictionaries, where key is name of AOV - for example and value is list of files for that AOV. - - Example:: - - ['/path/to/file.001.exr', '/path/to/file.002.exr'] - - or as dictionary: - - [ - { - "beauty": ['/path/to/beauty.001.exr', ...], - "mask": ['/path/to/mask.001.exr'] - } - ] - - """ - pass - - def add_additional_data(self, data): - """Add additional data to collected instance. - - This can be overridden by host implementation to add custom - additional data. - - """ - return data - - def post_collecting_action(self): - """Execute some code after collection is done. - - This is useful for example for restoring current render layer. - - """ - pass +__all__ = ( + "AbstractCollectRender", + "RenderInstance" +) diff --git a/openpype/lib/abstract_expected_files.py b/openpype/lib/abstract_expected_files.py index f9f3c17ef5..bb433a8b11 100644 --- a/openpype/lib/abstract_expected_files.py +++ b/openpype/lib/abstract_expected_files.py @@ -1,53 +1,32 @@ # -*- coding: utf-8 -*- -"""Abstract ExpectedFile class definition.""" -from abc import ABCMeta, abstractmethod -import six +"""Content was moved to 'openpype.pipeline.farm.abstract_expected_files'. + +Please change your imports as soon as possible. + +File will be probably removed in OpenPype 3.14.* +""" + +import warnings +from openpype.pipeline.publish import ExpectedFiles -@six.add_metaclass(ABCMeta) -class ExpectedFiles: - """Class grouping functionality for all supported renderers. - - Attributes: - multipart (bool): Flag if multipart exrs are used. - - """ - - multipart = False - - @abstractmethod - def get(self, render_instance): - """Get expected files for given renderer and render layer. - - This method should return dictionary of all files we are expecting - to be rendered from the host. Usually `render_instance` corresponds - to *render layer*. Result can be either flat list with the file - paths or it can be list of dictionaries. Each key corresponds to - for example AOV name or channel, etc. - - Example:: - - ['/path/to/file.001.exr', '/path/to/file.002.exr'] - - or as dictionary: - - [ - { - "beauty": ['/path/to/beauty.001.exr', ...], - "mask": ['/path/to/mask.001.exr'] - } - ] +class ExpectedFilesDeprecated(DeprecationWarning): + pass - Args: - render_instance (:class:`RenderInstance`): Data passed from - collector to determine files. This should be instance of - :class:`abstract_collect_render.RenderInstance` +warnings.simplefilter("always", ExpectedFilesDeprecated) +warnings.warn( + ( + "Content of 'abstract_expected_files' was moved." + "\nUsing deprecated source of 'abstract_expected_files'. Content was" + " move to 'openpype.pipeline.farm.abstract_expected_files'." + " Please change your imports as soon as possible." + ), + category=ExpectedFilesDeprecated, + stacklevel=4 +) - Returns: - list: Full paths to expected rendered files. - list of dict: Path to expected rendered files categorized by - AOVs, etc. - """ - raise NotImplementedError() +__all__ = ( + "ExpectedFiles", +) diff --git a/openpype/modules/deadline/abstract_submit_deadline.py b/openpype/modules/deadline/abstract_submit_deadline.py index 22902d79ea..3f54273a56 100644 --- a/openpype/modules/deadline/abstract_submit_deadline.py +++ b/openpype/modules/deadline/abstract_submit_deadline.py @@ -15,7 +15,7 @@ import attr import requests import pyblish.api -from openpype.lib.abstract_metaplugins import AbstractMetaInstancePlugin +from openpype.pipeline.publish import AbstractMetaInstancePlugin def requests_post(*args, **kwargs): diff --git a/openpype/pipeline/publish/__init__.py b/openpype/pipeline/publish/__init__.py index 03d730f37a..aa7fe0bdbf 100644 --- a/openpype/pipeline/publish/__init__.py +++ b/openpype/pipeline/publish/__init__.py @@ -16,6 +16,12 @@ from .lib import ( load_help_content_from_filepath, ) +from .abstract_expected_files import ExpectedFiles +from .abstract_collect_render import ( + RenderInstance, + AbstractCollectRender, +) + __all__ = ( "AbstractMetaInstancePlugin", @@ -31,4 +37,9 @@ __all__ = ( "publish_plugins_discover", "load_help_content_from_plugin", "load_help_content_from_filepath", + + "ExpectedFiles", + + "RenderInstance", + "AbstractCollectRender", ) diff --git a/openpype/pipeline/publish/abstract_collect_render.py b/openpype/pipeline/publish/abstract_collect_render.py new file mode 100644 index 0000000000..2e537227c3 --- /dev/null +++ b/openpype/pipeline/publish/abstract_collect_render.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Collect render template. + +TODO: use @dataclass when times come. + +""" +from abc import abstractmethod + +import attr +import six + +import pyblish.api + +from openpype.pipeline import legacy_io +from .publish_plugins import AbstractMetaContextPlugin + + +@attr.s +class RenderInstance(object): + """Data collected by collectors. + + This data class later on passed to collected instances. + Those attributes are required later on. + + """ + + # metadata + version = attr.ib() # instance version + time = attr.ib() # time of instance creation (get_formatted_current_time) + source = attr.ib() # path to source scene file + label = attr.ib() # label to show in GUI + subset = attr.ib() # subset name + task = attr.ib() # task name + asset = attr.ib() # asset name (AVALON_ASSET) + attachTo = attr.ib() # subset name to attach render to + setMembers = attr.ib() # list of nodes/members producing render output + publish = attr.ib() # bool, True to publish instance + name = attr.ib() # instance name + + # format settings + resolutionWidth = attr.ib() # resolution width (1920) + resolutionHeight = attr.ib() # resolution height (1080) + pixelAspect = attr.ib() # pixel aspect (1.0) + + # time settings + frameStart = attr.ib() # start frame + frameEnd = attr.ib() # start end + frameStep = attr.ib() # frame step + + handleStart = attr.ib(default=None) # start frame + handleEnd = attr.ib(default=None) # start frame + + # for software (like Harmony) where frame range cannot be set by DB + # handles need to be propagated if exist + ignoreFrameHandleCheck = attr.ib(default=False) + + # -------------------- + # With default values + # metadata + renderer = attr.ib(default="") # renderer - can be used in Deadline + review = attr.ib(default=False) # generate review from instance (bool) + priority = attr.ib(default=50) # job priority on farm + + family = attr.ib(default="renderlayer") + families = attr.ib(default=["renderlayer"]) # list of families + + # format settings + multipartExr = attr.ib(default=False) # flag for multipart exrs + convertToScanline = attr.ib(default=False) # flag for exr conversion + + tileRendering = attr.ib(default=False) # bool: treat render as tiles + tilesX = attr.ib(default=0) # number of tiles in X + tilesY = attr.ib(default=0) # number of tiles in Y + + # submit_publish_job + toBeRenderedOn = attr.ib(default=None) + deadlineSubmissionJob = attr.ib(default=None) + anatomyData = attr.ib(default=None) + outputDir = attr.ib(default=None) + context = attr.ib(default=None) + + @frameStart.validator + def check_frame_start(self, _, value): + """Validate if frame start is not larger then end.""" + if value > self.frameEnd: + raise ValueError("frameStart must be smaller " + "or equal then frameEnd") + + @frameEnd.validator + def check_frame_end(self, _, value): + """Validate if frame end is not less then start.""" + if value < self.frameStart: + raise ValueError("frameEnd must be smaller " + "or equal then frameStart") + + @tilesX.validator + def check_tiles_x(self, _, value): + """Validate if tile x isn't less then 1.""" + if not self.tileRendering: + return + if value < 1: + raise ValueError("tile X size cannot be less then 1") + + if value == 1 and self.tilesY == 1: + raise ValueError("both tiles X a Y sizes are set to 1") + + @tilesY.validator + def check_tiles_y(self, _, value): + """Validate if tile y isn't less then 1.""" + if not self.tileRendering: + return + if value < 1: + raise ValueError("tile Y size cannot be less then 1") + + if value == 1 and self.tilesX == 1: + raise ValueError("both tiles X a Y sizes are set to 1") + + +@six.add_metaclass(AbstractMetaContextPlugin) +class AbstractCollectRender(pyblish.api.ContextPlugin): + """Gather all publishable render layers from renderSetup.""" + + order = pyblish.api.CollectorOrder + 0.01 + label = "Collect Render" + sync_workfile_version = False + + def __init__(self, *args, **kwargs): + """Constructor.""" + super(AbstractCollectRender, self).__init__(*args, **kwargs) + self._file_path = None + self._asset = legacy_io.Session["AVALON_ASSET"] + self._context = None + + def process(self, context): + """Entry point to collector.""" + self._context = context + for instance in context: + # make sure workfile instance publishing is enabled + try: + if "workfile" in instance.data["families"]: + instance.data["publish"] = True + # TODO merge renderFarm and render.farm + if ("renderFarm" in instance.data["families"] or + "render.farm" in instance.data["families"]): + instance.data["remove"] = True + except KeyError: + # be tolerant if 'families' is missing. + pass + + self._file_path = context.data["currentFile"].replace("\\", "/") + + render_instances = self.get_instances(context) + for render_instance in render_instances: + exp_files = self.get_expected_files(render_instance) + assert exp_files, "no file names were generated, this is bug" + + # if we want to attach render to subset, check if we have AOV's + # in expectedFiles. If so, raise error as we cannot attach AOV + # (considered to be subset on its own) to another subset + if render_instance.attachTo: + assert isinstance(exp_files, list), ( + "attaching multiple AOVs or renderable cameras to " + "subset is not supported" + ) + + frame_start_render = int(render_instance.frameStart) + frame_end_render = int(render_instance.frameEnd) + if (render_instance.ignoreFrameHandleCheck or + int(context.data['frameStartHandle']) == frame_start_render + and int(context.data['frameEndHandle']) == frame_end_render): # noqa: W503, E501 + + handle_start = context.data['handleStart'] + handle_end = context.data['handleEnd'] + frame_start = context.data['frameStart'] + frame_end = context.data['frameEnd'] + frame_start_handle = context.data['frameStartHandle'] + frame_end_handle = context.data['frameEndHandle'] + else: + handle_start = 0 + handle_end = 0 + frame_start = frame_start_render + frame_end = frame_end_render + frame_start_handle = frame_start_render + frame_end_handle = frame_end_render + + data = { + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartHandle": frame_start_handle, + "frameEndHandle": frame_end_handle, + "byFrameStep": int(render_instance.frameStep), + + "author": context.data["user"], + # Add source to allow tracing back to the scene from + # which was submitted originally + "expectedFiles": exp_files, + } + if self.sync_workfile_version: + data["version"] = context.data["version"] + + # add additional data + data = self.add_additional_data(data) + render_instance_dict = attr.asdict(render_instance) + + instance = context.create_instance(render_instance.name) + instance.data["label"] = render_instance.label + instance.data.update(render_instance_dict) + instance.data.update(data) + + self.post_collecting_action() + + @abstractmethod + def get_instances(self, context): + """Get all renderable instances and their data. + + Args: + context (pyblish.api.Context): Context object. + + Returns: + list of :class:`RenderInstance`: All collected renderable instances + (like render layers, write nodes, etc.) + + """ + pass + + @abstractmethod + def get_expected_files(self, render_instance): + """Get list of expected files. + + Returns: + list: expected files. This can be either simple list of files with + their paths, or list of dictionaries, where key is name of AOV + for example and value is list of files for that AOV. + + Example:: + + ['/path/to/file.001.exr', '/path/to/file.002.exr'] + + or as dictionary: + + [ + { + "beauty": ['/path/to/beauty.001.exr', ...], + "mask": ['/path/to/mask.001.exr'] + } + ] + + """ + pass + + def add_additional_data(self, data): + """Add additional data to collected instance. + + This can be overridden by host implementation to add custom + additional data. + + """ + return data + + def post_collecting_action(self): + """Execute some code after collection is done. + + This is useful for example for restoring current render layer. + + """ + pass diff --git a/openpype/pipeline/publish/abstract_expected_files.py b/openpype/pipeline/publish/abstract_expected_files.py new file mode 100644 index 0000000000..f9f3c17ef5 --- /dev/null +++ b/openpype/pipeline/publish/abstract_expected_files.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +"""Abstract ExpectedFile class definition.""" +from abc import ABCMeta, abstractmethod +import six + + +@six.add_metaclass(ABCMeta) +class ExpectedFiles: + """Class grouping functionality for all supported renderers. + + Attributes: + multipart (bool): Flag if multipart exrs are used. + + """ + + multipart = False + + @abstractmethod + def get(self, render_instance): + """Get expected files for given renderer and render layer. + + This method should return dictionary of all files we are expecting + to be rendered from the host. Usually `render_instance` corresponds + to *render layer*. Result can be either flat list with the file + paths or it can be list of dictionaries. Each key corresponds to + for example AOV name or channel, etc. + + Example:: + + ['/path/to/file.001.exr', '/path/to/file.002.exr'] + + or as dictionary: + + [ + { + "beauty": ['/path/to/beauty.001.exr', ...], + "mask": ['/path/to/mask.001.exr'] + } + ] + + + Args: + render_instance (:class:`RenderInstance`): Data passed from + collector to determine files. This should be instance of + :class:`abstract_collect_render.RenderInstance` + + Returns: + list: Full paths to expected rendered files. + list of dict: Path to expected rendered files categorized by + AOVs, etc. + + """ + raise NotImplementedError()