diff --git a/openpype/modules/deadline/plugins/publish/submit_publish_job.py b/openpype/modules/deadline/plugins/publish/submit_publish_job.py index 4765772bcf..fd0d87a082 100644 --- a/openpype/modules/deadline/plugins/publish/submit_publish_job.py +++ b/openpype/modules/deadline/plugins/publish/submit_publish_job.py @@ -21,35 +21,10 @@ from openpype.pipeline import ( from openpype.tests.lib import is_in_tests from openpype.pipeline.farm.patterning import match_aov_pattern from openpype.lib import is_running_from_build - - -def get_resources(project_name, version, extension=None): - """Get the files from the specific version.""" - - # TODO this functions seems to be weird - # - it's looking for representation with one extension or first (any) - # representation from a version? - # - not sure how this should work, maybe it does for specific use cases - # but probably can't be used for all resources from 2D workflows - extensions = None - if extension: - extensions = [extension] - repre_docs = list(get_representations( - project_name, version_ids=[version["_id"]], extensions=extensions - )) - assert repre_docs, "This is a bug" - - representation = repre_docs[0] - directory = get_representation_path(representation) - print("Source: ", directory) - resources = sorted( - [ - os.path.normpath(os.path.join(directory, fname)) - for fname in os.listdir(directory) - ] - ) - - return resources +from openpype.pipeline.farm.pyblish import ( + create_skeleton_instance, + create_instances_for_aov +) def get_resource_files(resources, frame_range=None): @@ -356,241 +331,6 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): return deadline_publish_job_id - def _copy_extend_frames(self, instance, representation): - """Copy existing frames from latest version. - - This will copy all existing frames from subset's latest version back - to render directory and rename them to what renderer is expecting. - - Arguments: - instance (pyblish.plugin.Instance): instance to get required - data from - representation (dict): presentation to operate on - - """ - import speedcopy - - self.log.info("Preparing to copy ...") - start = instance.data.get("frameStart") - end = instance.data.get("frameEnd") - project_name = legacy_io.active_project() - - # get latest version of subset - # this will stop if subset wasn't published yet - project_name = legacy_io.active_project() - version = get_last_version_by_subset_name( - project_name, - instance.data.get("subset"), - asset_name=instance.data.get("asset") - ) - - # get its files based on extension - subset_resources = get_resources( - project_name, version, representation.get("ext") - ) - r_col, _ = clique.assemble(subset_resources) - - # if override remove all frames we are expecting to be rendered - # so we'll copy only those missing from current render - if instance.data.get("overrideExistingFrame"): - for frame in range(start, end + 1): - if frame not in r_col.indexes: - continue - r_col.indexes.remove(frame) - - # now we need to translate published names from representation - # back. This is tricky, right now we'll just use same naming - # and only switch frame numbers - resource_files = [] - r_filename = os.path.basename( - representation.get("files")[0]) # first file - op = re.search(self.R_FRAME_NUMBER, r_filename) - pre = r_filename[:op.start("frame")] - post = r_filename[op.end("frame"):] - assert op is not None, "padding string wasn't found" - for frame in list(r_col): - fn = re.search(self.R_FRAME_NUMBER, frame) - # silencing linter as we need to compare to True, not to - # type - assert fn is not None, "padding string wasn't found" - # list of tuples (source, destination) - staging = representation.get("stagingDir") - staging = self.anatomy.fill_root(staging) - resource_files.append( - (frame, - os.path.join(staging, - "{}{}{}".format(pre, - fn.group("frame"), - post))) - ) - - # test if destination dir exists and create it if not - output_dir = os.path.dirname(representation.get("files")[0]) - if not os.path.isdir(output_dir): - os.makedirs(output_dir) - - # copy files - for source in resource_files: - speedcopy.copy(source[0], source[1]) - self.log.info(" > {}".format(source[1])) - - self.log.info( - "Finished copying %i files" % len(resource_files)) - - def _create_instances_for_aov( - self, instance_data, exp_files, additional_data - ): - """Create instance for each AOV found. - - This will create new instance for every aov it can detect in expected - files list. - - Arguments: - instance_data (pyblish.plugin.Instance): skeleton data for instance - (those needed) later by collector - exp_files (list): list of expected files divided by aovs - - Returns: - list of instances - - """ - task = os.environ["AVALON_TASK"] - subset = instance_data["subset"] - cameras = instance_data.get("cameras", []) - instances = [] - # go through aovs in expected files - for aov, files in exp_files[0].items(): - cols, rem = clique.assemble(files) - # we shouldn't have any reminders. And if we do, it should - # be just one item for single frame renders. - if not cols and rem: - assert len(rem) == 1, ("Found multiple non related files " - "to render, don't know what to do " - "with them.") - col = rem[0] - ext = os.path.splitext(col)[1].lstrip(".") - else: - # but we really expect only one collection. - # Nothing else make sense. - assert len(cols) == 1, "only one image sequence type is expected" # noqa: E501 - ext = cols[0].tail.lstrip(".") - col = list(cols[0]) - - self.log.debug(col) - # create subset name `familyTaskSubset_AOV` - group_name = 'render{}{}{}{}'.format( - task[0].upper(), task[1:], - subset[0].upper(), subset[1:]) - - cam = [c for c in cameras if c in col.head] - if cam: - if aov: - subset_name = '{}_{}_{}'.format(group_name, cam, aov) - else: - subset_name = '{}_{}'.format(group_name, cam) - else: - if aov: - subset_name = '{}_{}'.format(group_name, aov) - else: - subset_name = '{}'.format(group_name) - - if isinstance(col, (list, tuple)): - staging = os.path.dirname(col[0]) - else: - staging = os.path.dirname(col) - - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path(staging) - ) - if success: - staging = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging)) - - self.log.info("Creating data for: {}".format(subset_name)) - - app = os.environ.get("AVALON_APP", "") - - preview = False - - if isinstance(col, list): - render_file_name = os.path.basename(col[0]) - else: - render_file_name = os.path.basename(col) - aov_patterns = self.aov_filter - - preview = match_aov_pattern(app, aov_patterns, render_file_name) - # toggle preview on if multipart is on - - if instance_data.get("multipartExr"): - self.log.debug("Adding preview tag because its multipartExr") - preview = True - self.log.debug("preview:{}".format(preview)) - new_instance = deepcopy(instance_data) - new_instance["subset"] = subset_name - new_instance["subsetGroup"] = group_name - if preview: - new_instance["review"] = True - - # create representation - if isinstance(col, (list, tuple)): - files = [os.path.basename(f) for f in col] - else: - files = os.path.basename(col) - - # Copy render product "colorspace" data to representation. - colorspace = "" - products = additional_data["renderProducts"].layer_data.products - for product in products: - if product.productName == aov: - colorspace = product.colorspace - break - - rep = { - "name": ext, - "ext": ext, - "files": files, - "frameStart": int(instance_data.get("frameStartHandle")), - "frameEnd": int(instance_data.get("frameEndHandle")), - # If expectedFile are absolute, we need only filenames - "stagingDir": staging, - "fps": new_instance.get("fps"), - "tags": ["review"] if preview else [], - "colorspaceData": { - "colorspace": colorspace, - "config": { - "path": additional_data["colorspaceConfig"], - "template": additional_data["colorspaceTemplate"] - }, - "display": additional_data["display"], - "view": additional_data["view"] - } - } - - # support conversion from tiled to scanline - if instance_data.get("convertToScanline"): - self.log.info("Adding scanline conversion.") - rep["tags"].append("toScanline") - - # poor man exclusion - if ext in self.skip_integration_repre_list: - rep["tags"].append("delete") - - self._solve_families(new_instance, preview) - - new_instance["representations"] = [rep] - - # if extending frames from existing version, copy files from there - # into our destination directory - if new_instance.get("extendFrames", False): - self._copy_extend_frames(new_instance, rep) - instances.append(new_instance) - self.log.debug("instances:{}".format(instances)) - return instances - def _get_representations(self, instance, exp_files): """Create representations for file sequences. @@ -748,8 +488,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # type: (pyblish.api.Instance) -> None """Process plugin. - Detect type of renderfarm submission and create and post dependent job - in case of Deadline. It creates json file with metadata needed for + Detect type of render farm submission and create and post dependent + job in case of Deadline. It creates json file with metadata needed for publishing in directory of render. Args: @@ -760,145 +500,16 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): self.log.info("Skipping local instance.") return - data = instance.data.copy() - context = instance.context - self.context = context - self.anatomy = instance.context.data["anatomy"] - - asset = data.get("asset") or legacy_io.Session["AVALON_ASSET"] - subset = data.get("subset") - - start = instance.data.get("frameStart") - if start is None: - start = context.data["frameStart"] - - end = instance.data.get("frameEnd") - if end is None: - end = context.data["frameEnd"] - - handle_start = instance.data.get("handleStart") - if handle_start is None: - handle_start = context.data["handleStart"] - - handle_end = instance.data.get("handleEnd") - if handle_end is None: - handle_end = context.data["handleEnd"] - - fps = instance.data.get("fps") - if fps is None: - fps = context.data["fps"] - - if data.get("extendFrames", False): - start, end = self._extend_frames( - asset, - subset, - start, - end, - data["overrideExistingFrame"]) - - try: - source = data["source"] - except KeyError: - source = context.data["currentFile"] - - success, rootless_path = ( - self.anatomy.find_root_template_from_path(source) - ) - if success: - source = rootless_path - - else: - # `rootless_path` is not set to `source` if none of roots match - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues." - ).format(source)) - - family = "render" - if "prerender" in instance.data["families"]: - family = "prerender" - families = [family] - - # pass review to families if marked as review - if data.get("review"): - families.append("review") - - instance_skeleton_data = { - "family": family, - "subset": subset, - "families": families, - "asset": asset, - "frameStart": start, - "frameEnd": end, - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStartHandle": start - handle_start, - "frameEndHandle": end + handle_end, - "comment": instance.data["comment"], - "fps": fps, - "source": source, - "extendFrames": data.get("extendFrames"), - "overrideExistingFrame": data.get("overrideExistingFrame"), - "pixelAspect": data.get("pixelAspect", 1), - "resolutionWidth": data.get("resolutionWidth", 1920), - "resolutionHeight": data.get("resolutionHeight", 1080), - "multipartExr": data.get("multipartExr", False), - "jobBatchName": data.get("jobBatchName", ""), - "useSequenceForReview": data.get("useSequenceForReview", True), - # map inputVersions `ObjectId` -> `str` so json supports it - "inputVersions": list(map(str, data.get("inputVersions", []))) - } - - # skip locking version if we are creating v01 - instance_version = instance.data.get("version") # take this if exists - if instance_version != 1: - instance_skeleton_data["version"] = instance_version - - # transfer specific families from original instance to new render - for item in self.families_transfer: - if item in instance.data.get("families", []): - instance_skeleton_data["families"] += [item] - - # transfer specific properties from original instance based on - # mapping dictionary `instance_transfer` - for key, values in self.instance_transfer.items(): - if key in instance.data.get("families", []): - for v in values: - instance_skeleton_data[v] = instance.data.get(v) - - # look into instance data if representations are not having any - # which are having tag `publish_on_farm` and include them - for repre in instance.data.get("representations", []): - staging_dir = repre.get("stagingDir") - if staging_dir: - success, rootless_staging_dir = ( - self.anatomy.find_root_template_from_path( - staging_dir - ) - ) - if success: - repre["stagingDir"] = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(staging_dir)) - repre["stagingDir"] = staging_dir - - if "publish_on_farm" in repre.get("tags"): - # create representations attribute of not there - if "representations" not in instance_skeleton_data.keys(): - instance_skeleton_data["representations"] = [] - - instance_skeleton_data["representations"].append(repre) + instance_skeleton_data = create_skeleton_instance( + instance, + families_transfer=self.families_transfer, + instance_transfer=self.instance_transfer) instances = None - assert data.get("expectedFiles"), ("Submission from old Pype version" - " - missing expectedFiles") """ - if content of `expectedFiles` are dictionaries, we will handle - it as list of AOVs, creating instance from every one of them. + if content of `expectedFiles` list are dictionaries, we will handle + it as list of AOVs, creating instance for every one of them. Example: -------- @@ -920,7 +531,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): This will create instances for `beauty` and `Z` subset adding those files to their respective representations. - If we've got only list of files, we collect all filesequences. + If we have only list of files, we collect all file sequences. More then one doesn't probably make sense, but we'll handle it like creating one instance with multiple representations. @@ -938,55 +549,14 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): `foo` and `xxx` """ - self.log.info(data.get("expectedFiles")) - - if isinstance(data.get("expectedFiles")[0], dict): - # we cannot attach AOVs to other subsets as we consider every - # AOV subset of its own. - - additional_data = { - "renderProducts": instance.data["renderProducts"], - "colorspaceConfig": instance.data["colorspaceConfig"], - "display": instance.data["colorspaceDisplay"], - "view": instance.data["colorspaceView"] - } - - # Get templated path from absolute config path. - anatomy = instance.context.data["anatomy"] - colorspaceTemplate = instance.data["colorspaceConfig"] - success, rootless_staging_dir = ( - anatomy.find_root_template_from_path(colorspaceTemplate) - ) - if success: - colorspaceTemplate = rootless_staging_dir - else: - self.log.warning(( - "Could not find root path for remapping \"{}\"." - " This may cause issues on farm." - ).format(colorspaceTemplate)) - additional_data["colorspaceTemplate"] = colorspaceTemplate - - if len(data.get("attachTo")) > 0: - assert len(data.get("expectedFiles")[0].keys()) == 1, ( - "attaching multiple AOVs or renderable cameras to " - "subset is not supported") - - # create instances for every AOV we found in expected files. - # note: this is done for every AOV and every render camere (if - # there are multiple renderable cameras in scene) - instances = self._create_instances_for_aov( - instance_skeleton_data, - data.get("expectedFiles"), - additional_data - ) - self.log.info("got {} instance{}".format( - len(instances), - "s" if len(instances) > 1 else "")) + if isinstance(instance.data.get("expectedFiles")[0], dict): + instances = create_instances_for_aov( + instance, instance_skeleton_data) else: representations = self._get_representations( instance_skeleton_data, - data.get("expectedFiles") + instance.data.get("expectedFiles") ) if "representations" not in instance_skeleton_data.keys(): @@ -1029,11 +599,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): render_job = None submission_type = "" if instance.data.get("toBeRenderedOn") == "deadline": - render_job = data.pop("deadlineSubmissionJob", None) + render_job = instance.data.pop("deadlineSubmissionJob", None) submission_type = "deadline" if instance.data.get("toBeRenderedOn") == "muster": - render_job = data.pop("musterSubmissionJob", None) + render_job = instance.data.pop("musterSubmissionJob", None) submission_type = "muster" if not render_job and instance.data.get("tileRendering") is False: @@ -1056,9 +626,9 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "jobBatchName") else: render_job["Props"]["Batch"] = os.path.splitext( - os.path.basename(context.data.get("currentFile")))[0] + os.path.basename(instance.context.data.get("currentFile")))[0] # User is deadline user - render_job["Props"]["User"] = context.data.get( + render_job["Props"]["User"] = instance.context.data.get( "deadlineUser", getpass.getuser()) render_job["Props"]["Env"] = { @@ -1080,15 +650,15 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # publish job file publish_job = { - "asset": asset, - "frameStart": start, - "frameEnd": end, - "fps": context.data.get("fps", None), - "source": source, - "user": context.data["user"], - "version": context.data["version"], # this is workfile version - "intent": context.data.get("intent"), - "comment": context.data.get("comment"), + "asset": instance_skeleton_data["asset"], + "frameStart": instance_skeleton_data["frameStart"], + "frameEnd": instance_skeleton_data["frameEnd"], + "fps": instance_skeleton_data["fps"], + "source": instance_skeleton_data["source"], + "user": instance.context.data["user"], + "version": instance.context.data["version"], # this is workfile version + "intent": instance.context.data.get("intent"), + "comment": instance.context.data.get("comment"), "job": render_job or None, "session": legacy_io.Session.copy(), "instances": instances @@ -1098,7 +668,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): publish_job["deadline_publish_job_id"] = deadline_publish_job_id # add audio to metadata file if available - audio_file = context.data.get("audioFile") + audio_file = instance.context.data.get("audioFile") if audio_file and os.path.isfile(audio_file): publish_job.update({"audio": audio_file}) diff --git a/openpype/pipeline/farm/pyblish.py b/openpype/pipeline/farm/pyblish.py index 3ad62962d2..35a944444f 100644 --- a/openpype/pipeline/farm/pyblish.py +++ b/openpype/pipeline/farm/pyblish.py @@ -1,32 +1,165 @@ +from openpype.pipeline import ( + get_current_project_name, + get_representation_path, + Anatomy, +) +from openpype.client import ( + get_last_version_by_subset_name, + get_representations +) from openpype.lib import Logger import attr import pyblish.api +from openpype.pipeline.publish import KnownPublishError +import os +import clique +from copy import deepcopy +import re @attr.s -class InstanceSkeleton(object): - # family = attr.ib(factory=) - pass +class TimeData: + """Structure used to handle time related data.""" + start = attr.ib(type=int) + end = attr.ib(type=int) + fps = attr.ib() + step = attr.ib(default=1, type=int) + handle_start = attr.ib(default=0, type=int) + handle_end = attr.ib(default=0, type=int) -def remap_source(source, anatomy): +def remap_source(path, anatomy): + """Try to remap path to rootless path. + + Args: + path (str): Path to be remapped to rootless. + anatomy (Anatomy): Anatomy object to handle remapping + itself. + + Returns: + str: Remapped path. + + Throws: + ValueError: if the root cannot be found. + + """ success, rootless_path = ( - anatomy.find_root_template_from_path(source) + anatomy.find_root_template_from_path(path) ) if success: source = rootless_path else: - # `rootless_path` is not set to `source` if none of roots match - log = Logger.get_logger("farm_publishing") - log.warning( - ("Could not find root path for remapping \"{}\"." - " This may cause issues.").format(source)) + raise ValueError( + "Root from template path cannot be found: {}".format(path)) return source -def create_skeleton_instance(instance): - # type: (pyblish.api.Instance) -> dict - """Create skelenton instance from original instance data. +def extend_frames(asset, subset, start, end): + """Get latest version of asset nad update frame range. + + Based on minimum and maximum values. + + Arguments: + asset (str): asset name + subset (str): subset name + start (int): start frame + end (int): end frame + + Returns: + (int, int): update frame start/end + + """ + # Frame comparison + prev_start = None + prev_end = None + + project_name = get_current_project_name() + version = get_last_version_by_subset_name( + project_name, + subset, + asset_name=asset + ) + + # Set prev start / end frames for comparison + if not prev_start and not prev_end: + prev_start = version["data"]["frameStart"] + prev_end = version["data"]["frameEnd"] + + updated_start = min(start, prev_start) + updated_end = max(end, prev_end) + + + return updated_start, updated_end + + +def get_time_data_from_instance_or_context(instance): + """Get time data from instance (or context). + + If time data is not found on instance, data from context will be used. + + Args: + instance (pyblish.api.Instance): Source instance. + + Returns: + TimeData: dataclass holding time information. + + """ + return TimeData( + start=instance.data.get["frameStart"] or \ + instance.context.data.get("frameStart"), + end=instance.data.get("frameEnd") or \ + instance.context.data.get("frameEnd"), + fps=instance.data.get("fps") or \ + instance.context.data.get("fps"), + handle_start=instance.data.get("handleStart") or \ + instance.context.data.get("handleStart"), # noqa: E501 + handle_end=instance.data.get("handleStart") or \ + instance.context.data.get("handleStart") + ) + + +def get_transferable_representations(instance): + """Transfer representations from original instance. + + This will get all representations on the original instance that + are flagged with `publish_on_farm` and return them to be included + on skeleton instance if needed. + + Args: + instance (pyblish.api.Instance): Original instance to be processed. + + Return: + list of dicts: List of transferable representations. + + """ + anatomy = instance.data.get("anatomy") # type: Anatomy + to_transfer = [] + + for representation in instance.data.get("representations", []): + if "publish_on_farm" not in representation.get("tags"): + continue + + trans_rep = representation.copy() + + staging_dir = trans_rep.get("stagingDir") + + if staging_dir: + try: + trans_rep["stagingDir"] = remap_source(staging_dir, anatomy) + except ValueError: + log = Logger.get_logger("farm_publishing") + log.warning( + ("Could not find root path for remapping \"{}\". " + "This may cause issues on farm.").format(staging_dir)) + + to_transfer.append(trans_rep) + return to_transfer + + +def create_skeleton_instance( + instance, families_transfer=None, instance_transfer=None): + # type: (pyblish.api.Instance, list, dict) -> dict + """Create skeleton instance from original instance data. This will create dictionary containing skeleton - common - data used for publishing rendered instances. @@ -36,26 +169,64 @@ def create_skeleton_instance(instance): Args: instance (pyblish.api.Instance): Original instance to be used as a source of data. + families_transfer (list): List of family names to transfer + from the original instance to the skeleton. + instance_transfer (dict): Dict with keys as families and + values as a list of property names to transfer to the + new skeleton. Returns: dict: Dictionary with skeleton instance data. """ - context = instance.context + # list of family names to transfer to new family if present - return { + context = instance.context + data = instance.data.copy() + anatomy = data["anatomy"] # type: Anatomy + + families = [data["family"]] + + # pass review to families if marked as review + if data.get("review"): + families.append("review") + + # get time related data from instance (or context) + time_data = get_time_data_from_instance_or_context(instance) + + if data.get("extendFrames", False): + time_data.start, time_data.end = extend_frames( + data["asset"], + data["subset"], + time_data.start, + time_data.end, + ) + + source = data.get("source") or context.data.get("currentFile") + success, rootless_path = ( + anatomy.find_root_template_from_path(source) + ) + if success: + source = rootless_path + else: + # `rootless_path` is not set to `source` if none of roots match + log = Logger.get_logger("farm_publishing") + log.warning(("Could not find root path for remapping \"{}\". " + "This may cause issues.").format(source)) + + instance_skeleton_data = { "family": "render" if "prerender" not in instance.data["families"] else "prerender", # noqa: E401 - "subset": subset, + "subset": data["subset"], "families": families, - "asset": asset, - "frameStart": start, - "frameEnd": end, - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStartHandle": start - handle_start, - "frameEndHandle": end + handle_end, - "comment": instance.data["comment"], - "fps": fps, + "asset": data["asset"], + "frameStart": time_data.start, + "frameEnd": time_data.end, + "handleStart": time_data.handle_start, + "handleEnd": time_data.handle_end, + "frameStartHandle": time_data.start - time_data.handle_start, + "frameEndHandle": time_data.end + time_data.handle_end, + "comment": data.get("comment"), + "fps": time_data.fps, "source": source, "extendFrames": data.get("extendFrames"), "overrideExistingFrame": data.get("overrideExistingFrame"), @@ -68,3 +239,360 @@ def create_skeleton_instance(instance): # map inputVersions `ObjectId` -> `str` so json supports it "inputVersions": list(map(str, data.get("inputVersions", []))) } + + # skip locking version if we are creating v01 + instance_version = data.get("version") # take this if exists + if instance_version != 1: + instance_skeleton_data["version"] = instance_version + + # transfer specific families from original instance to new render + for item in families_transfer: + if item in instance.data.get("families", []): + instance_skeleton_data["families"] += [item] + + # transfer specific properties from original instance based on + # mapping dictionary `instance_transfer` + for key, values in instance_transfer.items(): + if key in instance.data.get("families", []): + for v in values: + instance_skeleton_data[v] = instance.data.get(v) + + representations = get_transferable_representations(instance) + instance_skeleton_data["representations"] = [] + instance_skeleton_data["representations"] += representations + + return instance_skeleton_data + +def _solve_families(families): + """Solve families. + + TODO: This is ugly and needs to be refactored. Ftrack family should be + added in different way (based on if the module is enabled?) + + """ + # if we have one representation with preview tag + # flag whole instance for review and for ftrack + if "ftrack" not in families and os.environ.get("FTRACK_SERVER"): + families.append("ftrack") + if "review" not in families: + families.append("review") + return families +def create_instances_for_aov(instance, skeleton): + """Create instances from AOVs. + + This will create new pyblish.api.Instances by going over expected + files defined on original instance. + + Args: + instance (pyblish.api.Instance): Original instance. + skeleton (dict): Skeleton instance data. + + Returns: + list of pyblish.api.Instance: Instances created from + expected files. + + """ + # we cannot attach AOVs to other subsets as we consider every + # AOV subset of its own. + + log = Logger.get_logger("farm_publishing") + additional_color_data = { + "renderProducts": instance.data["renderProducts"], + "colorspaceConfig": instance.data["colorspaceConfig"], + "display": instance.data["colorspaceDisplay"], + "view": instance.data["colorspaceView"] + } + + # Get templated path from absolute config path. + anatomy = instance.context.data["anatomy"] + colorspace_template = instance.data["colorspaceConfig"] + try: + additional_color_data["colorspaceTemplate"] = remap_source( + colorspace_template, anatomy) + except ValueError as e: + log.warning(e) + + # if there are subset to attach to and more than one AOV, + # we cannot proceed. + if ( + len(instance.data.get("attachTo", [])) > 0 + and len(instance.data.get("expectedFiles")[0].keys()) != 1 + ): + raise KnownPublishError( + "attaching multiple AOVs or renderable cameras to " + "subset is not supported yet.") + + # create instances for every AOV we found in expected files. + # NOTE: this is done for every AOV and every render camera (if + # there are multiple renderable cameras in scene) + return _create_instances_for_aov( + instance, + skeleton, + additional_color_data + ) + + +def _create_instances_for_aov(instance, skeleton, additional_data): + """Create instance for each AOV found. + + This will create new instance for every AOV it can detect in expected + files list. + + Args: + instance (pyblish.api.Instance): Original instance. + skeleton (dict): Skeleton data for instance (those needed) later + by collector. + additional_data (dict): ... + + + Returns: + list of instances + + Throws: + ValueError: + + """ + # TODO: this needs to be taking the task from context or instance + task = os.environ["AVALON_TASK"] + + anatomy = instance.data["anatomy"] + subset = skeleton["subset"] + cameras = instance.data.get("cameras", []) + exp_files = instance.data["expectedFiles"] + log = Logger.get_logger("farm_publishing") + + instances = [] + # go through AOVs in expected files + for aov, files in exp_files[0].items(): + cols, rem = clique.assemble(files) + # we shouldn't have any reminders. And if we do, it should + # be just one item for single frame renders. + if not cols and rem: + if len(rem) != 1: + raise ValueError("Found multiple non related files " + "to render, don't know what to do " + "with them.") + col = rem[0] + ext = os.path.splitext(col)[1].lstrip(".") + else: + # but we really expect only one collection. + # Nothing else make sense. + if len(cols) != 1: + raise ValueError("Only one image sequence type is expected.") # noqa: E501 + ext = cols[0].tail.lstrip(".") + col = list(cols[0]) + + # create subset name `familyTaskSubset_AOV` + group_name = 'render{}{}{}{}'.format( + task[0].upper(), task[1:], + subset[0].upper(), subset[1:]) + + cam = [c for c in cameras if c in col.head] + if cam: + if aov: + subset_name = '{}_{}_{}'.format(group_name, cam, aov) + else: + subset_name = '{}_{}'.format(group_name, cam) + else: + if aov: + subset_name = '{}_{}'.format(group_name, aov) + else: + subset_name = '{}'.format(group_name) + + if isinstance(col, (list, tuple)): + staging = os.path.dirname(col[0]) + else: + staging = os.path.dirname(col) + + try: + staging = remap_source(staging, anatomy) + except ValueError as e: + log.warning(e) + + log.info("Creating data for: {}".format(subset_name)) + + app = os.environ.get("AVALON_APP", "") + + preview = False + + if isinstance(col, list): + render_file_name = os.path.basename(col[0]) + else: + render_file_name = os.path.basename(col) + aov_patterns = self.aov_filter + + preview = match_aov_pattern(app, aov_patterns, render_file_name) + # toggle preview on if multipart is on + + if instance.data.get("multipartExr"): + log.debug("Adding preview tag because its multipartExr") + preview = True + + + new_instance = deepcopy(skeleton) + new_instance["subsetGroup"] = group_name + if preview: + new_instance["review"] = True + + # create representation + if isinstance(col, (list, tuple)): + files = [os.path.basename(f) for f in col] + else: + files = os.path.basename(col) + + # Copy render product "colorspace" data to representation. + colorspace = "" + products = additional_data["renderProducts"].layer_data.products + for product in products: + if product.productName == aov: + colorspace = product.colorspace + break + + rep = { + "name": ext, + "ext": ext, + "files": files, + "frameStart": int(skeleton["frameStartHandle"]), + "frameEnd": int(skeleton["frameEndHandle"]), + # If expectedFile are absolute, we need only filenames + "stagingDir": staging, + "fps": new_instance.get("fps"), + "tags": ["review"] if preview else [], + "colorspaceData": { + "colorspace": colorspace, + "config": { + "path": additional_data["colorspaceConfig"], + "template": additional_data["colorspaceTemplate"] + }, + "display": additional_data["display"], + "view": additional_data["view"] + } + } + + # support conversion from tiled to scanline + if instance.data.get("convertToScanline"): + log.info("Adding scanline conversion.") + rep["tags"].append("toScanline") + + # poor man exclusion + if ext in self.skip_integration_repre_list: + rep["tags"].append("delete") + + if preview: + new_instance["families"] = _solve_families(new_instance) + + new_instance["representations"] = [rep] + + # if extending frames from existing version, copy files from there + # into our destination directory + if new_instance.get("extendFrames", False): + copy_extend_frames(new_instance, rep) + instances.append(new_instance) + log.debug("instances:{}".format(instances)) + return instances + +def get_resources(project_name, version, extension=None): + """Get the files from the specific version.""" + + # TODO this functions seems to be weird + # - it's looking for representation with one extension or first (any) + # representation from a version? + # - not sure how this should work, maybe it does for specific use cases + # but probably can't be used for all resources from 2D workflows + extensions = None + if extension: + extensions = [extension] + repre_docs = list(get_representations( + project_name, version_ids=[version["_id"]], extensions=extensions + )) + assert repre_docs, "This is a bug" + + representation = repre_docs[0] + directory = get_representation_path(representation) + print("Source: ", directory) + resources = sorted( + [ + os.path.normpath(os.path.join(directory, fname)) + for fname in os.listdir(directory) + ] + ) + + return resources + +def copy_extend_frames(self, instance, representation): + """Copy existing frames from latest version. + + This will copy all existing frames from subset's latest version back + to render directory and rename them to what renderer is expecting. + + Arguments: + instance (pyblish.plugin.Instance): instance to get required + data from + representation (dict): presentation to operate on + + """ + import speedcopy + + log = Logger.get_logger("farm_publishing") + log.info("Preparing to copy ...") + start = instance.data.get("frameStart") + end = instance.data.get("frameEnd") + project_name = instance.context.data["project"] + + # get latest version of subset + # this will stop if subset wasn't published yet + + version = get_last_version_by_subset_name( + project_name, + instance.data.get("subset"), + asset_name=instance.data.get("asset") + ) + + # get its files based on extension + subset_resources = get_resources( + project_name, version, representation.get("ext") + ) + r_col, _ = clique.assemble(subset_resources) + + # if override remove all frames we are expecting to be rendered + # so we'll copy only those missing from current render + if instance.data.get("overrideExistingFrame"): + for frame in range(start, end + 1): + if frame not in r_col.indexes: + continue + r_col.indexes.remove(frame) + + # now we need to translate published names from representation + # back. This is tricky, right now we'll just use same naming + # and only switch frame numbers + resource_files = [] + r_filename = os.path.basename( + representation.get("files")[0]) # first file + op = re.search(self.R_FRAME_NUMBER, r_filename) + pre = r_filename[:op.start("frame")] + post = r_filename[op.end("frame"):] + assert op is not None, "padding string wasn't found" + for frame in list(r_col): + fn = re.search(self.R_FRAME_NUMBER, frame) + # silencing linter as we need to compare to True, not to + # type + assert fn is not None, "padding string wasn't found" + # list of tuples (source, destination) + staging = representation.get("stagingDir") + staging = self.anatomy.fill_root(staging) + resource_files.append( + (frame, os.path.join( + staging, "{}{}{}".format(pre, fn["frame"], post))) + ) + + # test if destination dir exists and create it if not + output_dir = os.path.dirname(representation.get("files")[0]) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + + # copy files + for source in resource_files: + speedcopy.copy(source[0], source[1]) + log.info(" > {}".format(source[1])) + + log.info("Finished copying %i files" % len(resource_files)) diff --git a/openpype/pipeline/farm/pyblish.pyi b/openpype/pipeline/farm/pyblish.pyi new file mode 100644 index 0000000000..3667f2d8a5 --- /dev/null +++ b/openpype/pipeline/farm/pyblish.pyi @@ -0,0 +1,23 @@ +import pyblish.api +from openpype.pipeline import Anatomy +from typing import Tuple, Union, List + + +class TimeData: + start: int + end: int + fps: float | int + step: int + handle_start: int + handle_end: int + + def __init__(self, start: int, end: int, fps: float | int, step: int, handle_start: int, handle_end: int): + ... + ... + +def remap_source(source: str, anatomy: Anatomy): ... +def extend_frames(asset: str, subset: str, start: int, end: int) -> Tuple[int, int]: ... +def get_time_data_from_instance_or_context(instance: pyblish.api.Instance) -> TimeData: ... +def get_transferable_representations(instance: pyblish.api.Instance) -> list: ... +def create_skeleton_instance(instance: pyblish.api.Instance, families_transfer: list = ..., instance_transfer: dict = ...) -> dict: ... +def create_instances_for_aov(instance: pyblish.api.Instance, skeleton: dict) -> List[pyblish.api.Instance]: ...