From 30d598911d78c803b4d5a7316097f53cdadfce9f Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:24:45 +0100 Subject: [PATCH 01/40] removed old integrators --- pype/plugins/global/publish/integrate.py | 417 ----------------- .../publish/integrate_rendered_frames.py | 423 ------------------ 2 files changed, 840 deletions(-) delete mode 100644 pype/plugins/global/publish/integrate.py delete mode 100644 pype/plugins/global/publish/integrate_rendered_frames.py diff --git a/pype/plugins/global/publish/integrate.py b/pype/plugins/global/publish/integrate.py deleted file mode 100644 index 87b9e1a9bd..0000000000 --- a/pype/plugins/global/publish/integrate.py +++ /dev/null @@ -1,417 +0,0 @@ -import os -import logging -import shutil - -import errno -import pyblish.api -from avalon import api, io -from avalon.vendor import filelink - - -log = logging.getLogger(__name__) - - -class IntegrateAsset(pyblish.api.InstancePlugin): - """Resolve any dependency issies - - This plug-in resolves any paths which, if not updated might break - the published file. - - The order of families is important, when working with lookdev you want to - first publish the texture, update the texture paths in the nodes and then - publish the shading network. Same goes for file dependent assets. - """ - - label = "Integrate Asset" - order = pyblish.api.IntegratorOrder - families = [] - exclude_families = ["clip"] - - def process(self, instance): - if [ef for ef in self.exclude_families - if instance.data["family"] in ef]: - return - - self.register(instance) - - self.log.info("Integrating Asset in to the database ...") - if instance.data.get('transfer', True): - self.integrate(instance) - - def register(self, instance): - # Required environment variables - PROJECT = api.Session["AVALON_PROJECT"] - ASSET = instance.data.get("asset") or api.Session["AVALON_ASSET"] - LOCATION = api.Session["AVALON_LOCATION"] - - context = instance.context - # Atomicity - # - # Guarantee atomic publishes - each asset contains - # an identical set of members. - # __ - # / o - # / \ - # | o | - # \ / - # o __/ - # - assert all(result["success"] for result in context.data["results"]), ( - "Atomicity not held, aborting.") - - # Assemble - # - # | - # v - # ---> <---- - # ^ - # | - # - stagingdir = instance.data.get("stagingDir") - assert stagingdir, ("Incomplete instance \"%s\": " - "Missing reference to staging area." % instance) - - # extra check if stagingDir actually exists and is available - - self.log.debug("Establishing staging directory @ %s" % stagingdir) - - # Ensure at least one file is set up for transfer in staging dir. - files = instance.data.get("files", []) - assert files, "Instance has no files to transfer" - assert isinstance(files, (list, tuple)), ( - "Instance 'files' must be a list, got: {0}".format(files) - ) - - project = io.find_one({"type": "project"}) - - asset = io.find_one({ - "type": "asset", - "name": ASSET, - "parent": project["_id"] - }) - - assert all([project, asset]), ("Could not find current project or " - "asset '%s'" % ASSET) - - subset = self.get_subset(asset, instance) - - # get next version - latest_version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - {"name": True}, - sort=[("name", -1)] - ) - - next_version = 1 - if latest_version is not None: - next_version += latest_version["name"] - - self.log.info("Verifying version from assumed destination") - - assumed_data = instance.data["assumedTemplateData"] - assumed_version = assumed_data["version"] - if assumed_version != next_version: - raise AttributeError("Assumed version 'v{0:03d}' does not match" - "next version in database " - "('v{1:03d}')".format(assumed_version, - next_version)) - - self.log.debug("Next version: v{0:03d}".format(next_version)) - - version_data = self.create_version_data(context, instance) - version = self.create_version(subset=subset, - version_number=next_version, - locations=[LOCATION], - data=version_data) - - self.log.debug("Creating version ...") - version_id = io.insert_one(version).inserted_id - - # Write to disk - # _ - # | | - # _| |_ - # ____\ / - # |\ \ / \ - # \ \ v \ - # \ \________. - # \|________| - # - root = api.registered_root() - hierarchy = "" - parents = io.find_one({ - "type": 'asset', - "name": ASSET - })['data']['parents'] - if parents and len(parents) > 0: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = os.path.join(*parents) - - template_data = {"root": root, - "project": {"name": PROJECT, - "code": project['data']['code']}, - "silo": asset['silo'], - "asset": ASSET, - "family": instance.data['family'], - "subset": subset["name"], - "version": int(version["name"]), - "hierarchy": hierarchy} - - # template_publish = project["config"]["template"]["publish"] - anatomy = instance.context.data['anatomy'] - - # Find the representations to transfer amongst the files - # Each should be a single representation (as such, a single extension) - representations = [] - destination_list = [] - if 'transfers' not in instance.data: - instance.data['transfers'] = [] - - for files in instance.data["files"]: - - # Collection - # _______ - # |______|\ - # | |\| - # | || - # | || - # | || - # |_______| - # - - if isinstance(files, list): - collection = files - # Assert that each member has identical suffix - _, ext = os.path.splitext(collection[0]) - assert all(ext == os.path.splitext(name)[1] - for name in collection), ( - "Files had varying suffixes, this is a bug" - ) - - assert not any(os.path.isabs(name) for name in collection) - - template_data["representation"] = ext[1:] - - for fname in collection: - - src = os.path.join(stagingdir, fname) - anatomy_filled = anatomy.format(template_data) - dst = anatomy_filled["publish"]["path"] - - instance.data["transfers"].append([src, dst]) - template = anatomy.templates["publish"]["path"] - - else: - # Single file - # _______ - # | |\ - # | | - # | | - # | | - # |_______| - # - fname = files - assert not os.path.isabs(fname), ( - "Given file name is a full path" - ) - _, ext = os.path.splitext(fname) - - template_data["representation"] = ext[1:] - - src = os.path.join(stagingdir, fname) - anatomy_filled = anatomy.format(template_data) - dst = anatomy_filled["publish"]["path"] - - instance.data["transfers"].append([src, dst]) - template = anatomy.templates["publish"]["path"] - - representation = { - "schema": "pype:representation-2.0", - "type": "representation", - "parent": version_id, - "name": ext[1:], - "data": {'path': dst, 'template': template}, - "dependencies": instance.data.get("dependencies", "").split(), - - # Imprint shortcut to context - # for performance reasons. - "context": { - "root": root, - "project": {"name": PROJECT, - "code": project['data']['code']}, - 'task': api.Session["AVALON_TASK"], - "silo": asset['silo'], - "asset": ASSET, - "family": instance.data['family'], - "subset": subset["name"], - "version": version["name"], - "hierarchy": hierarchy, - "representation": ext[1:] - } - } - - destination_list.append(dst) - instance.data['destination_list'] = destination_list - representations.append(representation) - - self.log.info("Registering {} items".format(len(representations))) - - io.insert_many(representations) - - def integrate(self, instance): - """Move the files - - Through `instance.data["transfers"]` - - Args: - instance: the instance to integrate - """ - - transfers = instance.data.get("transfers", list()) - - for src, dest in transfers: - self.log.info("Copying file .. {} -> {}".format(src, dest)) - self.copy_file(src, dest) - - # Produce hardlinked copies - # Note: hardlink can only be produced between two files on the same - # server/disk and editing one of the two will edit both files at once. - # As such it is recommended to only make hardlinks between static files - # to ensure publishes remain safe and non-edited. - hardlinks = instance.data.get("hardlinks", list()) - for src, dest in hardlinks: - self.log.info("Hardlinking file .. {} -> {}".format(src, dest)) - self.hardlink_file(src, dest) - - def copy_file(self, src, dst): - """ Copy given source to destination - - Arguments: - src (str): the source file which needs to be copied - dst (str): the destination of the sourc file - Returns: - None - """ - - dirname = os.path.dirname(dst) - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST: - pass - else: - self.log.critical("An unexpected error occurred.") - raise - - shutil.copy(src, dst) - - def hardlink_file(self, src, dst): - - dirname = os.path.dirname(dst) - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST: - pass - else: - self.log.critical("An unexpected error occurred.") - raise - - filelink.create(src, dst, filelink.HARDLINK) - - def get_subset(self, asset, instance): - - subset = io.find_one({ - "type": "subset", - "parent": asset["_id"], - "name": instance.data["subset"] - }) - - if subset is None: - subset_name = instance.data["subset"] - self.log.info("Subset '%s' not found, creating.." % subset_name) - - _id = io.insert_one({ - "schema": "avalon-core:subset-2.0", - "type": "subset", - "name": subset_name, - "data": {}, - "parent": asset["_id"] - }).inserted_id - - subset = io.find_one({"_id": _id}) - - return subset - - def create_version(self, subset, version_number, locations, data=None): - """ Copy given source to destination - - Args: - subset (dict): the registered subset of the asset - version_number (int): the version number - locations (list): the currently registered locations - - Returns: - dict: collection of data to create a version - """ - # Imprint currently registered location - version_locations = [location for location in locations if - location is not None] - - return {"schema": "avalon-core:version-2.0", - "type": "version", - "parent": subset["_id"], - "name": version_number, - "locations": version_locations, - "data": data} - - def create_version_data(self, context, instance): - """Create the data collection for the version - - Args: - context: the current context - instance: the current instance being published - - Returns: - dict: the required information with instance.data as key - """ - - families = [] - current_families = instance.data.get("families", list()) - instance_family = instance.data.get("family", None) - - if instance_family is not None: - families.append(instance_family) - families += current_families - - self.log.debug("Registered root: {}".format(api.registered_root())) - # create relative source path for DB - try: - source = instance.data['source'] - except KeyError: - source = context.data["currentFile"] - - relative_path = os.path.relpath(source, api.registered_root()) - source = os.path.join("{root}", relative_path).replace("\\", "/") - - self.log.debug("Source: {}".format(source)) - version_data = {"families": families, - "time": context.data["time"], - "author": context.data["user"], - "source": source, - "comment": context.data.get("comment"), - "machine": context.data.get("machine"), - "fps": context.data.get("fps")} - - # Include optional data if present in - optionals = [ - "frameStart", "frameEnd", "step", "handles", "sourceHashes" - ] - for key in optionals: - if key in instance.data: - version_data[key] = instance.data[key] - - return version_data diff --git a/pype/plugins/global/publish/integrate_rendered_frames.py b/pype/plugins/global/publish/integrate_rendered_frames.py deleted file mode 100644 index 5819051146..0000000000 --- a/pype/plugins/global/publish/integrate_rendered_frames.py +++ /dev/null @@ -1,423 +0,0 @@ -import os -import logging -import shutil -import clique - -import errno -import pyblish.api -from avalon import api, io - - -log = logging.getLogger(__name__) - - -class IntegrateFrames(pyblish.api.InstancePlugin): - """Resolve any dependency issies - - This plug-in resolves any paths which, if not updated might break - the published file. - - The order of families is important, when working with lookdev you want to - first publish the texture, update the texture paths in the nodes and then - publish the shading network. Same goes for file dependent assets. - """ - - label = "Integrate Frames" - order = pyblish.api.IntegratorOrder - families = ["imagesequence"] - - family_targets = [".frames", ".local", ".review", "imagesequence", "render", "source"] - exclude_families = ["clip"] - - def process(self, instance): - if [ef for ef in self.exclude_families - if instance.data["family"] in ef]: - return - - families = [f for f in instance.data["families"] - for search in self.family_targets - if search in f] - - if not families: - return - - self.register(instance) - - # self.log.info("Integrating Asset in to the database ...") - # self.log.info("instance.data: {}".format(instance.data)) - if instance.data.get('transfer', True): - self.integrate(instance) - - def register(self, instance): - - # Required environment variables - PROJECT = api.Session["AVALON_PROJECT"] - ASSET = instance.data.get("asset") or api.Session["AVALON_ASSET"] - LOCATION = api.Session["AVALON_LOCATION"] - - context = instance.context - # Atomicity - # - # Guarantee atomic publishes - each asset contains - # an identical set of members. - # __ - # / o - # / \ - # | o | - # \ / - # o __/ - # - assert all(result["success"] for result in context.data["results"]), ( - "Atomicity not held, aborting.") - - # Assemble - # - # | - # v - # ---> <---- - # ^ - # | - # - stagingdir = instance.data.get("stagingDir") - assert stagingdir, ("Incomplete instance \"%s\": " - "Missing reference to staging area." % instance) - - # extra check if stagingDir actually exists and is available - - self.log.debug("Establishing staging directory @ %s" % stagingdir) - - project = io.find_one({"type": "project"}) - - asset = io.find_one({ - "type": "asset", - "name": ASSET, - "parent": project["_id"] - }) - - assert all([project, asset]), ("Could not find current project or " - "asset '%s'" % ASSET) - - subset = self.get_subset(asset, instance) - - # get next version - latest_version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - {"name": True}, - sort=[("name", -1)] - ) - - next_version = 1 - if latest_version is not None: - next_version += latest_version["name"] - - self.log.info("Verifying version from assumed destination") - - assumed_data = instance.data["assumedTemplateData"] - assumed_version = assumed_data["version"] - if assumed_version != next_version: - raise AttributeError("Assumed version 'v{0:03d}' does not match" - "next version in database " - "('v{1:03d}')".format(assumed_version, - next_version)) - - if instance.data.get('version'): - next_version = int(instance.data.get('version')) - - self.log.debug("Next version: v{0:03d}".format(next_version)) - - version_data = self.create_version_data(context, instance) - version = self.create_version(subset=subset, - version_number=next_version, - locations=[LOCATION], - data=version_data) - - self.log.debug("Creating version ...") - version_id = io.insert_one(version).inserted_id - - # Write to disk - # _ - # | | - # _| |_ - # ____\ / - # |\ \ / \ - # \ \ v \ - # \ \________. - # \|________| - # - root = api.registered_root() - hierarchy = "" - parents = io.find_one({"type": 'asset', "name": ASSET})[ - 'data']['parents'] - if parents and len(parents) > 0: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = os.path.join(*parents) - - template_data = {"root": root, - "project": {"name": PROJECT, - "code": project['data']['code']}, - "silo": asset.get('silo'), - "task": api.Session["AVALON_TASK"], - "asset": ASSET, - "family": instance.data['family'], - "subset": subset["name"], - "version": int(version["name"]), - "hierarchy": hierarchy} - - # template_publish = project["config"]["template"]["publish"] - anatomy = instance.context.data['anatomy'] - - # Find the representations to transfer amongst the files - # Each should be a single representation (as such, a single extension) - representations = [] - destination_list = [] - - if 'transfers' not in instance.data: - instance.data['transfers'] = [] - - for files in instance.data["files"]: - # Collection - # _______ - # |______|\ - # | |\| - # | || - # | || - # | || - # |_______| - # - if isinstance(files, list): - - src_collections, remainder = clique.assemble(files) - src_collection = src_collections[0] - # Assert that each member has identical suffix - src_head = src_collection.format("{head}") - src_tail = ext = src_collection.format("{tail}") - - test_dest_files = list() - for i in [1, 2]: - template_data["representation"] = src_tail[1:] - template_data["frame"] = src_collection.format( - "{padding}") % i - anatomy_filled = anatomy.format(template_data) - test_dest_files.append(anatomy_filled["render"]["path"]) - - dst_collections, remainder = clique.assemble(test_dest_files) - dst_collection = dst_collections[0] - dst_head = dst_collection.format("{head}") - dst_tail = dst_collection.format("{tail}") - - for i in src_collection.indexes: - src_padding = src_collection.format("{padding}") % i - src_file_name = "{0}{1}{2}".format( - src_head, src_padding, src_tail) - dst_padding = dst_collection.format("{padding}") % i - dst = "{0}{1}{2}".format(dst_head, dst_padding, dst_tail) - - src = os.path.join(stagingdir, src_file_name) - instance.data["transfers"].append([src, dst]) - - else: - # Single file - # _______ - # | |\ - # | | - # | | - # | | - # |_______| - # - - template_data.pop("frame", None) - - fname = files - - self.log.info("fname: {}".format(fname)) - - assert not os.path.isabs(fname), ( - "Given file name is a full path" - ) - _, ext = os.path.splitext(fname) - - template_data["representation"] = ext[1:] - - src = os.path.join(stagingdir, fname) - - anatomy_filled = anatomy.format(template_data) - dst = anatomy_filled["render"]["path"] - - instance.data["transfers"].append([src, dst]) - - if ext[1:] not in ["jpeg", "jpg", "mov", "mp4", "wav"]: - template_data["frame"] = "#" * int(anatomy_filled["render"]["padding"]) - - anatomy_filled = anatomy.format(template_data) - path_to_save = anatomy_filled["render"]["path"] - template = anatomy.templates["render"]["path"] - - self.log.debug("path_to_save: {}".format(path_to_save)) - - representation = { - "schema": "pype:representation-2.0", - "type": "representation", - "parent": version_id, - "name": ext[1:], - "data": {'path': path_to_save, 'template': template}, - "dependencies": instance.data.get("dependencies", "").split(), - - # Imprint shortcut to context - # for performance reasons. - "context": { - "root": root, - "project": { - "name": PROJECT, - "code": project['data']['code'] - }, - "task": api.Session["AVALON_TASK"], - "silo": asset['silo'], - "asset": ASSET, - "family": instance.data['family'], - "subset": subset["name"], - "version": int(version["name"]), - "hierarchy": hierarchy, - "representation": ext[1:] - } - } - - destination_list.append(dst) - instance.data['destination_list'] = destination_list - representations.append(representation) - - self.log.info("Registering {} items".format(len(representations))) - io.insert_many(representations) - - def integrate(self, instance): - """Move the files - - Through `instance.data["transfers"]` - - Args: - instance: the instance to integrate - """ - - transfers = instance.data["transfers"] - - for src, dest in transfers: - src = os.path.normpath(src) - dest = os.path.normpath(dest) - if src in dest: - continue - - self.log.info("Copying file .. {} -> {}".format(src, dest)) - self.copy_file(src, dest) - - def copy_file(self, src, dst): - """ Copy given source to destination - - Arguments: - src (str): the source file which needs to be copied - dst (str): the destination of the sourc file - Returns: - None - """ - - dirname = os.path.dirname(dst) - try: - os.makedirs(dirname) - except OSError as e: - if e.errno == errno.EEXIST: - pass - else: - self.log.critical("An unexpected error occurred.") - raise - - shutil.copy(src, dst) - - def get_subset(self, asset, instance): - - subset = io.find_one({ - "type": "subset", - "parent": asset["_id"], - "name": instance.data["subset"] - }) - - if subset is None: - subset_name = instance.data["subset"] - self.log.info("Subset '%s' not found, creating.." % subset_name) - - _id = io.insert_one({ - "schema": "pype:subset-2.0", - "type": "subset", - "name": subset_name, - "data": {}, - "parent": asset["_id"] - }).inserted_id - - subset = io.find_one({"_id": _id}) - - return subset - - def create_version(self, subset, version_number, locations, data=None): - """ Copy given source to destination - - Args: - subset (dict): the registered subset of the asset - version_number (int): the version number - locations (list): the currently registered locations - - Returns: - dict: collection of data to create a version - """ - # Imprint currently registered location - version_locations = [location for location in locations if - location is not None] - - return {"schema": "pype:version-2.0", - "type": "version", - "parent": subset["_id"], - "name": version_number, - "locations": version_locations, - "data": data} - - def create_version_data(self, context, instance): - """Create the data collection for the version - - Args: - context: the current context - instance: the current instance being published - - Returns: - dict: the required information with instance.data as key - """ - - families = [] - current_families = instance.data.get("families", list()) - instance_family = instance.data.get("family", None) - - if instance_family is not None: - families.append(instance_family) - families += current_families - - try: - source = instance.data['source'] - except KeyError: - source = context.data["currentFile"] - - relative_path = os.path.relpath(source, api.registered_root()) - source = os.path.join("{root}", relative_path).replace("\\", "/") - - version_data = {"families": families, - "time": context.data["time"], - "author": context.data["user"], - "source": source, - "comment": context.data.get("comment")} - - # Include optional data if present in - optionals = ["frameStart", "frameEnd", "step", - "handles", "colorspace", "fps", "outputDir"] - - for key in optionals: - if key in instance.data: - version_data[key] = instance.data.get(key, None) - - return version_data From 9bcdf7f72a96b9839aee1e9c49acb75475b55cc8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:25:33 +0100 Subject: [PATCH 02/40] added avalon entities collector --- .../global/publish/collect_avalon_entities.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 pype/plugins/global/publish/collect_avalon_entities.py diff --git a/pype/plugins/global/publish/collect_avalon_entities.py b/pype/plugins/global/publish/collect_avalon_entities.py new file mode 100644 index 0000000000..c256dffd52 --- /dev/null +++ b/pype/plugins/global/publish/collect_avalon_entities.py @@ -0,0 +1,46 @@ +"""Collect Anatomy and global anatomy data. + +Requires: + session -> AVALON_PROJECT, AVALON_ASSET + +Provides: + context -> projectEntity - project entity from database + context -> assetEntity - asset entity from database +""" + +from avalon import io, api +import pyblish.api + + +class CollectAvalonEntities(pyblish.api.ContextPlugin): + """Collect Anatomy into Context""" + + order = pyblish.api.CollectorOrder + label = "Collect Avalon Entities" + + def process(self, context): + project_name = api.Session["AVALON_PROJECT"] + asset_name = api.Session["AVALON_ASSET"] + + project_entity = io.find_one({ + "type": "project", + "name": project_name + }) + assert project_entity, ( + "Project '{0}' was not found." + ).format(project_name) + self.log.debug("Collected Project entity \"{}\"".format(project_entity)) + + asset_entity = io.find_one({ + "type": "asset", + "name": asset_name, + "parent": project_entity["_id"] + }) + assert asset_entity, ( + "No asset found by the name '{0}' in project '{1}'" + ).format(asset_name, project_name) + + self.log.debug("Collected Asset entity \"{}\"".format(asset_entity)) + + context.data["projectEntity"] = project_entity + context.data["assetEntity"] = asset_entity From a2d75afe7a8e78fd2481c18a095aa96b9382a9e7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:25:57 +0100 Subject: [PATCH 03/40] collect anatomy also collect global anatomy data --- .../plugins/global/publish/collect_anatomy.py | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/collect_anatomy.py b/pype/plugins/global/publish/collect_anatomy.py index 9412209850..0831c16d32 100644 --- a/pype/plugins/global/publish/collect_anatomy.py +++ b/pype/plugins/global/publish/collect_anatomy.py @@ -1,10 +1,24 @@ -""" +"""Collect Anatomy and global anatomy data. + Requires: - None + session -> AVALON_TASK + projectEntity, assetEntity -> collect_avalon_entities *(pyblish.api.CollectorOrder) + username -> collect_pype_user *(pyblish.api.CollectorOrder + 0.001) + datetimeData -> collect_datetime_data *(pyblish.api.CollectorOrder) + +Optional: + comment -> collect_comment *(pyblish.api.CollectorOrder) + intent -> collected in pyblish-lite + Provides: context -> anatomy (pypeapp.Anatomy) + context -> anatomyData """ +import os +import json + +from avalon import io, api, lib from pypeapp import Anatomy import pyblish.api @@ -12,9 +26,52 @@ import pyblish.api class CollectAnatomy(pyblish.api.ContextPlugin): """Collect Anatomy into Context""" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder + 0.002 label = "Collect Anatomy" def process(self, context): - context.data['anatomy'] = Anatomy() - self.log.info("Anatomy templates collected...") + root_path = api.registered_root() + task_name = api.Session["AVALON_TASK"] + + project_entity = context.data["projectEntity"] + asset_entity = context.data["assetEntity"] + + project_name = project_entity["name"] + + context.data["anatomy"] = Anatomy(project_name) + self.log.info( + "Anatomy object collected for project \"{}\".".format(project_name) + ) + + hierarchy_items = asset_entity["data"]["parents"] + hierarchy = "" + if hierarchy_items: + hierarchy = os.path.join(*hierarchy_items) + + context_data = { + "root": root_path, + "project": { + "name": project_name, + "code": project_entity["data"].get("code") + }, + "asset": asset_entity["name"], + "hierarchy": hierarchy.replace("\\", "/"), + "task": task_name, + + "username": context.data["user"] + } + + avalon_app_name = os.environ.get("AVALON_APP_NAME") + if avalon_app_name: + application_def = lib.get_application(avalon_app_name) + app_dir = application_def.get("application_dir") + if app_dir: + context_data["app"] = app_dir + + datetime_data = context.data.get("datetimeData") or {} + context_data.update(datetime_data) + + context.data["anatomyData"] = context_data + + self.log.info("Global anatomy Data collected") + self.log.debug(json.dumps(context_data, indent=4)) From 54f76e7f7f9ec884bdbbe915a5088e7aaf8e3e10 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:26:19 +0100 Subject: [PATCH 04/40] collect templates replaced with collect instance anatomy data --- .../publish/collect_instance_anatomy_data.py | 119 ++++++++++++++++++ .../global/publish/collect_templates.py | 117 ----------------- 2 files changed, 119 insertions(+), 117 deletions(-) create mode 100644 pype/plugins/global/publish/collect_instance_anatomy_data.py delete mode 100644 pype/plugins/global/publish/collect_templates.py diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py new file mode 100644 index 0000000000..a1a9278d2a --- /dev/null +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -0,0 +1,119 @@ +""" +Requires: + context -> anatomyData + context -> projectEntity + context -> assetEntity + instance -> asset + instance -> subset + instance -> family + +Optional: + instance -> resolutionWidth + instance -> resolutionHeight + instance -> fps + +Provides: + instance -> anatomyData +""" + +import copy +import json + +from avalon import io +import pyblish.api + + +class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): + """Fill templates with data needed for publish""" + + order = pyblish.api.CollectorOrder + 0.1 + label = "Collect instance anatomy data" + hosts = ["maya", "nuke", "standalonepublisher"] + + def process(self, instance): + # get all the stuff from the database + anatomy_data = copy.deepcopy(instance.context.data["anatomyData"]) + project_entity = instance.context.data["projectEntity"] + context_asset_entity = instance.context.data["assetEntity"] + + asset_name = instance.data["asset"] + # Check if asset name is the same as what is in context + # - they may be different, e.g. in NukeStudio + if context_asset_entity["name"] == asset_name: + asset_entity = context_asset_entity + + else: + asset_entity = io.find_one({ + "type": "asset", + "name": asset_name, + "parent": project_entity["_id"] + }) + + instance.context.data["assetEntity"] = asset_entity + instance.context.data["projectEntity"] = project_entity + + subset_name = instance.data["subset"] + subset_entity = io.find_one({ + "type": "subset", + "name": subset_name, + "parent": asset_entity["_id"] + }) + + version_number = instance.data.get("version") + if version_number is None: + version_number = instance.context.data.get("version") + + latest_version = None + if subset_entity is None: + self.log.debug("Subset entity does not exist yet.") + else: + version_entity = io.find_one( + { + "type": "version", + "parent": subset_entity["_id"] + }, + sort=[("name", -1)] + ) + if version_entity: + latest_version = version_entity["name"] + + if version_number is None: + # TODO we should be able to change this version by studio + # preferences (like start with version number `0`) + version_number = 1 + if latest_version is not None: + version_number += int(latest_version) + + # Version should not be collected since may be instance + anatomy_data.update({ + "asset": asset_entity["name"], + "family": instance.data["family"], + "subset": subset_name, + "version": version_number + }) + + resolution_width = instance.data.get("resolutionWidth") + if resolution_width: + anatomy_data["resolution_width"] = resolution_width + + resolution_height = instance.data.get("resolutionHeight") + if resolution_height: + anatomy_data["resolution_height"] = resolution_height + + fps = instance.data.get("fps") + if resolution_height: + anatomy_data["fps"] = fps + + instance.data["anatomyData"] = anatomy_data + instance.data["latestVersion"] = latest_version + # TODO check if template is used anywhere + # instance.data["template"] = template + + # TODO we should move this to any Validator + # # We take the parent folder of representation 'filepath' + # instance.data["assumedDestination"] = os.path.dirname( + # (anatomy.format(template_data))["publish"]["path"] + # ) + + self.log.info("Instance anatomy Data collected") + self.log.debug(json.dumps(anatomy_data, indent=4)) diff --git a/pype/plugins/global/publish/collect_templates.py b/pype/plugins/global/publish/collect_templates.py deleted file mode 100644 index f065b3c246..0000000000 --- a/pype/plugins/global/publish/collect_templates.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Requires: - session -> AVALON_PROJECT - context -> anatomy (pypeapp.Anatomy) - instance -> subset - instance -> asset - instance -> family - -Provides: - instance -> template - instance -> assumedTemplateData - instance -> assumedDestination -""" - -import os - -from avalon import io, api -import pyblish.api - - -class CollectTemplates(pyblish.api.InstancePlugin): - """Fill templates with data needed for publish""" - - order = pyblish.api.CollectorOrder + 0.1 - label = "Collect and fill Templates" - hosts = ["maya", "nuke", "standalonepublisher"] - - def process(self, instance): - # get all the stuff from the database - subset_name = instance.data["subset"] - asset_name = instance.data["asset"] - project_name = api.Session["AVALON_PROJECT"] - - project = io.find_one( - { - "type": "project", - "name": project_name - }, - projection={"config": True, "data": True} - ) - - template = project["config"]["template"]["publish"] - anatomy = instance.context.data['anatomy'] - - asset = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project["_id"] - }) - - assert asset, ("No asset found by the name '{}' " - "in project '{}'".format(asset_name, project_name)) - silo = asset.get('silo') - - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset["_id"] - }) - - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - - # if there is a subset there ought to be version - if version is not None: - version_number += int(version["name"]) - - hierarchy = asset['data']['parents'] - if hierarchy: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = os.path.join(*hierarchy) - - template_data = {"root": api.Session["AVALON_PROJECTS"], - "project": {"name": project_name, - "code": project['data']['code']}, - "silo": silo, - "family": instance.data['family'], - "asset": asset_name, - "subset": subset_name, - "version": version_number, - "hierarchy": hierarchy.replace("\\", "/"), - "representation": "TEMP"} - - # Add datetime data to template data - datetime_data = instance.context.data.get("datetimeData") or {} - template_data.update(datetime_data) - - resolution_width = instance.data.get("resolutionWidth") - resolution_height = instance.data.get("resolutionHeight") - fps = instance.data.get("fps") - - if resolution_width: - template_data["resolution_width"] = resolution_width - if resolution_width: - template_data["resolution_height"] = resolution_height - if resolution_width: - template_data["fps"] = fps - - instance.data["template"] = template - instance.data["assumedTemplateData"] = template_data - - # We take the parent folder of representation 'filepath' - instance.data["assumedDestination"] = os.path.dirname( - (anatomy.format(template_data))["publish"]["path"] - ) - self.log.info("Assumed Destination has been created...") - self.log.debug("__ assumedTemplateData: `{}`".format(instance.data["assumedTemplateData"])) - self.log.debug("__ template: `{}`".format(instance.data["template"])) From 1515f47f0fad2700efaa69022ac682456b7e4c50 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:26:33 +0100 Subject: [PATCH 05/40] extract burnin uses anatomyData --- pype/plugins/global/publish/extract_burnin.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index e50ba891d2..b95c15f340 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -32,21 +32,15 @@ class ExtractBurnin(pype.api.Extractor): frame_end = int(instance.data.get("frameEnd") or 1) duration = frame_end - frame_start + 1 - prep_data = { - "username": instance.context.data['user'], - "asset": os.environ['AVALON_ASSET'], - "task": os.environ['AVALON_TASK'], + prep_data = copy.deepcopy(instance.data["anatomyData"]) + prep_data.update({ "frame_start": frame_start, "frame_end": frame_end, "duration": duration, "version": int(version), "comment": instance.context.data.get("comment", ""), "intent": instance.context.data.get("intent", "") - } - - # Add datetime data to preparation data - datetime_data = instance.context.data.get("datetimeData") or {} - prep_data.update(datetime_data) + }) slate_frame_start = frame_start slate_frame_end = frame_end @@ -64,10 +58,6 @@ class ExtractBurnin(pype.api.Extractor): "slate_duration": slate_duration }) - # Update data with template data - template_data = instance.data.get("assumedTemplateData") or {} - prep_data.update(template_data) - # get anatomy project anatomy = instance.context.data['anatomy'] From e4b23553dffd42aad2ee07b5e207787b1b52c4f8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:26:55 +0100 Subject: [PATCH 06/40] integrate assumed destinatoin replaced with collect resources path --- .../global/publish/collect_resources_path.py | 132 ++++++++++++++++ .../publish/integrate_assumed_destination.py | 147 ------------------ 2 files changed, 132 insertions(+), 147 deletions(-) create mode 100644 pype/plugins/global/publish/collect_resources_path.py delete mode 100644 pype/plugins/global/publish/integrate_assumed_destination.py diff --git a/pype/plugins/global/publish/collect_resources_path.py b/pype/plugins/global/publish/collect_resources_path.py new file mode 100644 index 0000000000..52e926e09c --- /dev/null +++ b/pype/plugins/global/publish/collect_resources_path.py @@ -0,0 +1,132 @@ +import os +import copy + +import pyblish.api +from avalon import io + + +class IntegrateResourcesPath(pyblish.api.InstancePlugin): + """Generate the assumed destination path where the file will be stored""" + + label = "Integrate Prepare Resource" + order = pyblish.api.IntegratorOrder - 0.05 + families = ["clip", "projectfile", "plate"] + + def process(self, instance): + project_entity = instance.context["projectEntity"] + asset_entity = instance.context["assetEntity"] + + template_data = copy.deepcopy(instance.data["anatomyData"]) + + asset_name = instance.data["asset"] + if asset_name != asset_entity["name"]: + asset_entity = io.find_one({ + "type": "asset", + "name": asset_name, + "parent": project_entity["_id"] + }) + assert asset_entity, ( + "No asset found by the name '{}' in project '{}'".format( + asset_name, project_entity["name"] + ) + ) + + instance.data["assetEntity"] = asset_entity + + template_data["name"] = asset_entity["name"] + silo_name = asset_entity.get("silo") + if silo_name: + template_data["silo"] = silo_name + + parents = asset_entity["data"].get("parents") or [] + hierarchy = "/".join(parents) + template_data["hierarchy"] = hierarchy + + subset_name = instance.data["subset"] + self.log.info(subset_name) + + subset = io.find_one({ + "type": "subset", + "name": subset_name, + "parent": asset_entity["_id"] + }) + + # assume there is no version yet, we start at `1` + version = None + version_number = 1 + if subset is not None: + version = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[("name", -1)] + ) + + # if there is a subset there ought to be version + if version is not None: + version_number += version["name"] + + if instance.data.get('version'): + version_number = int(instance.data.get('version')) + + anatomy = instance.context.data["anatomy"] + padding = int(anatomy.templates['render']['padding']) + + template_data.update({ + "subset": subset_name, + "frame": ('#' * padding), + "version": version_number, + "representation": "TEMP" + }) + + anatomy_filled = anatomy.format(template_data) + + template_names = ["publish"] + for repre in instance.data["representations"]: + template_name = repre.get("anatomy_template") + if template_name and template_name not in template_names: + template_names.append(template_name) + + resources = instance.data.get("resources", list()) + transfers = instance.data.get("transfers", list()) + + for template_name in template_names: + mock_template = anatomy_filled[template_name]["path"] + + # For now assume resources end up in a "resources" folder in the + # published folder + mock_destination = os.path.join( + os.path.dirname(mock_template), "resources" + ) + + # Clean the path + mock_destination = os.path.abspath( + os.path.normpath(mock_destination) + ).replace("\\", "/") + + # Define resource destination and transfers + for resource in resources: + # Add destination to the resource + source_filename = os.path.basename( + resource["source"]).replace("\\", "/") + destination = os.path.join(mock_destination, source_filename) + + # Force forward slashes to fix issue with software unable + # to work correctly with backslashes in specific scenarios + # (e.g. escape characters in PLN-151 V-Ray UDIM) + destination = destination.replace("\\", "/") + + resource['destination'] = destination + + # Collect transfers for the individual files of the resource + # e.g. all individual files of a cache or UDIM textures. + files = resource['files'] + for fsrc in files: + fname = os.path.basename(fsrc) + fdest = os.path.join( + mock_destination, fname).replace("\\", "/") + transfers.append([fsrc, fdest]) + + instance.data["resources"] = resources + instance.data["transfers"] = transfers diff --git a/pype/plugins/global/publish/integrate_assumed_destination.py b/pype/plugins/global/publish/integrate_assumed_destination.py deleted file mode 100644 index d090e2711a..0000000000 --- a/pype/plugins/global/publish/integrate_assumed_destination.py +++ /dev/null @@ -1,147 +0,0 @@ -import pyblish.api -import os - -from avalon import io, api - - -class IntegrateAssumedDestination(pyblish.api.InstancePlugin): - """Generate the assumed destination path where the file will be stored""" - - label = "Integrate Assumed Destination" - order = pyblish.api.IntegratorOrder - 0.05 - families = ["clip", "projectfile", "plate"] - - def process(self, instance): - - anatomy = instance.context.data['anatomy'] - - self.create_destination_template(instance, anatomy) - - template_data = instance.data["assumedTemplateData"] - # self.log.info(anatomy.templates) - anatomy_filled = anatomy.format(template_data) - - # self.log.info(anatomy_filled) - mock_template = anatomy_filled["publish"]["path"] - - # For now assume resources end up in a "resources" folder in the - # published folder - mock_destination = os.path.join(os.path.dirname(mock_template), - "resources") - - # Clean the path - mock_destination = os.path.abspath( - os.path.normpath(mock_destination)).replace("\\", "/") - - # Define resource destination and transfers - resources = instance.data.get("resources", list()) - transfers = instance.data.get("transfers", list()) - for resource in resources: - - # Add destination to the resource - source_filename = os.path.basename( - resource["source"]).replace("\\", "/") - destination = os.path.join(mock_destination, source_filename) - - # Force forward slashes to fix issue with software unable - # to work correctly with backslashes in specific scenarios - # (e.g. escape characters in PLN-151 V-Ray UDIM) - destination = destination.replace("\\", "/") - - resource['destination'] = destination - - # Collect transfers for the individual files of the resource - # e.g. all individual files of a cache or UDIM textures. - files = resource['files'] - for fsrc in files: - fname = os.path.basename(fsrc) - fdest = os.path.join( - mock_destination, fname).replace("\\", "/") - transfers.append([fsrc, fdest]) - - instance.data["resources"] = resources - instance.data["transfers"] = transfers - - def create_destination_template(self, instance, anatomy): - """Create a filepath based on the current data available - - Example template: - {root}/{project}/{asset}/publish/{subset}/v{version:0>3}/ - {subset}.{representation} - Args: - instance: the instance to publish - - Returns: - file path (str) - """ - - # get all the stuff from the database - subset_name = instance.data["subset"] - self.log.info(subset_name) - asset_name = instance.data["asset"] - project_name = api.Session["AVALON_PROJECT"] - a_template = anatomy.templates - - project = io.find_one( - {"type": "project", "name": project_name}, - projection={"config": True, "data": True} - ) - - template = a_template['publish']['path'] - # anatomy = instance.context.data['anatomy'] - - asset = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project["_id"] - }) - - assert asset, ("No asset found by the name '{}' " - "in project '{}'".format(asset_name, project_name)) - - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset["_id"] - }) - - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - - # if there is a subset there ought to be version - if version is not None: - version_number += version["name"] - - if instance.data.get('version'): - version_number = int(instance.data.get('version')) - - padding = int(a_template['render']['padding']) - - hierarchy = asset['data']['parents'] - if hierarchy: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = "/".join(hierarchy) - - template_data = {"root": api.Session["AVALON_PROJECTS"], - "project": {"name": project_name, - "code": project['data']['code']}, - "family": instance.data['family'], - "asset": asset_name, - "subset": subset_name, - "frame": ('#' * padding), - "version": version_number, - "hierarchy": hierarchy, - "representation": "TEMP"} - - instance.data["assumedTemplateData"] = template_data - self.log.info(template_data) - instance.data["template"] = template From f6992a3d44532fac0d11a87c5d7cdfe0a0db715d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:27:20 +0100 Subject: [PATCH 07/40] collector for project data was removed --- .../global/publish/collect_project_data.py | 24 ------------------- .../nukestudio/publish/collect_clips.py | 2 +- 2 files changed, 1 insertion(+), 25 deletions(-) delete mode 100644 pype/plugins/global/publish/collect_project_data.py diff --git a/pype/plugins/global/publish/collect_project_data.py b/pype/plugins/global/publish/collect_project_data.py deleted file mode 100644 index acdbc2c41f..0000000000 --- a/pype/plugins/global/publish/collect_project_data.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Requires: - None - -Provides: - context -> projectData -""" - -import pyblish.api -import pype.api as pype - - -class CollectProjectData(pyblish.api.ContextPlugin): - """Collecting project data from avalon db""" - - label = "Collect Project Data" - order = pyblish.api.CollectorOrder - 0.1 - hosts = ["nukestudio"] - - def process(self, context): - # get project data from avalon db - context.data["projectData"] = pype.get_project()["data"] - - return diff --git a/pype/plugins/nukestudio/publish/collect_clips.py b/pype/plugins/nukestudio/publish/collect_clips.py index 3759d50f6a..82053b6811 100644 --- a/pype/plugins/nukestudio/publish/collect_clips.py +++ b/pype/plugins/nukestudio/publish/collect_clips.py @@ -17,7 +17,7 @@ class CollectClips(api.ContextPlugin): self.log.debug("Created `assetsShared` in context") context.data["assetsShared"] = dict() - projectdata = context.data["projectData"] + projectdata = context.data["projectEntity"]["data"] version = context.data.get("version", "001") sequence = context.data.get("activeSequence") selection = context.data.get("selection") From 5177b891ac5b1b1be0f19c621630be169b08741d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:27:42 +0100 Subject: [PATCH 08/40] extract yeti rig and extract look uses anatomyData --- pype/plugins/maya/publish/extract_look.py | 69 +++++-------------- pype/plugins/maya/publish/extract_yeti_rig.py | 10 +-- 2 files changed, 23 insertions(+), 56 deletions(-) diff --git a/pype/plugins/maya/publish/extract_look.py b/pype/plugins/maya/publish/extract_look.py index fa6ecd72c3..4000011520 100644 --- a/pype/plugins/maya/publish/extract_look.py +++ b/pype/plugins/maya/publish/extract_look.py @@ -1,6 +1,7 @@ import os import sys import json +import copy import tempfile import contextlib import subprocess @@ -333,7 +334,7 @@ class ExtractLook(pype.api.Extractor): anatomy = instance.context.data["anatomy"] - self.create_destination_template(instance, anatomy) + destination_dir = self.create_destination_template(instance, anatomy) # Compute destination location basename, ext = os.path.splitext(os.path.basename(filepath)) @@ -343,7 +344,7 @@ class ExtractLook(pype.api.Extractor): ext = ".tx" return os.path.join( - instance.data["assumedDestination"], "resources", basename + ext + destination_dir, "resources", basename + ext ) def _process_texture(self, filepath, do_maketx, staging, linearise, force): @@ -421,38 +422,17 @@ class ExtractLook(pype.api.Extractor): file path (str) """ - # get all the stuff from the database + asset_entity = instance.context["assetEntity"] + + template_data = copy.deepcopy(instance.data["anatomyData"]) + subset_name = instance.data["subset"] self.log.info(subset_name) - asset_name = instance.data["asset"] - project_name = api.Session["AVALON_PROJECT"] - a_template = anatomy.templates - - project = io.find_one( - { - "type": "project", - "name": project_name - }, - projection={"config": True, "data": True} - ) - - template = a_template["publish"]["path"] - # anatomy = instance.context.data['anatomy'] - - asset = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project["_id"] - }) - - assert asset, ("No asset found by the name '{}' " - "in project '{}'").format(asset_name, project_name) - silo = asset.get("silo") subset = io.find_one({ "type": "subset", "name": subset_name, - "parent": asset["_id"] + "parent": asset_entity["_id"] }) # assume there is no version yet, we start at `1` @@ -471,33 +451,18 @@ class ExtractLook(pype.api.Extractor): if version is not None: version_number += version["name"] - if instance.data.get("version"): - version_number = int(instance.data.get("version")) + if instance.data.get('version'): + version_number = int(instance.data.get('version')) - padding = int(a_template["render"]["padding"]) + anatomy = instance.context.data["anatomy"] + padding = int(anatomy.templates['render']['padding']) - hierarchy = asset["data"]["parents"] - if hierarchy: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = "/".join(hierarchy) - - template_data = { - "root": api.Session["AVALON_PROJECTS"], - "project": {"name": project_name, "code": project["data"]["code"]}, - "silo": silo, - "family": instance.data["family"], - "asset": asset_name, + template_data.update({ "subset": subset_name, "frame": ("#" * padding), "version": version_number, - "hierarchy": hierarchy, - "representation": "TEMP", - } + "representation": "TEMP" + }) + anatomy_filled = anatomy.format(template_data) - instance.data["assumedTemplateData"] = template_data - self.log.info(template_data) - instance.data["template"] = template - # We take the parent folder of representation 'filepath' - instance.data["assumedDestination"] = os.path.dirname( - anatomy.format(template_data)["publish"]["path"] - ) + return os.path.dirname(anatomy_filled["publish"]["path"]) diff --git a/pype/plugins/maya/publish/extract_yeti_rig.py b/pype/plugins/maya/publish/extract_yeti_rig.py index 892bc0bea6..d390a1365a 100644 --- a/pype/plugins/maya/publish/extract_yeti_rig.py +++ b/pype/plugins/maya/publish/extract_yeti_rig.py @@ -1,6 +1,7 @@ import os import json import contextlib +import copy from maya import cmds @@ -111,11 +112,12 @@ class ExtractYetiRig(pype.api.Extractor): self.log.info("Writing metadata file") # Create assumed destination folder for imageSearchPath - assumed_temp_data = instance.data["assumedTemplateData"] - template = instance.data["template"] - template_formatted = template.format(**assumed_temp_data) + template_data = copy.deepcopy(instance.data["anatomyData"]) - destination_folder = os.path.dirname(template_formatted) + anatomy = instance.context["anatomy"] + filled = anatomy.format(template_data) + + destination_folder = os.path.dir(filled["publish"]["path"]) image_search_path = os.path.join(destination_folder, "resources") image_search_path = os.path.normpath(image_search_path) From f1486a9cd42820684d03c317442a0919b597bfef Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:27:59 +0100 Subject: [PATCH 09/40] integrate_new uses anatomyData --- pype/plugins/global/publish/integrate_new.py | 80 ++++++++------------ 1 file changed, 30 insertions(+), 50 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 7d95534897..c6bc1ffbab 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -2,6 +2,7 @@ import os from os.path import getsize import logging import sys +import copy import clique import errno import pyblish.api @@ -100,12 +101,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): def register(self, instance): # Required environment variables - PROJECT = api.Session["AVALON_PROJECT"] - ASSET = instance.data.get("asset") or api.Session["AVALON_ASSET"] - TASK = instance.data.get("task") or api.Session["AVALON_TASK"] - LOCATION = api.Session["AVALON_LOCATION"] + anatomy_data = instance.data["anatomyData"] + asset_entity = instance.data["assetEntity"] + avalon_location = api.Session["AVALON_LOCATION"] + + io.install() context = instance.context + # Atomicity # # Guarantee atomic publishes - each asset contains @@ -140,35 +143,27 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): # stagingdir = instance.data.get("stagingDir") if not stagingdir: - self.log.info('''{} is missing reference to staging - directory Will try to get it from - representation'''.format(instance)) + self.log.info(( + "{0} is missing reference to staging directory." + " Will try to get it from representation." + ).format(instance)) - # extra check if stagingDir actually exists and is available - - self.log.debug("Establishing staging directory @ %s" % stagingdir) + else: + self.log.debug( + "Establishing staging directory @ {0}".format(stagingdir) + ) # Ensure at least one file is set up for transfer in staging dir. - repres = instance.data.get("representations", None) + repres = instance.data.get("representations") assert repres, "Instance has no files to transfer" assert isinstance(repres, (list, tuple)), ( - "Instance 'files' must be a list, got: {0}".format(repres) + "Instance 'files' must be a list, got: {0} {1}".format( + str(type(repres)), str(repres) + ) ) - # FIXME: io is not initialized at this point for shell host - io.install() - project = io.find_one({"type": "project"}) - - asset = io.find_one({ - "type": "asset", - "name": ASSET, - "parent": project["_id"] - }) - - assert all([project, asset]), ("Could not find current project or " - "asset '%s'" % ASSET) - - subset = self.get_subset(asset, instance) + intent = context.data.get("intent") + subset = self.get_subset(asset_entity, instance) # get next version latest_version = io.find_one( @@ -229,16 +224,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): # \ \________. # \|________| # - root = api.registered_root() - hierarchy = "" - parents = io.find_one({ - "type": 'asset', - "name": ASSET - })['data']['parents'] - if parents and len(parents) > 0: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = os.path.join(*parents) - anatomy = instance.context.data['anatomy'] # Find the representations to transfer amongst the files @@ -261,20 +246,15 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): # |_______| # # create template data for Anatomy - template_data = {"root": root, - "project": {"name": PROJECT, - "code": project['data']['code']}, - "silo": asset.get('silo'), - "task": TASK, - "asset": ASSET, - "family": instance.data['family'], - "subset": subset["name"], - "version": int(version["name"]), - "hierarchy": hierarchy} + template_data = copy.deepcopy(anatomy_data) + # TODO cleanup this code, should be already in anatomyData + template_data.update({ + "subset": subset["name"], + "version": int(version["name"]) + }) - # Add datetime data to template data - datetime_data = context.data.get("datetimeData") or {} - template_data.update(datetime_data) + if intent is not None: + template_data["intent"] = intent resolution_width = repre.get("resolutionWidth") resolution_height = repre.get("resolutionHeight") @@ -292,6 +272,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): stagingdir = repre['stagingDir'] if repre.get('anatomy_template'): template_name = repre['anatomy_template'] + template = os.path.normpath( anatomy.templates[template_name]["path"]) @@ -322,7 +303,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): template_filled = anatomy_filled[template_name]["path"] if repre_context is None: repre_context = template_filled.used_values - test_dest_files.append( os.path.normpath(template_filled) ) From 91d51f145844aed301c3f7a721e807e0dfb154a7 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 3 Feb 2020 19:28:33 +0100 Subject: [PATCH 10/40] removed deprecated validate templates --- .../global/publish/validate_templates.py | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 pype/plugins/global/publish/validate_templates.py diff --git a/pype/plugins/global/publish/validate_templates.py b/pype/plugins/global/publish/validate_templates.py deleted file mode 100644 index f24f6b1a2e..0000000000 --- a/pype/plugins/global/publish/validate_templates.py +++ /dev/null @@ -1,43 +0,0 @@ -import pyblish.api -import os - - -class ValidateTemplates(pyblish.api.ContextPlugin): - """Check if all templates were filled""" - - label = "Validate Templates" - order = pyblish.api.ValidatorOrder - 0.1 - hosts = ["maya", "houdini", "nuke"] - - def process(self, context): - - anatomy = context.data["anatomy"] - if not anatomy: - raise RuntimeError("Did not find anatomy") - else: - data = { - "root": os.environ["PYPE_STUDIO_PROJECTS_PATH"], - "project": {"name": "D001_projectsx", - "code": "prjX"}, - "ext": "exr", - "version": 3, - "task": "animation", - "asset": "sh001", - "app": "maya", - "hierarchy": "ep101/sq01/sh010"} - - anatomy_filled = anatomy.format(data) - self.log.info(anatomy_filled) - - data = {"root": os.environ["PYPE_STUDIO_PROJECTS_PATH"], - "project": {"name": "D001_projectsy", - "code": "prjY"}, - "ext": "abc", - "version": 1, - "task": "lookdev", - "asset": "bob", - "app": "maya", - "hierarchy": "ep101/sq01/bob"} - - anatomy_filled = context.data["anatomy"].format(data) - self.log.info(anatomy_filled["work"]["folder"]) From 670f660a9724a83691913ffbfece7b9ae22cd414 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 10:59:30 +0100 Subject: [PATCH 11/40] changed collect instance anatomy data order to 0.49 --- pype/plugins/global/publish/collect_instance_anatomy_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index a1a9278d2a..76ab8dc3f6 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -26,7 +26,7 @@ import pyblish.api class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): """Fill templates with data needed for publish""" - order = pyblish.api.CollectorOrder + 0.1 + order = pyblish.api.CollectorOrder + 0.49 label = "Collect instance anatomy data" hosts = ["maya", "nuke", "standalonepublisher"] From 1a04dca10a7c2af07d779bc1339c941eb5ebf44d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 11:00:38 +0100 Subject: [PATCH 12/40] asset entity may not exist so collecting instance anatomy data was changed to not crash --- .../publish/collect_instance_anatomy_data.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index 76ab8dc3f6..ecef9d10f2 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -53,40 +53,42 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): instance.context.data["projectEntity"] = project_entity subset_name = instance.data["subset"] - subset_entity = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset_entity["_id"] - }) - version_number = instance.data.get("version") - if version_number is None: - version_number = instance.context.data.get("version") - latest_version = None - if subset_entity is None: - self.log.debug("Subset entity does not exist yet.") - else: - version_entity = io.find_one( - { - "type": "version", - "parent": subset_entity["_id"] - }, - sort=[("name", -1)] - ) - if version_entity: - latest_version = version_entity["name"] + if asset_entity: + subset_entity = io.find_one({ + "type": "subset", + "name": subset_name, + "parent": asset_entity["_id"] + }) + + + if subset_entity is None: + self.log.debug("Subset entity does not exist yet.") + else: + version_entity = io.find_one( + { + "type": "version", + "parent": subset_entity["_id"] + }, + sort=[("name", -1)] + ) + if version_entity: + latest_version = version_entity["name"] + + # If version is not specified for instance or context if version_number is None: # TODO we should be able to change this version by studio # preferences (like start with version number `0`) version_number = 1 + # use latest version (+1) if already any exist if latest_version is not None: version_number += int(latest_version) # Version should not be collected since may be instance anatomy_data.update({ - "asset": asset_entity["name"], + "asset": asset_name, "family": instance.data["family"], "subset": subset_name, "version": version_number From a14b05ccd1b5f75d4ccde0349de1096ec0425592 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 11:00:48 +0100 Subject: [PATCH 13/40] removed comments --- .../global/publish/collect_instance_anatomy_data.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index ecef9d10f2..838fb1a113 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -108,14 +108,6 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): instance.data["anatomyData"] = anatomy_data instance.data["latestVersion"] = latest_version - # TODO check if template is used anywhere - # instance.data["template"] = template - - # TODO we should move this to any Validator - # # We take the parent folder of representation 'filepath' - # instance.data["assumedDestination"] = os.path.dirname( - # (anatomy.format(template_data))["publish"]["path"] - # ) self.log.info("Instance anatomy Data collected") self.log.debug(json.dumps(anatomy_data, indent=4)) From f70f307cc40d95bfa5181e27ef7384108e9fb10b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:16:35 +0100 Subject: [PATCH 14/40] removed missleading comments --- pype/plugins/global/publish/integrate_new.py | 52 -------------------- 1 file changed, 52 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index c6bc1ffbab..c3a03324aa 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -109,38 +109,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): context = instance.context - # Atomicity - # - # Guarantee atomic publishes - each asset contains - # an identical set of members. - # __ - # / o - # / \ - # | o | - # \ / - # o __/ - # - # for result in context.data["results"]: - # if not result["success"]: - # self.log.debug(result) - # exc_type, exc_value, exc_traceback = result["error_info"] - # extracted_traceback = traceback.extract_tb(exc_traceback)[-1] - # self.log.debug( - # "Error at line {}: \"{}\"".format( - # extracted_traceback[1], result["error"] - # ) - # ) - # assert all(result["success"] for result in context.data["results"]),( - # "Atomicity not held, aborting.") - - # Assemble - # - # | - # v - # ---> <---- - # ^ - # | - # stagingdir = instance.data.get("stagingDir") if not stagingdir: self.log.info(( @@ -214,16 +182,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): version_id = existing_version['_id'] instance.data['version'] = version['name'] - # Write to disk - # _ - # | | - # _| |_ - # ____\ / - # |\ \ / \ - # \ \ v \ - # \ \________. - # \|________| - # anatomy = instance.context.data['anatomy'] # Find the representations to transfer amongst the files @@ -235,16 +193,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): instance.data['transfers'] = [] for idx, repre in enumerate(instance.data["representations"]): - - # Collection - # _______ - # |______|\ - # | |\| - # | || - # | || - # | || - # |_______| - # # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) # TODO cleanup this code, should be already in anatomyData From 6ef1a7e17605233bc8ea0dc25e3912d0d0a9dc9d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:17:15 +0100 Subject: [PATCH 15/40] formatting --- pype/plugins/global/publish/integrate_new.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index c3a03324aa..b71b5fb298 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -159,10 +159,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if version_data_instance: version_data.update(version_data_instance) - version = self.create_version(subset=subset, - version_number=next_version, - locations=[LOCATION], - data=version_data) + version = self.create_version( + subset=subset, + version_number=next_version, + locations=[avalon_location], + data=version_data + ) self.log.debug("Creating version ...") existing_version = io.find_one({ From 1dcdac7ae051cb359fb481a812027a553e4c79e1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:18:11 +0100 Subject: [PATCH 16/40] asset_entity check moved back since nukestudio instances may have not set value --- pype/plugins/global/publish/integrate_new.py | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index b71b5fb298..774a54ea7c 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -102,13 +102,37 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): def register(self, instance): # Required environment variables anatomy_data = instance.data["anatomyData"] - asset_entity = instance.data["assetEntity"] avalon_location = api.Session["AVALON_LOCATION"] io.install() context = instance.context + project_entity = instance.data["projectEntity"] + + asset_name = instance.data["asset"] + asset_entity = instance.data.get("assetEntity") + if not asset_entity: + asset_entity = io.find_one({ + "type": "asset", + "name": asset_name, + "parent": project_entity["_id"] + }) + + assert asset_entity, ( + "No asset found by the name \"{0}\" in project \"{1}\"" + ).format(asset_name, project_entity["name"]) + + instance.data["assetEntity"] = asset_entity + + # update anatomy data with asset specific keys + # - name should already been set + hierarchy = "" + parents = asset_entity["data"]["parents"] + if parents: + hierarchy = "/".join(parents) + anatomy_data["hierarchy"] = hierarchy + stagingdir = instance.data.get("stagingDir") if not stagingdir: self.log.info(( From e23cc33de7ad1e9156f1a659a0619e2fc2609f68 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:18:59 +0100 Subject: [PATCH 17/40] latest version is not queried before check if instance have already version set in data --- pype/plugins/global/publish/integrate_new.py | 33 +++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 774a54ea7c..24162c4cf1 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -157,22 +157,27 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): intent = context.data.get("intent") subset = self.get_subset(asset_entity, instance) - # get next version - latest_version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - {"name": True}, - sort=[("name", -1)] - ) + # TODO iLLiCiT use "latestVersion" from `instance.data` + # and store version in anatomyData instance collector + # instead of query again + instance_version = instance.data.get('version') + if instance_version is not None: + next_version = int(instance_version) - next_version = 1 - if latest_version is not None: - next_version += latest_version["name"] + else: + # get next version + latest_version = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + {"name": True}, + sort=[("name", -1)] + ) - if instance.data.get('version'): - next_version = int(instance.data.get('version')) + next_version = 1 + if latest_version is not None: + next_version += int(latest_version["name"]) self.log.debug("Next version: v{0:03d}".format(next_version)) From 29c6768da935380dd499834857b896c64d2b05f5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:19:19 +0100 Subject: [PATCH 18/40] intent added to anatomy data --- pype/plugins/global/publish/integrate_new.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 24162c4cf1..093a9e354c 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -154,7 +154,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): ) ) - intent = context.data.get("intent") subset = self.get_subset(asset_entity, instance) # TODO iLLiCiT use "latestVersion" from `instance.data` @@ -213,6 +212,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): version_id = existing_version['_id'] instance.data['version'] = version['name'] + intent = context.data.get("intent") + if intent is not None: + anatomy_data["intent"] = intent + anatomy = instance.context.data['anatomy'] # Find the representations to transfer amongst the files From 3a5ab92687bbf7cc89ade7ec453997d5189e0f64 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:19:44 +0100 Subject: [PATCH 19/40] removed subset and version anatomy update since they are already set for whole instance --- pype/plugins/global/publish/integrate_new.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 093a9e354c..fc7cbf4afa 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -229,12 +229,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): for idx, repre in enumerate(instance.data["representations"]): # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) - # TODO cleanup this code, should be already in anatomyData - template_data.update({ - "subset": subset["name"], - "version": int(version["name"]) - }) - if intent is not None: template_data["intent"] = intent From 9113fb1c7f72b1e1ad7a0e32ac16fcb26cd67139 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:20:18 +0100 Subject: [PATCH 20/40] added check if index_frame_start exist --- pype/plugins/global/publish/integrate_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index fc7cbf4afa..6d85e29732 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -298,7 +298,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): index_frame_start = int(repre.get("frameStart")) # exception for slate workflow - if "slate" in instance.data["families"]: + if index_frame_start and "slate" in instance.data["families"]: index_frame_start -= 1 dst_padding_exp = src_padding_exp From fde457d445c18d2f87591017df23e3915b8e55b4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:21:27 +0100 Subject: [PATCH 21/40] intent added to version data --- pype/plugins/global/publish/integrate_new.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 6d85e29732..5dba744346 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -601,6 +601,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "fps": context.data.get( "fps", instance.data.get("fps"))} + intent = context.data.get("intent") + if intent is not None: + version_data["intent"] = intent + # Include optional data if present in optionals = [ "frameStart", "frameEnd", "step", "handles", From df512a5a4a17d9d8b14ceba2bc60a908eccbfe5e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:21:52 +0100 Subject: [PATCH 22/40] formatting changes --- pype/plugins/global/publish/integrate_new.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 5dba744346..1ff1dfe520 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -331,7 +331,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if not dst_start_frame: dst_start_frame = dst_padding - dst = "{0}{1}{2}".format( dst_head, dst_start_frame, @@ -503,14 +502,14 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): filelink.create(src, dst, filelink.HARDLINK) def get_subset(self, asset, instance): + subset_name = instance.data["subset"] subset = io.find_one({ "type": "subset", "parent": asset["_id"], - "name": instance.data["subset"] + "name": subset_name }) if subset is None: - subset_name = instance.data["subset"] self.log.info("Subset '%s' not found, creating.." % subset_name) self.log.debug("families. %s" % instance.data.get('families')) self.log.debug( From 6bd8706579b5b1c19ae0ea0c3859e53fdda02013 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:22:06 +0100 Subject: [PATCH 23/40] added few TODOs --- pype/plugins/global/publish/integrate_new.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 1ff1dfe520..15165f4217 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -203,6 +203,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if existing_version is None: version_id = io.insert_one(version).inserted_id else: + # TODO query by _id and + # remove old version and representations but keep their ids io.update_many({ 'type': 'version', 'parent': subset["_id"], @@ -304,6 +306,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): dst_padding_exp = src_padding_exp dst_start_frame = None for i in src_collection.indexes: + # TODO 1.) do not count padding in each index iteration + # 2.) do not count dst_padding from src_padding before + # index_frame_start check src_padding = src_padding_exp % i src_file_name = "{0}{1}{2}".format( From 178fed2ae22893670dcfff056c13f44ed64c925b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:32:12 +0100 Subject: [PATCH 24/40] updated instance input/output docsting --- pype/plugins/global/publish/collect_instance_anatomy_data.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index 838fb1a113..404480b30b 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -8,12 +8,17 @@ Requires: instance -> family Optional: + instance -> version instance -> resolutionWidth instance -> resolutionHeight instance -> fps Provides: + instance -> projectEntity + instance -> assetEntity instance -> anatomyData + instance -> version + instance -> latestVersion """ import copy From ed8b56b6de17330054b2c9469ea63133a1ed5a36 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 15:32:54 +0100 Subject: [PATCH 25/40] fixed per key instance.data value assignment --- .../global/publish/collect_instance_anatomy_data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index 404480b30b..8a98b6cbb2 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -54,9 +54,6 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): "parent": project_entity["_id"] }) - instance.context.data["assetEntity"] = asset_entity - instance.context.data["projectEntity"] = project_entity - subset_name = instance.data["subset"] version_number = instance.data.get("version") latest_version = None @@ -68,7 +65,6 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): "parent": asset_entity["_id"] }) - if subset_entity is None: self.log.debug("Subset entity does not exist yet.") else: @@ -84,7 +80,7 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): # If version is not specified for instance or context if version_number is None: - # TODO we should be able to change this version by studio + # TODO we should be able to change default version by studio # preferences (like start with version number `0`) version_number = 1 # use latest version (+1) if already any exist @@ -111,8 +107,12 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): if resolution_height: anatomy_data["fps"] = fps + instance.data["projectEntity"] = project_entity + instance.data["assetEntity"] = asset_entity instance.data["anatomyData"] = anatomy_data instance.data["latestVersion"] = latest_version + # TODO should be version number set here? + instance.data["version"] = version_number self.log.info("Instance anatomy Data collected") self.log.debug(json.dumps(anatomy_data, indent=4)) From 876ff064b6b6c1a941888e43758196525b49872c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 16:11:29 +0100 Subject: [PATCH 26/40] reduced collect resources path because of already collected data in instance anatomy data --- .../global/publish/collect_resources_path.py | 62 +------------------ 1 file changed, 3 insertions(+), 59 deletions(-) diff --git a/pype/plugins/global/publish/collect_resources_path.py b/pype/plugins/global/publish/collect_resources_path.py index 52e926e09c..de78874cd6 100644 --- a/pype/plugins/global/publish/collect_resources_path.py +++ b/pype/plugins/global/publish/collect_resources_path.py @@ -13,70 +13,14 @@ class IntegrateResourcesPath(pyblish.api.InstancePlugin): families = ["clip", "projectfile", "plate"] def process(self, instance): - project_entity = instance.context["projectEntity"] - asset_entity = instance.context["assetEntity"] - template_data = copy.deepcopy(instance.data["anatomyData"]) - asset_name = instance.data["asset"] - if asset_name != asset_entity["name"]: - asset_entity = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project_entity["_id"] - }) - assert asset_entity, ( - "No asset found by the name '{}' in project '{}'".format( - asset_name, project_entity["name"] - ) - ) - - instance.data["assetEntity"] = asset_entity - - template_data["name"] = asset_entity["name"] - silo_name = asset_entity.get("silo") - if silo_name: - template_data["silo"] = silo_name - - parents = asset_entity["data"].get("parents") or [] - hierarchy = "/".join(parents) - template_data["hierarchy"] = hierarchy - - subset_name = instance.data["subset"] - self.log.info(subset_name) - - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset_entity["_id"] - }) - - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - - # if there is a subset there ought to be version - if version is not None: - version_number += version["name"] - - if instance.data.get('version'): - version_number = int(instance.data.get('version')) - anatomy = instance.context.data["anatomy"] - padding = int(anatomy.templates['render']['padding']) + padding = int(anatomy.templates["render"]["padding"]) + # add possible representation specific key to anatomy data template_data.update({ - "subset": subset_name, - "frame": ('#' * padding), - "version": version_number, + "frame": ("#" * padding), "representation": "TEMP" }) From 3fdfcec29bf6b62023fe34a8b1d1b01fe2198edf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 16:17:18 +0100 Subject: [PATCH 27/40] version_number is used from instance.data in integrate_new --- pype/plugins/global/publish/integrate_new.py | 32 +++----------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 15165f4217..aff92ea308 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -156,40 +156,18 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): subset = self.get_subset(asset_entity, instance) - # TODO iLLiCiT use "latestVersion" from `instance.data` - # and store version in anatomyData instance collector - # instead of query again - instance_version = instance.data.get('version') - if instance_version is not None: - next_version = int(instance_version) - - else: - # get next version - latest_version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - {"name": True}, - sort=[("name", -1)] - ) - - next_version = 1 - if latest_version is not None: - next_version += int(latest_version["name"]) - - self.log.debug("Next version: v{0:03d}".format(next_version)) + version_number = instance.data["version"] + self.log.debug("Next version: v{0:03d}".format(version_number)) version_data = self.create_version_data(context, instance) version_data_instance = instance.data.get('versionData') - if version_data_instance: version_data.update(version_data_instance) version = self.create_version( subset=subset, - version_number=next_version, + version_number=version_number, locations=[avalon_location], data=version_data ) @@ -198,7 +176,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): existing_version = io.find_one({ 'type': 'version', 'parent': subset["_id"], - 'name': next_version + 'name': version_number }) if existing_version is None: version_id = io.insert_one(version).inserted_id @@ -208,7 +186,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): io.update_many({ 'type': 'version', 'parent': subset["_id"], - 'name': next_version + 'name': version_number }, {'$set': version} ) version_id = existing_version['_id'] From ebdc7c3700f17f636573fa45e4ad500f261200f9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 16:23:03 +0100 Subject: [PATCH 28/40] added few todos --- pype/plugins/global/publish/collect_resources_path.py | 5 +++-- pype/plugins/global/publish/integrate_new.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/collect_resources_path.py b/pype/plugins/global/publish/collect_resources_path.py index de78874cd6..fe152584b6 100644 --- a/pype/plugins/global/publish/collect_resources_path.py +++ b/pype/plugins/global/publish/collect_resources_path.py @@ -16,11 +16,12 @@ class IntegrateResourcesPath(pyblish.api.InstancePlugin): template_data = copy.deepcopy(instance.data["anatomyData"]) anatomy = instance.context.data["anatomy"] - padding = int(anatomy.templates["render"]["padding"]) + frame_padding = int(anatomy.templates["render"]["padding"]) # add possible representation specific key to anatomy data + # TODO ability to set host specific "frame" value template_data.update({ - "frame": ("#" * padding), + "frame": ("#" * frame_padding), "representation": "TEMP" }) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index aff92ea308..570a093cdc 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -102,7 +102,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): def register(self, instance): # Required environment variables anatomy_data = instance.data["anatomyData"] - avalon_location = api.Session["AVALON_LOCATION"] io.install() @@ -165,6 +164,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if version_data_instance: version_data.update(version_data_instance) + # TODO remove avalon_location (shall we?) + avalon_location = api.Session["AVALON_LOCATION"] + # TODO rename method from `create_version` to + # `prepare_version` or similar... version = self.create_version( subset=subset, version_number=version_number, From 66466bc24bf3ba98f1a715dfbd0fe9352ba6a65c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 18:07:17 +0100 Subject: [PATCH 29/40] collect resources path uses anatomy publish.folder key with ability of backwards compatibility --- .../global/publish/collect_resources_path.py | 95 ++++++++----------- 1 file changed, 39 insertions(+), 56 deletions(-) diff --git a/pype/plugins/global/publish/collect_resources_path.py b/pype/plugins/global/publish/collect_resources_path.py index fe152584b6..9fc8c576f5 100644 --- a/pype/plugins/global/publish/collect_resources_path.py +++ b/pype/plugins/global/publish/collect_resources_path.py @@ -1,77 +1,60 @@ +""" +Requires: + context -> anatomy + context -> anatomyData + +Provides: + instance -> publishDir + instance -> resourcesDir +""" + import os import copy import pyblish.api -from avalon import io +from avalon import api -class IntegrateResourcesPath(pyblish.api.InstancePlugin): - """Generate the assumed destination path where the file will be stored""" +class CollectResourcesPath(pyblish.api.InstancePlugin): + """Generate directory path where the files and resources will be stored""" - label = "Integrate Prepare Resource" - order = pyblish.api.IntegratorOrder - 0.05 - families = ["clip", "projectfile", "plate"] + label = "Collect Resources Path" + order = pyblish.api.CollectorOrder + 0.995 def process(self, instance): + anatomy = instance.context.data["anatomy"] + template_data = copy.deepcopy(instance.data["anatomyData"]) - anatomy = instance.context.data["anatomy"] - frame_padding = int(anatomy.templates["render"]["padding"]) - - # add possible representation specific key to anatomy data - # TODO ability to set host specific "frame" value + # This is for cases of Deprecated anatomy without `folder` + # TODO remove when all clients have solved this issue template_data.update({ - "frame": ("#" * frame_padding), + "frame": "FRAME_TEMP", "representation": "TEMP" }) anatomy_filled = anatomy.format(template_data) - template_names = ["publish"] - for repre in instance.data["representations"]: - template_name = repre.get("anatomy_template") - if template_name and template_name not in template_names: - template_names.append(template_name) + if "folder" in anatomy.templates["publish"]: + publish_folder = anatomy_filled["publish"]["folder"] + else: + # solve deprecated situation when `folder` key is not underneath + # `publish` anatomy + project_name = api.Session["AVALON_PROJECT"] + self.log.warning(( + "Deprecation warning: Anatomy does not have set `folder`" + " key underneath `publish` (in global of for project `{}`)." + ).format(project_name)) - resources = instance.data.get("resources", list()) - transfers = instance.data.get("transfers", list()) + file_path = anatomy_filled["publish"]["path"] + # Directory + publish_folder = os.path.dirname(file_path) - for template_name in template_names: - mock_template = anatomy_filled[template_name]["path"] + publish_folder = os.path.normpath(publish_folder) + resources_folder = os.path.join(publish_folder, "resources") - # For now assume resources end up in a "resources" folder in the - # published folder - mock_destination = os.path.join( - os.path.dirname(mock_template), "resources" - ) + instance.data["publishDir"] = publish_folder + instance.data["resourcesDir"] = resources_folder - # Clean the path - mock_destination = os.path.abspath( - os.path.normpath(mock_destination) - ).replace("\\", "/") - - # Define resource destination and transfers - for resource in resources: - # Add destination to the resource - source_filename = os.path.basename( - resource["source"]).replace("\\", "/") - destination = os.path.join(mock_destination, source_filename) - - # Force forward slashes to fix issue with software unable - # to work correctly with backslashes in specific scenarios - # (e.g. escape characters in PLN-151 V-Ray UDIM) - destination = destination.replace("\\", "/") - - resource['destination'] = destination - - # Collect transfers for the individual files of the resource - # e.g. all individual files of a cache or UDIM textures. - files = resource['files'] - for fsrc in files: - fname = os.path.basename(fsrc) - fdest = os.path.join( - mock_destination, fname).replace("\\", "/") - transfers.append([fsrc, fdest]) - - instance.data["resources"] = resources - instance.data["transfers"] = transfers + self.log.debug("publishDir: \"{}\"".format(publish_folder)) + self.log.debug("resourcesDir: \"{}\"".format(resources_folder)) From 6f26d0160ce62817843d935b10ba2a937e715a38 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 18:11:32 +0100 Subject: [PATCH 30/40] integrated assued destination was moved back with name integrate resources path --- .../publish/integrate_resources_path.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 pype/plugins/global/publish/integrate_resources_path.py diff --git a/pype/plugins/global/publish/integrate_resources_path.py b/pype/plugins/global/publish/integrate_resources_path.py new file mode 100644 index 0000000000..56dc0e5ef7 --- /dev/null +++ b/pype/plugins/global/publish/integrate_resources_path.py @@ -0,0 +1,49 @@ +import os +import pyblish.api + + +class IntegrateResourcesPath(pyblish.api.InstancePlugin): + """Generate directory path where the files and resources will be stored""" + + label = "Integrate Resources Path" + order = pyblish.api.IntegratorOrder - 0.05 + families = ["clip", "projectfile", "plate"] + + def process(self, instance): + resources = instance.data.get("resources") or [] + transfers = instance.data.get("transfers") or [] + + if not resources and not transfers: + self.log.debug( + "Instance does not have `resources` and `transfers`" + ) + return + + resources_folder = instance.data["resourcesDir"] + + # Define resource destination and transfers + for resource in resources: + # Add destination to the resource + source_filename = os.path.basename( + resource["source"]).replace("\\", "/") + destination = os.path.join(resources_folder, source_filename) + + # Force forward slashes to fix issue with software unable + # to work correctly with backslashes in specific scenarios + # (e.g. escape characters in PLN-151 V-Ray UDIM) + destination = destination.replace("\\", "/") + + resource['destination'] = destination + + # Collect transfers for the individual files of the resource + # e.g. all individual files of a cache or UDIM textures. + files = resource['files'] + for fsrc in files: + fname = os.path.basename(fsrc) + fdest = os.path.join( + resources_folder, fname + ).replace("\\", "/") + transfers.append([fsrc, fdest]) + + instance.data["resources"] = resources + instance.data["transfers"] = transfers From fcffa08177efd97ffc08bbf3520eb2be6a8d02f6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 18:27:03 +0100 Subject: [PATCH 31/40] extract look uses `resourcesDir` instead of computing itself --- pype/plugins/maya/publish/extract_look.py | 63 +---------------------- 1 file changed, 2 insertions(+), 61 deletions(-) diff --git a/pype/plugins/maya/publish/extract_look.py b/pype/plugins/maya/publish/extract_look.py index 4000011520..58196433aa 100644 --- a/pype/plugins/maya/publish/extract_look.py +++ b/pype/plugins/maya/publish/extract_look.py @@ -331,10 +331,9 @@ class ExtractLook(pype.api.Extractor): maya_path)) def resource_destination(self, instance, filepath, do_maketx): - anatomy = instance.context.data["anatomy"] - destination_dir = self.create_destination_template(instance, anatomy) + resources_dir = instance.data["resourcesDir"] # Compute destination location basename, ext = os.path.splitext(os.path.basename(filepath)) @@ -344,7 +343,7 @@ class ExtractLook(pype.api.Extractor): ext = ".tx" return os.path.join( - destination_dir, "resources", basename + ext + resources_dir, basename + ext ) def _process_texture(self, filepath, do_maketx, staging, linearise, force): @@ -408,61 +407,3 @@ class ExtractLook(pype.api.Extractor): return converted, COPY, texture_hash return filepath, COPY, texture_hash - - def create_destination_template(self, instance, anatomy): - """Create a filepath based on the current data available - - Example template: - {root}/{project}/{silo}/{asset}/publish/{subset}/v{version:0>3}/ - {subset}.{representation} - Args: - instance: the instance to publish - - Returns: - file path (str) - """ - - asset_entity = instance.context["assetEntity"] - - template_data = copy.deepcopy(instance.data["anatomyData"]) - - subset_name = instance.data["subset"] - self.log.info(subset_name) - - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset_entity["_id"] - }) - - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - - # if there is a subset there ought to be version - if version is not None: - version_number += version["name"] - - if instance.data.get('version'): - version_number = int(instance.data.get('version')) - - anatomy = instance.context.data["anatomy"] - padding = int(anatomy.templates['render']['padding']) - - template_data.update({ - "subset": subset_name, - "frame": ("#" * padding), - "version": version_number, - "representation": "TEMP" - }) - anatomy_filled = anatomy.format(template_data) - - return os.path.dirname(anatomy_filled["publish"]["path"]) From e92537d34a9c63b7cf09f1b4a46f11c30d76e90d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 18:27:34 +0100 Subject: [PATCH 32/40] extract effects can compute resources dir with anatomyData (need changes) --- .../nukestudio/publish/extract_effects.py | 182 +++++++++--------- 1 file changed, 96 insertions(+), 86 deletions(-) diff --git a/pype/plugins/nukestudio/publish/extract_effects.py b/pype/plugins/nukestudio/publish/extract_effects.py index 15d2a80a55..9e43bee1c8 100644 --- a/pype/plugins/nukestudio/publish/extract_effects.py +++ b/pype/plugins/nukestudio/publish/extract_effects.py @@ -2,10 +2,12 @@ import os import json import re +import copy import pyblish.api import tempfile from avalon import io, api + class ExtractVideoTracksLuts(pyblish.api.InstancePlugin): """Collect video tracks effects into context.""" @@ -71,9 +73,11 @@ class ExtractVideoTracksLuts(pyblish.api.InstancePlugin): ) data["source"] = data["sourcePath"] + # WARNING instance should not be created in Extractor! # create new instance instance = instance.context.create_instance(**data) - + # TODO replace line below with `instance.data["resourcesDir"]` + # when instance is created during collection part dst_dir = self.resource_destination_dir(instance) # change paths in effects to files @@ -141,103 +145,109 @@ class ExtractVideoTracksLuts(pyblish.api.InstancePlugin): return (v, dst) def resource_destination_dir(self, instance): - anatomy = instance.context.data['anatomy'] - self.create_destination_template(instance, anatomy) + # WARNING this is from `collect_instance_anatomy_data.py` + anatomy_data = copy.deepcopy(instance.context.data["anatomyData"]) + project_entity = instance.context.data["projectEntity"] + context_asset_entity = instance.context.data["assetEntity"] - return os.path.join( - instance.data["assumedDestination"], - "resources" - ) - - def create_destination_template(self, instance, anatomy): - """Create a filepath based on the current data available - - Example template: - {root}/{project}/{silo}/{asset}/publish/{subset}/v{version:0>3}/ - {subset}.{representation} - Args: - instance: the instance to publish - - Returns: - file path (str) - """ - - # get all the stuff from the database - subset_name = instance.data["subset"] - self.log.info(subset_name) asset_name = instance.data["asset"] - project_name = api.Session["AVALON_PROJECT"] - a_template = anatomy.templates + if context_asset_entity["name"] == asset_name: + asset_entity = context_asset_entity - project = io.find_one( - { - "type": "project", - "name": project_name - }, - projection={"config": True, "data": True} - ) + else: + asset_entity = io.find_one({ + "type": "asset", + "name": asset_name, + "parent": project_entity["_id"] + }) - template = a_template['publish']['path'] - # anatomy = instance.context.data['anatomy'] + subset_name = instance.data["subset"] + version_number = instance.data.get("version") + latest_version = None - asset = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project["_id"] + if asset_entity: + subset_entity = io.find_one({ + "type": "subset", + "name": subset_name, + "parent": asset_entity["_id"] + }) + + if subset_entity is None: + self.log.debug("Subset entity does not exist yet.") + else: + version_entity = io.find_one( + { + "type": "version", + "parent": subset_entity["_id"] + }, + sort=[("name", -1)] + ) + if version_entity: + latest_version = version_entity["name"] + + if version_number is None: + version_number = 1 + if latest_version is not None: + version_number += int(latest_version) + + anatomy_data.update({ + "asset": asset_name, + "family": instance.data["family"], + "subset": subset_name, + "version": version_number }) - assert asset, ("No asset found by the name '{}' " - "in project '{}'".format(asset_name, project_name)) - silo = asset.get('silo') + resolution_width = instance.data.get("resolutionWidth") + if resolution_width: + anatomy_data["resolution_width"] = resolution_width - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset["_id"] + resolution_height = instance.data.get("resolutionHeight") + if resolution_height: + anatomy_data["resolution_height"] = resolution_height + + fps = instance.data.get("fps") + if resolution_height: + anatomy_data["fps"] = fps + + instance.data["projectEntity"] = project_entity + instance.data["assetEntity"] = asset_entity + instance.data["anatomyData"] = anatomy_data + instance.data["latestVersion"] = latest_version + instance.data["version"] = version_number + + # WARNING this is from `collect_resources_path.py` + anatomy = instance.context.data["anatomy"] + + template_data = copy.deepcopy(instance.data["anatomyData"]) + + # This is for cases of Deprecated anatomy without `folder` + # TODO remove when all clients have solved this issue + template_data.update({ + "frame": "FRAME_TEMP", + "representation": "TEMP" }) - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) + anatomy_filled = anatomy.format(template_data) - # if there is a subset there ought to be version - if version is not None: - version_number += version["name"] + if "folder" in anatomy.templates["publish"]: + publish_folder = anatomy_filled["publish"]["folder"] + else: + # solve deprecated situation when `folder` key is not underneath + # `publish` anatomy + project_name = api.Session["AVALON_PROJECT"] + self.log.warning(( + "Deprecation warning: Anatomy does not have set `folder`" + " key underneath `publish` (in global of for project `{}`)." + ).format(project_name)) - if instance.data.get('version'): - version_number = int(instance.data.get('version')) + file_path = anatomy_filled["publish"]["path"] + # Directory + publish_folder = os.path.dirname(file_path) - padding = int(a_template['render']['padding']) + publish_folder = os.path.normpath(publish_folder) + resources_folder = os.path.join(publish_folder, "resources") - hierarchy = asset['data']['parents'] - if hierarchy: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = "/".join(hierarchy) + instance.data["publishDir"] = publish_folder + instance.data["resourcesDir"] = resources_folder - template_data = {"root": api.Session["AVALON_PROJECTS"], - "project": {"name": project_name, - "code": project['data']['code']}, - "silo": silo, - "family": instance.data['family'], - "asset": asset_name, - "subset": subset_name, - "frame": ('#' * padding), - "version": version_number, - "hierarchy": hierarchy, - "representation": "TEMP"} - - instance.data["assumedTemplateData"] = template_data - self.log.info(template_data) - instance.data["template"] = template - # We take the parent folder of representation 'filepath' - instance.data["assumedDestination"] = os.path.dirname( - anatomy.format(template_data)["publish"]["path"] - ) + return resources_folder From f6e6220869a53f7411b55e03468761a9f5f7c323 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 4 Feb 2020 18:54:58 +0100 Subject: [PATCH 33/40] extract yeti rig uses resourcesDir --- pype/plugins/maya/publish/extract_yeti_rig.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pype/plugins/maya/publish/extract_yeti_rig.py b/pype/plugins/maya/publish/extract_yeti_rig.py index d390a1365a..70a509564f 100644 --- a/pype/plugins/maya/publish/extract_yeti_rig.py +++ b/pype/plugins/maya/publish/extract_yeti_rig.py @@ -1,7 +1,6 @@ import os import json import contextlib -import copy from maya import cmds @@ -111,16 +110,7 @@ class ExtractYetiRig(pype.api.Extractor): self.log.info("Writing metadata file") - # Create assumed destination folder for imageSearchPath - template_data = copy.deepcopy(instance.data["anatomyData"]) - - anatomy = instance.context["anatomy"] - filled = anatomy.format(template_data) - - destination_folder = os.path.dir(filled["publish"]["path"]) - - image_search_path = os.path.join(destination_folder, "resources") - image_search_path = os.path.normpath(image_search_path) + image_search_path = resources_dir = instance.data["resourcesDir"] settings = instance.data.get("rigsettings", None) if settings: From f6ae5b2213b6ee21f2e27f8a2a347a669259cc12 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 5 Feb 2020 09:59:25 +0100 Subject: [PATCH 34/40] task name is also checked --- .../global/publish/collect_instance_anatomy_data.py | 12 +++++++++--- pype/plugins/global/publish/integrate_new.py | 4 ++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/collect_instance_anatomy_data.py b/pype/plugins/global/publish/collect_instance_anatomy_data.py index 8a98b6cbb2..9c6a8b08f2 100644 --- a/pype/plugins/global/publish/collect_instance_anatomy_data.py +++ b/pype/plugins/global/publish/collect_instance_anatomy_data.py @@ -87,13 +87,19 @@ class CollectInstanceAnatomyData(pyblish.api.InstancePlugin): if latest_version is not None: version_number += int(latest_version) - # Version should not be collected since may be instance - anatomy_data.update({ + anatomy_updates = { "asset": asset_name, "family": instance.data["family"], "subset": subset_name, "version": version_number - }) + } + + task_name = instance.data.get("task") + if task_name: + anatomy_updates["task"] = task_name + + # Version should not be collected since may be instance + anatomy_data.update(anatomy_updates) resolution_width = instance.data.get("resolutionWidth") if resolution_width: diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 570a093cdc..d27582bb71 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -132,6 +132,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): hierarchy = "/".join(parents) anatomy_data["hierarchy"] = hierarchy + task_name = instance.data.get("task") + if task_name: + anatomy_data["task"] = task_name + stagingdir = instance.data.get("stagingDir") if not stagingdir: self.log.info(( From 4de7478d9dcf176eb349a52a20eddac76e0424e8 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 14 Feb 2020 17:19:13 +0100 Subject: [PATCH 35/40] again remove collect templates --- .../global/publish/collect_templates.py | 119 ------------------ 1 file changed, 119 deletions(-) delete mode 100644 pype/plugins/global/publish/collect_templates.py diff --git a/pype/plugins/global/publish/collect_templates.py b/pype/plugins/global/publish/collect_templates.py deleted file mode 100644 index 3104b5b705..0000000000 --- a/pype/plugins/global/publish/collect_templates.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -Requires: - session -> AVALON_PROJECT - context -> anatomy (pypeapp.Anatomy) - instance -> subset - instance -> asset - instance -> family - -Provides: - instance -> template - instance -> assumedTemplateData - instance -> assumedDestination -""" - -import os - -from avalon import io, api -import pyblish.api - - -class CollectTemplates(pyblish.api.InstancePlugin): - """Fill templates with data needed for publish""" - - order = pyblish.api.CollectorOrder + 0.1 - label = "Collect and fill Templates" - hosts = ["maya", "nuke", "standalonepublisher"] - - def process(self, instance): - # get all the stuff from the database - subset_name = instance.data["subset"] - asset_name = instance.data["asset"] - project_name = api.Session["AVALON_PROJECT"] - - project = io.find_one( - { - "type": "project", - "name": project_name - }, - projection={"config": True, "data": True} - ) - - template = project["config"]["template"]["publish"] - anatomy = instance.context.data['anatomy'] - - asset = io.find_one({ - "type": "asset", - "name": asset_name, - "parent": project["_id"] - }) - - assert asset, ("No asset found by the name '{}' " - "in project '{}'".format(asset_name, project_name)) - silo = asset.get('silo') - - subset = io.find_one({ - "type": "subset", - "name": subset_name, - "parent": asset["_id"] - }) - - # assume there is no version yet, we start at `1` - version = None - version_number = 1 - if subset is not None: - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - - # if there is a subset there ought to be version - if version is not None: - version_number += int(version["name"]) - - hierarchy = asset['data']['parents'] - if hierarchy: - # hierarchy = os.path.sep.join(hierarchy) - hierarchy = os.path.join(*hierarchy) - else: - hierarchy = "" - - template_data = {"root": api.Session["AVALON_PROJECTS"], - "project": {"name": project_name, - "code": project['data']['code']}, - "silo": silo, - "family": instance.data['family'], - "asset": asset_name, - "subset": subset_name, - "version": version_number, - "hierarchy": hierarchy.replace("\\", "/"), - "representation": "TEMP"} - - # Add datetime data to template data - datetime_data = instance.context.data.get("datetimeData") or {} - template_data.update(datetime_data) - - resolution_width = instance.data.get("resolutionWidth") - resolution_height = instance.data.get("resolutionHeight") - fps = instance.data.get("fps") - - if resolution_width: - template_data["resolution_width"] = resolution_width - if resolution_width: - template_data["resolution_height"] = resolution_height - if resolution_width: - template_data["fps"] = fps - - instance.data["template"] = template - instance.data["assumedTemplateData"] = template_data - - # We take the parent folder of representation 'filepath' - instance.data["assumedDestination"] = os.path.dirname( - (anatomy.format(template_data))["publish"]["path"] - ) - self.log.info("Assumed Destination has been created...") - self.log.debug("__ assumedTemplateData: `{}`".format(instance.data["assumedTemplateData"])) - self.log.debug("__ template: `{}`".format(instance.data["template"])) From 0618b7a85ff9767ac6f5d4eaf3f58bd72f2b433c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 14 Feb 2020 17:20:04 +0100 Subject: [PATCH 36/40] fix order --- pype/plugins/global/publish/collect_resources_path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/collect_resources_path.py b/pype/plugins/global/publish/collect_resources_path.py index 9fc8c576f5..734d1f84e4 100644 --- a/pype/plugins/global/publish/collect_resources_path.py +++ b/pype/plugins/global/publish/collect_resources_path.py @@ -19,7 +19,7 @@ class CollectResourcesPath(pyblish.api.InstancePlugin): """Generate directory path where the files and resources will be stored""" label = "Collect Resources Path" - order = pyblish.api.CollectorOrder + 0.995 + order = pyblish.api.CollectorOrder + 0.495 def process(self, instance): anatomy = instance.context.data["anatomy"] From a7ca458e4ee1550859fee03f84592aea9615947e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 14 Feb 2020 17:24:56 +0100 Subject: [PATCH 37/40] collect scene has publish set to True by default --- pype/plugins/maya/publish/collect_scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/maya/publish/collect_scene.py b/pype/plugins/maya/publish/collect_scene.py index f2fbb4d623..089019f2d3 100644 --- a/pype/plugins/maya/publish/collect_scene.py +++ b/pype/plugins/maya/publish/collect_scene.py @@ -35,7 +35,7 @@ class CollectMayaScene(pyblish.api.ContextPlugin): "subset": subset, "asset": os.getenv("AVALON_ASSET", None), "label": subset, - "publish": False, + "publish": True, "family": 'workfile', "families": ['workfile'], "setMembers": [current_file] From 377513f01f77c49d656f152157a1245e63e3bab6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 14 Feb 2020 17:25:25 +0100 Subject: [PATCH 38/40] removed locations from version --- pype/plugins/global/publish/integrate_new.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index d27582bb71..8735f8fed7 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -168,14 +168,11 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if version_data_instance: version_data.update(version_data_instance) - # TODO remove avalon_location (shall we?) - avalon_location = api.Session["AVALON_LOCATION"] # TODO rename method from `create_version` to # `prepare_version` or similar... version = self.create_version( subset=subset, version_number=version_number, - locations=[avalon_location], data=version_data ) @@ -528,26 +525,21 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return subset - def create_version(self, subset, version_number, locations, data=None): + def create_version(self, subset, version_number, data=None): """ Copy given source to destination Args: subset (dict): the registered subset of the asset version_number (int): the version number - locations (list): the currently registered locations Returns: dict: collection of data to create a version """ - # Imprint currently registered location - version_locations = [location for location in locations if - location is not None] return {"schema": "pype:version-3.0", "type": "version", "parent": subset["_id"], "name": version_number, - "locations": version_locations, "data": data} def create_version_data(self, context, instance): From 5f5a80818c20e26deeded4f616d477a479999ee8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Sun, 16 Feb 2020 20:07:42 +0100 Subject: [PATCH 39/40] fix(global): fixing version collection --- pype/plugins/nuke/publish/collect_writes.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index bf1c6a4b66..c29f676ef7 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -52,9 +52,9 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): output_dir = os.path.dirname(path) self.log.debug('output dir: {}'.format(output_dir)) - # get version to instance for integration - instance.data['version'] = instance.context.data.get( - "version", pype.get_version_from_path(nuke.root().name())) + # # get version to instance for integration + # instance.data['version'] = instance.context.data.get( + # "version", pype.get_version_from_path(nuke.root().name())) self.log.debug('Write Version: %s' % instance.data('version')) @@ -92,16 +92,7 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): # Add version data to instance version_data = { - "handles": handle_start, - "handleStart": handle_start, - "handleEnd": handle_end, - "frameStart": first_frame + handle_start, - "frameEnd": last_frame - handle_end, - "version": int(instance.data['version']), "colorspace": node["colorspace"].value(), - "families": ["render"], - "subset": instance.data["subset"], - "fps": instance.context.data["fps"] } instance.data["family"] = "write" From e8499b43ff4cf6a0b9a15b502fbf164474ca0e49 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Sun, 16 Feb 2020 20:08:18 +0100 Subject: [PATCH 40/40] fix(global): wrong version format print --- pype/plugins/global/publish/integrate_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index d27582bb71..bb65a02bce 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -160,7 +160,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): subset = self.get_subset(asset_entity, instance) version_number = instance.data["version"] - self.log.debug("Next version: v{0:03d}".format(version_number)) + self.log.debug("Next version: v{}".format(version_number)) version_data = self.create_version_data(context, instance)