From 52e88951f4ced27620c6f34a220bc5ce0c622d17 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 21 Feb 2024 11:53:41 +0100 Subject: [PATCH 01/21] Traypublisher CSV ingest ayon conversion kickof Traypublisher CSV ingest ayon conversion kickoff Added functionality for ingesting CSV files into projects. Includes commands to ingest CSV data, publish the content, and create instances based on the CSV data. --- client/ayon_core/hosts/traypublisher/addon.py | 61 +- .../hosts/traypublisher/csv_publish.py | 76 ++ .../plugins/create/create_csv_ingest.py | 686 ++++++++++++++++++ .../collect_csv_ingest_instance_data.py | 36 + .../plugins/publish/extract_csv_file.py | 31 + .../publish/validate_existing_version.py | 1 + .../plugins/publish/validate_frame_ranges.py | 2 + client/ayon_core/plugins/publish/integrate.py | 3 +- 8 files changed, 894 insertions(+), 2 deletions(-) create mode 100644 client/ayon_core/hosts/traypublisher/csv_publish.py create mode 100644 client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py create mode 100644 client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py create mode 100644 client/ayon_core/hosts/traypublisher/plugins/publish/extract_csv_file.py diff --git a/client/ayon_core/hosts/traypublisher/addon.py b/client/ayon_core/hosts/traypublisher/addon.py index 70bdfe9a64..f3884aedfe 100644 --- a/client/ayon_core/hosts/traypublisher/addon.py +++ b/client/ayon_core/hosts/traypublisher/addon.py @@ -1,6 +1,6 @@ import os -from ayon_core.lib import get_ayon_launcher_args +from pathlib import Path from ayon_core.lib.execute import run_detached_process from ayon_core.addon import ( click_wrap, @@ -57,3 +57,62 @@ def launch(): from ayon_core.tools import traypublisher traypublisher.main() + + +@cli_main.command() +@click_wrap.option( + "--csv-filepath", + help="Full path to CSV file with data", + type=str, + required=True +) +@click_wrap.option( + "--project-name", + help="Project name in which the context will be used", + type=str, + required=True +) +@click_wrap.option( + "--asset-name", + help="Asset name in which the context will be used", + type=str, + required=True +) +@click_wrap.option( + "--task-name", + help="Task name under Asset in which the context will be used", + type=str, + required=False +) +@click_wrap.option( + "--ignore-validators", + help="Option to ignore validators", + type=bool, + is_flag=True, + required=False +) +def ingestcsv( + csv_filepath, + project_name, + asset_name, + task_name, + ignore_validators +): + """Ingest CSV file into project. + + This command will ingest CSV file into project. CSV file must be in + specific format. See documentation for more information. + """ + from .csv_publish import csvpublish + + # use Path to check if csv_filepath exists + if not Path(csv_filepath).exists(): + raise FileNotFoundError(f"File {csv_filepath} does not exist.") + + csvpublish( + csv_filepath, + project_name, + asset_name, + task_name, + ignore_validators + ) diff --git a/client/ayon_core/hosts/traypublisher/csv_publish.py b/client/ayon_core/hosts/traypublisher/csv_publish.py new file mode 100644 index 0000000000..f8eed2f2c5 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/csv_publish.py @@ -0,0 +1,76 @@ +import os + +import pyblish.api +import pyblish.util + +from ayon_core.client import get_asset_by_name +from ayon_core.lib.attribute_definitions import FileDefItem +from ayon_core.pipeline import install_host +from ayon_core.pipeline.create import CreateContext + +from ayon_core.hosts.traypublisher.api import TrayPublisherHost + + +def csvpublish( + csv_filepath, + project_name, + asset_name, + task_name=None, + ignore_validators=False +): + """Publish CSV file. + + Args: + csv_filepath (str): Path to CSV file. + project_name (str): Project name. + asset_name (str): Asset name. + task_name (Optional[str]): Task name. + ignore_validators (Optional[bool]): Option to ignore validators. + """ + + # initialization of host + host = TrayPublisherHost() + install_host(host) + + # setting host context into project + host.set_project_name(project_name) + + # add asset context to environment + # TODO: perhaps this can be done in a better way? + os.environ.update({ + "AVALON_PROJECT": project_name, + "AVALON_ASSET": asset_name, + "AVALON_TASK": task_name or "" + }) + + # form precreate data with field values + file_field = FileDefItem.from_paths([csv_filepath], False).pop().to_dict() + precreate_data = { + "csv_filepath_data": file_field, + } + + # create context initialization + create_context = CreateContext(host, headless=True) + asset_doc = get_asset_by_name( + project_name, + asset_name + ) + + create_context.create( + "io.openpype.creators.traypublisher.csv_ingest", + "Main", + asset_doc=asset_doc, + task_name=task_name, + pre_create_data=precreate_data, + ) + + # publishing context initialization + pyblish_context = pyblish.api.Context() + pyblish_context.data["create_context"] = create_context + + # redefine targets (skip 'local' to disable validators) + if ignore_validators: + targets = ["default", "ingest"] + + # publishing + pyblish.util.publish(context=pyblish_context, targets=targets) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py new file mode 100644 index 0000000000..aa986657dc --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -0,0 +1,686 @@ +import os +import re +import csv +import clique +from copy import deepcopy, copy + +from ayon_core.client import get_asset_by_name +from ayon_core.pipeline.create import get_subset_name +from ayon_core.pipeline import CreatedInstance +from ayon_core.lib import FileDef, BoolDef +from ayon_core.lib.transcoding import ( + VIDEO_EXTENSIONS, IMAGE_EXTENSIONS +) + +from ayon_core.hosts.traypublisher.api.plugin import ( + TrayPublishCreator +) + + +class IngestCSV(TrayPublishCreator): + """CSV ingest creator class""" + + icon = "fa.file" + + label = "CSV Ingest" + family = "csv_ingest_file" + identifier = "io.ayon_core.creators.traypublisher.csv_ingest" + + default_variants = ["Main"] + + description = "Ingest products' data from CSV file" + detailed_description = """ +Ingest products' data from CSV file following column and representation +configuration in project settings. +""" + + # Position in the list of creators. + order = 10 + + # settings for this creator + columns_config = {} + representations_config = {} + + + def create(self, subset_name, instance_data, pre_create_data): + """Create an product from each row found in the CSV. + + Args: + subset_name (str): The subset name. + instance_data (dict): The instance data. + pre_create_data (dict): + """ + + csv_filepath_data = pre_create_data.get("csv_filepath_data", {}) + + folder = csv_filepath_data.get("directory", "") + if not os.path.exists(folder): + raise FileNotFoundError( + f"Directory '{folder}' does not exist." + ) + filename = csv_filepath_data.get("filenames", []) + self._process_csv_file(subset_name, instance_data, folder, filename[0]) + + def _process_csv_file( + self, subset_name, instance_data, staging_dir, filename): + """Process CSV file. + + Args: + subset_name (str): The subset name. + instance_data (dict): The instance data. + staging_dir (str): The staging directory. + filename (str): The filename. + """ + + # create new instance from the csv file via self function + self._pass_data_to_csv_instance( + instance_data, + staging_dir, + filename + ) + + csv_instance = CreatedInstance( + self.family, subset_name, instance_data, self + ) + self._store_new_instance(csv_instance) + + csv_instance["csvFileData"] = { + "filename": filename, + "staging_dir": staging_dir, + } + + # from special function get all data from csv file and convert them + # to new instances + csv_data_for_instances = self._get_data_from_csv( + staging_dir, filename) + + # create instances from csv data via self function + self._create_instances_from_csv_data( + csv_data_for_instances, staging_dir + ) + + def _create_instances_from_csv_data( + self, + csv_data_for_instances, + staging_dir + ): + """Create instances from csv data""" + + for asset_name, _data in csv_data_for_instances.items(): + asset_doc = _data["asset_doc"] + products = _data["products"] + + for instance_name, product_data in products.items(): + # get important instance variables + task_name = product_data["task_name"] + variant = product_data["variant"] + product_type = product_data["product_type"] + version = product_data["version"] + + # create subset/product name + product_name = get_subset_name( + product_type, + variant, + task_name, + asset_doc, + ) + + # make sure frame start/end is inherited from csv columns + # expected frame range data are handles excluded + for _, repre_data in product_data["representations"].items(): # noqa: E501 + frame_start = repre_data["frameStart"] + frame_end = repre_data["frameEnd"] + handle_start = repre_data["handleStart"] + handle_end = repre_data["handleEnd"] + fps = repre_data["fps"] + break + + # try to find any version comment in representation data + version_comment = next( + iter( + repre_data["comment"] + for _, repre_data in product_data["representations"].items() # noqa: E501 + if repre_data["comment"] + ), + None + ) + + # try to find any slate switch in representation data + slate_exists = any( + repre_data["slate"] + for _, repre_data in product_data["representations"].items() # noqa: E501 + ) + + # get representations from product data + representations = product_data["representations"] + label = f"{asset_name}_{product_name}_v{version:>03}" + + families = ["csv_ingest"] + if slate_exists: + # adding slate to families mainly for loaders to be able + # to filter out slates + families.append("slate") + + # make product data + product_data = { + "name": instance_name, + "asset": asset_name, + "families": families, + "label": label, + "task": task_name, + "variant": variant, + "source": "csv", + "frameStart": frame_start, + "frameEnd": frame_end, + "handleStart": handle_start, + "handleEnd": handle_end, + "fps": fps, + "version": version, + "comment": version_comment, + } + + # create new instance + new_instance = CreatedInstance( + product_type, product_name, product_data, self + ) + self._store_new_instance(new_instance) + + if not new_instance.get("prepared_data_for_repres"): + new_instance["prepared_data_for_repres"] = [] + + base_thumbnail_repre_data = { + "name": "thumbnail", + "ext": None, + "files": None, + "stagingDir": None, + "stagingDir_persistent": True, + "tags": ["thumbnail", "delete"], + } + # need to populate all thumbnails for all representations + # so we can check if unique thumbnail per representation + # is needed + thumbnails = [ + repre_data["thumbnailPath"] + for repre_data in representations.values() + if repre_data["thumbnailPath"] + ] + multiple_thumbnails = len(set(thumbnails)) > 1 + explicit_output_name = None + thumbnails_processed = False + for filepath, repre_data in representations.items(): + # check if any review derivate tag is present + reviewable = any( + tag for tag in repre_data.get("tags", []) + # tag can be `ftrackreview` or `review` + if "review" in tag + ) + # since we need to populate multiple thumbnails as + # representation with outputName for (Ftrack instance + # integrator) pairing with reviewable video representations + if ( + thumbnails + and multiple_thumbnails + and reviewable + ): + # multiple unique thumbnails per representation needs + # grouping by outputName + # mainly used in Ftrack instance integrator + explicit_output_name = repre_data["representationName"] + relative_thumbnail_path = repre_data["thumbnailPath"] + # representation might not have thumbnail path + # so ignore this one + if not relative_thumbnail_path: + continue + thumb_dir, thumb_file = \ + self._get_refactor_thumbnail_path( + staging_dir, relative_thumbnail_path) + filename, ext = os.path.splitext(thumb_file) + thumbnail_repr_data = deepcopy( + base_thumbnail_repre_data) + thumbnail_repr_data.update({ + "name": "thumbnail_{}".format(filename), + "ext": ext[1:], + "files": thumb_file, + "stagingDir": thumb_dir, + "outputName": explicit_output_name, + }) + new_instance["prepared_data_for_repres"].append( + ("_thumbnail_", thumbnail_repr_data) + ) + elif ( + thumbnails + and not multiple_thumbnails + and not thumbnails_processed + or not reviewable + ): + if not thumbnails: + continue + # here we will use only one thumbnail for + # all representations + relative_thumbnail_path = repre_data["thumbnailPath"] + if not relative_thumbnail_path: + relative_thumbnail_path = thumbnails.pop() + thumb_dir, thumb_file = \ + self._get_refactor_thumbnail_path( + staging_dir, relative_thumbnail_path) + _, ext = os.path.splitext(thumb_file) + thumbnail_repr_data = deepcopy( + base_thumbnail_repre_data) + thumbnail_repr_data.update({ + "ext": ext[1:], + "files": thumb_file, + "stagingDir": thumb_dir + }) + new_instance["prepared_data_for_repres"].append( + ("_thumbnail_", thumbnail_repr_data) + ) + thumbnails_processed = True + + # get representation data + representation_data = self._get_representation_data( + filepath, repre_data, staging_dir, + explicit_output_name + ) + + new_instance["prepared_data_for_repres"].append( + (repre_data["colorspace"], representation_data) + ) + + def _get_refactor_thumbnail_path( + self, staging_dir, relative_thumbnail_path): + thumbnail_abs_path = os.path.join( + staging_dir, relative_thumbnail_path) + return os.path.split( + thumbnail_abs_path) + + def _get_representation_data( + self, filepath, repre_data, staging_dir, explicit_output_name=None + ): + """Get representation data + + Args: + filepath (str): Filepath to representation file. + repre_data (dict): Representation data from CSV file. + staging_dir (str): Staging directory. + explicit_output_name (Optional[str]): Explicit output name. + For grouping purposes with reviewable components. + Defaults to None. + """ + + # get extension of file + basename = os.path.basename(filepath) + _, extension = os.path.splitext(filepath) + + # validate filepath is having correct extension based on output + config_repre_data = self.representations_config["representations"] + repre_name = repre_data["representationName"] + if repre_name not in config_repre_data: + raise KeyError( + f"Representation '{repre_name}' not found " + "in config representation data." + ) + validate_extensions = config_repre_data[repre_name]["extensions"] + if extension not in validate_extensions: + raise TypeError( + f"File extension '{extension}' not valid for " + f"output '{validate_extensions}'." + ) + + is_sequence = (extension in IMAGE_EXTENSIONS) + # convert ### string in file name to %03d + # this is for correct frame range validation + # example: file.###.exr -> file.%03d.exr + if "#" in basename: + padding = len(basename.split("#")) - 1 + basename = basename.replace("#" * padding, f"%0{padding}d") + is_sequence = True + + # make absolute path to file + absfilepath = os.path.normpath(os.path.join(staging_dir, filepath)) + dirname = os.path.dirname(absfilepath) + + # check if dirname exists + if not os.path.isdir(dirname): + raise NotADirectoryError( + f"Directory '{dirname}' does not exist." + ) + + # collect all data from dirname + paths_for_collection = [] + for file in os.listdir(dirname): + filepath = os.path.join(dirname, file) + paths_for_collection.append(filepath) + + collections, _ = clique.assemble(paths_for_collection) + + if collections: + collections = collections[0] + else: + if is_sequence: + raise ValueError( + f"No collections found in directory '{dirname}'." + ) + + frame_start = None + frame_end = None + if is_sequence: + files = [os.path.basename(file) for file in collections] + frame_start = list(collections.indexes)[0] + frame_end = list(collections.indexes)[-1] + else: + files = basename + + tags = deepcopy(repre_data["tags"]) + # if slate in repre_data is True then remove one frame from start + if repre_data["slate"]: + tags.append("has_slate") + + # get representation data + representation_data = { + "name": repre_name, + "ext": extension[1:], + "files": files, + "stagingDir": dirname, + "stagingDir_persistent": True, + "tags": tags, + } + if extension in VIDEO_EXTENSIONS: + representation_data.update({ + "fps": repre_data["fps"], + "outputName": repre_name, + }) + + if explicit_output_name: + representation_data["outputName"] = explicit_output_name + + if frame_start: + representation_data["frameStart"] = frame_start + if frame_end: + representation_data["frameEnd"] = frame_end + + return representation_data + + def _get_data_from_csv( + self, package_dir, filename + ): + """Generate instances from the csv file""" + # get current project name and code from context.data + project_name = self.create_context.get_current_project_name() + + csv_file_path = os.path.join( + package_dir, filename + ) + + # make sure csv file contains columns from following list + required_columns = [ + name for name, value in self.columns_config["columns"].items() + if value["required"] + ] + # get data from csv file + with open(csv_file_path, "r") as csv_file: + csv_reader = csv.DictReader( + csv_file, delimiter=self.columns_config["csv_delimiter"]) + + # fix fieldnames + # sometimes someone can keep extra space at the start or end of + # the column name + all_columns = [ + " ".join(column.rsplit()) for column in csv_reader.fieldnames] + # return back fixed fieldnames + csv_reader.fieldnames = all_columns + + # check if csv file contains all required columns + if any(column not in all_columns for column in required_columns): + raise KeyError( + f"Missing required columns: {required_columns}" + ) + + csv_data = {} + # get data from csv file + for row in csv_reader: + # Get required columns first + context_asset_name = self._get_row_value_with_validation( + "Folder Context", row) + task_name = self._get_row_value_with_validation( + "Task Name", row) + version = self._get_row_value_with_validation( + "Version", row) + + # Get optional columns + variant = self._get_row_value_with_validation( + "Variant", row) + product_type = self._get_row_value_with_validation( + "Product Type", row) + + pre_product_name = ( + f"{task_name}{variant}{product_type}" + f"{version}".replace(" ", "").lower() + ) + + # get representation data + filename, representation_data = \ + self._get_representation_row_data(row) + + # get all csv data into one dict and make sure there are no + # duplicates data are already validated and sorted under + # correct existing asset also check if asset exists and if + # task name is valid task in asset doc and representations + # are distributed under products following variants + if context_asset_name not in csv_data: + asset_doc = get_asset_by_name( + project_name, context_asset_name) + + # make sure asset exists + if not asset_doc: + raise ValueError( + f"Asset '{context_asset_name}' not found." + ) + # check if task name is valid task in asset doc + if task_name not in asset_doc["data"]["tasks"]: + raise ValueError( + f"Task '{task_name}' not found in asset doc." + ) + + csv_data[context_asset_name] = { + "asset_doc": asset_doc, + "products": { + pre_product_name: { + "task_name": task_name, + "variant": variant, + "product_type": product_type, + "version": version, + "representations": { + filename: representation_data, + }, + } + } + } + else: + asset_doc = csv_data[context_asset_name]["asset_doc"] + csv_products = csv_data[context_asset_name]["products"] + if pre_product_name not in csv_products: + csv_products[pre_product_name] = { + "task_name": task_name, + "variant": variant, + "product_type": product_type, + "version": version, + "representations": { + filename: representation_data, + }, + } + else: + csv_representations = \ + csv_products[pre_product_name]["representations"] + if filename in csv_representations: + raise ValueError( + f"Duplicate filename '{filename}' in csv file." + ) + csv_representations[filename] = representation_data + + return csv_data + + def _get_representation_row_data(self, row_data): + """Get representation row data""" + # Get required columns first + file_path = self._get_row_value_with_validation( + "File Path", row_data) + frame_start = self._get_row_value_with_validation( + "Frame Start", row_data) + frame_end = self._get_row_value_with_validation( + "Frame End", row_data) + handle_start = self._get_row_value_with_validation( + "Handle Start", row_data) + handle_end = self._get_row_value_with_validation( + "Handle End", row_data) + fps = self._get_row_value_with_validation( + "FPS", row_data) + + # Get optional columns + thumbnail_path = self._get_row_value_with_validation( + "Thumbnail", row_data) + colorspace = self._get_row_value_with_validation( + "Colorspace", row_data) + comment = self._get_row_value_with_validation( + "Version Comment", row_data) + repre = self._get_row_value_with_validation( + "Representation", row_data) + slate_exists = self._get_row_value_with_validation( + "Slate Exists", row_data) + repre_tags = self._get_row_value_with_validation( + "Representation Tags", row_data) + + # convert tags value to list + tags_list = copy(self.representations_config["default_tags"]) + if repre_tags: + tags_list = [] + tags_delimiter = self.representations_config["tags_delimiter"] + # strip spaces from repre_tags + if tags_delimiter in repre_tags: + tags = repre_tags.split(tags_delimiter) + for _tag in tags: + tags_list.append(("".join(_tag.strip())).lower()) + else: + tags_list.append(repre_tags) + + representation_data = { + "colorspace": colorspace, + "comment": comment, + "representationName": repre, + "slate": slate_exists, + "tags": tags_list, + "thumbnailPath": thumbnail_path, + "frameStart": int(frame_start), + "frameEnd": int(frame_end), + "handleStart": int(handle_start), + "handleEnd": int(handle_end), + "fps": float(fps), + } + + return file_path, representation_data + + def _get_row_value_with_validation( + self, column_name, row_data, default_value=None + ): + """Get row value with validation""" + columns_config = self.columns_config["columns"] + # get column data from column config + column_data = columns_config.get(column_name) + if not column_data: + raise KeyError( + f"Column '{column_name}' not found in column config." + ) + + # get column value from row + column_value = row_data.get(column_name) + column_required = column_data["required"] + + # check if column value is not empty string and column is required + if column_value == "" and column_required: + raise ValueError( + f"Value in column '{column_name}' is required." + ) + + # get column type + column_type = column_data["type"] + # get column validation regex + column_validation = column_data["validate"] + # get column default value + column_default = default_value or column_data["default"] + + if column_type in ["number", "decimal"] and column_default == 0: + column_default = None + + # check if column value is not empty string + if column_value == "": + # set default value if column value is empty string + column_value = column_default + + # set column value to correct type following column type + if column_type == "number" and column_value is not None: + column_value = int(column_value) + elif column_type == "decimal" and column_value is not None: + column_value = float(column_value) + elif column_type == "bool": + column_value = column_value in ["true", "True"] + + # check if column value matches validation regex + if ( + column_value is not None and + not re.match(str(column_validation), str(column_value)) + ): + raise ValueError( + f"Column '{column_name}' value '{column_value}' " + f"does not match validation regex '{column_validation}' \n" + f"Row data: {row_data} \n" + f"Column data: {column_data}" + ) + + return column_value + + def _pass_data_to_csv_instance( + self, instance_data, staging_dir, filename + ): + """Pass CSV representation file to instance data""" + + representation = { + "name": "csv", + "ext": "csv", + "files": filename, + "stagingDir": staging_dir, + "stagingDir_persistent": True, + } + + instance_data.update({ + "label": f"CSV: {filename}", + "representations": [representation], + "stagingDir": staging_dir, + "stagingDir_persistent": True, + }) + + def get_instance_attr_defs(self): + return [ + BoolDef( + "add_review_family", + default=True, + label="Review" + ) + ] + + def get_pre_create_attr_defs(self): + """Creating pre-create attributes at creator plugin. + + Returns: + list: list of attribute object instances + """ + # Use same attributes as for instance attrobites + attr_defs = [ + FileDef( + "csv_filepath_data", + folders=False, + extensions=[".csv"], + allow_sequences=False, + single_item=True, + label="CSV File", + ), + ] + return attr_defs diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py new file mode 100644 index 0000000000..0da3ebed81 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py @@ -0,0 +1,36 @@ +from pprint import pformat +import pyblish.api +from ayon_core.pipeline import publish + + +class CollectCSVIngestInstancesData( + pyblish.api.InstancePlugin, + publish.AYONPyblishPluginMixin, + publish.ColormanagedPyblishPluginMixin +): + """Collect CSV Ingest data from instance. + """ + + label = "Collect CSV Ingest instances data" + order = pyblish.api.CollectorOrder + 0.1 + hosts = ["traypublisher"] + families = ["csv_ingest"] + + def process(self, instance): + self.log.info(f"Collecting {instance.name}") + + # expecting [(colorspace, repre_data), ...] + prepared_repres_data_items = instance.data[ + "prepared_data_for_repres"] + + for colorspace, repre_data in prepared_repres_data_items: + # only apply colorspace to those which are not marked as thumbnail + if colorspace != "_thumbnail_": + # colorspace name is passed from CSV column + self.set_representation_colorspace( + repre_data, instance.context, colorspace + ) + + instance.data["representations"].append(repre_data) + + self.log.debug(pformat(instance.data)) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/extract_csv_file.py b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_csv_file.py new file mode 100644 index 0000000000..4bdf7c0493 --- /dev/null +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/extract_csv_file.py @@ -0,0 +1,31 @@ +import pyblish.api + +from ayon_core.pipeline import publish + + +class ExtractCSVFile(publish.Extractor): + """ + Extractor export CSV file + """ + + label = "Extract CSV file" + order = pyblish.api.ExtractorOrder - 0.45 + families = ["csv_ingest_file"] + hosts = ["traypublisher"] + + def process(self, instance): + + csv_file_data = instance.data["csvFileData"] + + representation_csv = { + 'name': "csv_data", + 'ext': "csv", + 'files': csv_file_data["filename"], + "stagingDir": csv_file_data["staging_dir"], + "stagingDir_persistent": True + } + + instance.data["representations"].append(representation_csv) + + self.log.info("Added CSV file representation: {}".format( + representation_csv)) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_existing_version.py b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_existing_version.py index 6a85f92ce1..7a35a19a85 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_existing_version.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_existing_version.py @@ -16,6 +16,7 @@ class ValidateExistingVersion( order = ValidateContentsOrder hosts = ["traypublisher"] + targets = ["local"] actions = [RepairAction] diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_frame_ranges.py b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_frame_ranges.py index cd4a98b84d..ca53a2c8ef 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/validate_frame_ranges.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/validate_frame_ranges.py @@ -16,6 +16,8 @@ class ValidateFrameRange(OptionalPyblishPluginMixin, label = "Validate Frame Range" hosts = ["traypublisher"] families = ["render", "plate"] + targets = ["local"] + order = ValidateContentsOrder optional = True diff --git a/client/ayon_core/plugins/publish/integrate.py b/client/ayon_core/plugins/publish/integrate.py index a502031595..2788875c23 100644 --- a/client/ayon_core/plugins/publish/integrate.py +++ b/client/ayon_core/plugins/publish/integrate.py @@ -140,7 +140,8 @@ class IntegrateAsset(pyblish.api.InstancePlugin): "uasset", "blendScene", "yeticacheUE", - "tycache" + "tycache", + "csv_ingest_file", ] default_template_name = "publish" From 6274e5259a8b1a0daa31fe96847ead0a6e1cea74 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 22 Feb 2024 17:15:20 +0100 Subject: [PATCH 02/21] Settings models and validators for batch movie creation and CSV ingestion plugins. Update addon version to 0.1.4. --- .../server/settings/creator_plugins.py | 295 ++++++++++++++++++ server_addon/traypublisher/server/version.py | 2 +- 2 files changed, 296 insertions(+), 1 deletion(-) diff --git a/server_addon/traypublisher/server/settings/creator_plugins.py b/server_addon/traypublisher/server/settings/creator_plugins.py index bf66d9a088..7ce241faa6 100644 --- a/server_addon/traypublisher/server/settings/creator_plugins.py +++ b/server_addon/traypublisher/server/settings/creator_plugins.py @@ -1,4 +1,7 @@ +from pydantic import validator from ayon_server.settings import BaseSettingsModel, SettingsField +from ayon_server.settings.validators import ensure_unique_names +from ayon_server.exceptions import BadRequestException class BatchMovieCreatorPlugin(BaseSettingsModel): @@ -22,11 +25,137 @@ class BatchMovieCreatorPlugin(BaseSettingsModel): ) +class ColumnItemModel(BaseSettingsModel): + """Allows to publish multiple video files in one go.
Name of matching + asset is parsed from file names ('asset.mov', 'asset_v001.mov', + 'my_asset_to_publish.mov')""" + + name: str = SettingsField( + title="Name", + default="" + ) + + type: str = SettingsField( + title="Type", + default="" + ) + + default: str = SettingsField( + title="Default", + default="" + ) + + required: bool = SettingsField( + title="Required", + default=False + ) + + validate: str = SettingsField( + title="Validate", + default="" + ) + + +class ColumnConfigModel(BaseSettingsModel): + """Allows to publish multiple video files in one go.
Name of matching + asset is parsed from file names ('asset.mov', 'asset_v001.mov', + 'my_asset_to_publish.mov')""" + + csv_delimiter: str = SettingsField( + title="CSV delimiter", + default="," + ) + + columns: list[ColumnItemModel] = SettingsField( + title="Columns", + default_factory=list + ) + + @validator("columns") + def validate_unique_outputs(cls, value): + ensure_unique_names(value) + return value + + +class RepresentationItemModel(BaseSettingsModel): + """Allows to publish multiple video files in one go.
Name of matching + asset is parsed from file names ('asset.mov', 'asset_v001.mov', + 'my_asset_to_publish.mov')""" + + name: str = SettingsField( + title="Name", + default="" + ) + + extensions: list[str] = SettingsField( + title="Extensions", + default_factory=list + ) + + @validator("extensions") + def validate_extension(cls, value): + for ext in value: + if not ext.startswith("."): + raise BadRequestException("Extension must start with '.'") + return value + + +class RepresentationConfigModel(BaseSettingsModel): + """Allows to publish multiple video files in one go.
Name of matching + asset is parsed from file names ('asset.mov', 'asset_v001.mov', + 'my_asset_to_publish.mov')""" + + tags_delimiter: str = SettingsField( + title="Tags delimiter", + default=";" + ) + + default_tags: list[str] = SettingsField( + title="Default tags", + default_factory=list + ) + + representation: list[RepresentationItemModel] = SettingsField( + title="Representation", + default_factory=list + ) + + @validator("representation") + def validate_unique_outputs(cls, value): + ensure_unique_names(value) + return value + + +class IngestCSVPluginModel(BaseSettingsModel): + """Allows to publish multiple video files in one go.
Name of matching + asset is parsed from file names ('asset.mov', 'asset_v001.mov', + 'my_asset_to_publish.mov')""" + + enabled: bool = SettingsField( + title="Enabled", + default=False + ) + + columns_config: ColumnConfigModel = SettingsField( + title="Columns config", + default_factory=ColumnConfigModel + ) + + representations_config: dict = SettingsField( + title="Representations config", + default_factory=dict + ) + + class TrayPublisherCreatePluginsModel(BaseSettingsModel): BatchMovieCreator: BatchMovieCreatorPlugin = SettingsField( title="Batch Movie Creator", default_factory=BatchMovieCreatorPlugin ) + IngestCSV: IngestCSVPluginModel = SettingsField( + title="Ingest CSV", + default_factory=IngestCSVPluginModel + ) DEFAULT_CREATORS = { @@ -41,4 +170,170 @@ DEFAULT_CREATORS = { ".mov" ] }, + "IngestCSV": { + "enabled": True, + "columns_config": { + "csv_delimiter": ",", + "columns": [ + { + "name": "File Path", + "type": "text", + "default": "", + "required": True, + "validate": "^([a-z0-9#._\\/]*)$" + }, + { + "name": "Folder Context", + "type": "text", + "default": "", + "required": True, + "validate": "^([a-zA-Z0-9_]*)$" + }, + { + "name": "Task Name", + "type": "text", + "default": "", + "required": True, + "validate": "^(.*)$" + }, + { + "name": "Version", + "type": "number", + "default": 1, + "required": True, + "validate": "^(\\d{1,3})$" + }, + { + "name": "Frame Start", + "type": "number", + "default": 0, + "required": True, + "validate": "^(\\d{1,8})$" + }, + { + "name": "Frame End", + "type": "number", + "default": 0, + "required": True, + "validate": "^(\\d{1,8})$" + }, + { + "name": "Handle Start", + "type": "number", + "default": 0, + "required": True, + "validate": "^(\\d)$" + }, + { + "name": "Handle End", + "type": "number", + "default": 0, + "required": True, + "validate": "^(\\d)$" + }, + { + "name": "FPS", + "type": "decimal", + "default": 0.0, + "required": True, + "validate": "^[0-9]*\\.[0-9]+$|^[0-9]+$" + }, + { + "name": "Thumbnail", + "type": "text", + "default": "", + "required": False, + "validate": "^([a-z0-9#._\\/]*)$" + }, + { + "name": "Colorspace", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + }, + { + "name": "Version Comment", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + }, + { + "name": "Representation", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + }, + { + "name": "Product Type", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + }, + { + "name": "Variant", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + }, + { + "name": "Slate Exists", + "type": "bool", + "default": True, + "required": False, + "validate": "(True|False)" + }, + { + "name": "Representation Tags", + "type": "text", + "default": "", + "required": False, + "validate": "^(.*)$" + } + ] + }, + "representations_config": { + "tags_delimiter": ";", + "default_tags": [ + "review" + ], + "representations": [ + { + "name": "preview", + "extensions": [ + ".mp4", + ".mov" + ] + }, + { + "name": "exr", + "extensions": [ + ".exr" + ] + }, + { + "name": "edit", + "extensions": [ + ".mov" + ] + }, + { + "name": "review", + "extensions": [ + ".mov" + ] + }, + { + "name": "nuke", + "extensions": [ + ".nk" + ] + } + ] + } + } } diff --git a/server_addon/traypublisher/server/version.py b/server_addon/traypublisher/server/version.py index e57ad00718..de699158fd 100644 --- a/server_addon/traypublisher/server/version.py +++ b/server_addon/traypublisher/server/version.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- """Package declaring addon version.""" -__version__ = "0.1.3" +__version__ = "0.1.4" From fe0ccb2a4126bb7f632b3ace39b9f592d1ae0fe7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 22 Feb 2024 17:15:36 +0100 Subject: [PATCH 03/21] Refactor column validation and retrieval logic. - Refactored how columns are validated and retrieved for better clarity. --- .../plugins/create/create_csv_ingest.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index aa986657dc..a11810f902 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -312,14 +312,20 @@ configuration in project settings. _, extension = os.path.splitext(filepath) # validate filepath is having correct extension based on output - config_repre_data = self.representations_config["representations"] repre_name = repre_data["representationName"] - if repre_name not in config_repre_data: + repre_config_data = None + for repre in self.representations_config["representations"]: + if repre["name"] == repre_name: + repre_config_data = repre + break + + if not repre_config_data: raise KeyError( f"Representation '{repre_name}' not found " "in config representation data." ) - validate_extensions = config_repre_data[repre_name]["extensions"] + + validate_extensions = repre_config_data["extensions"] if extension not in validate_extensions: raise TypeError( f"File extension '{extension}' not valid for " @@ -413,8 +419,8 @@ configuration in project settings. # make sure csv file contains columns from following list required_columns = [ - name for name, value in self.columns_config["columns"].items() - if value["required"] + column["name"] for column in self.columns_config["columns"] + if column["required"] ] # get data from csv file with open(csv_file_path, "r") as csv_file: @@ -582,9 +588,14 @@ configuration in project settings. self, column_name, row_data, default_value=None ): """Get row value with validation""" - columns_config = self.columns_config["columns"] + # get column data from column config - column_data = columns_config.get(column_name) + column_data = None + for column in self.columns_config["columns"]: + if column["name"] == column_name: + column_data = column + break + if not column_data: raise KeyError( f"Column '{column_name}' not found in column config." From 5595c021e244470d936059b98e44552a3b078e29 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 22 Feb 2024 22:15:56 +0100 Subject: [PATCH 04/21] Update column attribute names and validation patterns in creator plugins settings. Improve error message for file extension validation. --- .../plugins/create/create_csv_ingest.py | 4 +- .../server/settings/creator_plugins.py | 80 +++++++++---------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index a11810f902..7c379784f7 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -603,7 +603,7 @@ configuration in project settings. # get column value from row column_value = row_data.get(column_name) - column_required = column_data["required"] + column_required = column_data["required_column"] # check if column value is not empty string and column is required if column_value == "" and column_required: @@ -614,7 +614,7 @@ configuration in project settings. # get column type column_type = column_data["type"] # get column validation regex - column_validation = column_data["validate"] + column_validation = column_data["validation_pattern"] # get column default value column_default = default_value or column_data["default"] diff --git a/server_addon/traypublisher/server/settings/creator_plugins.py b/server_addon/traypublisher/server/settings/creator_plugins.py index 7ce241faa6..9f9c31da98 100644 --- a/server_addon/traypublisher/server/settings/creator_plugins.py +++ b/server_addon/traypublisher/server/settings/creator_plugins.py @@ -45,14 +45,14 @@ class ColumnItemModel(BaseSettingsModel): default="" ) - required: bool = SettingsField( - title="Required", + required_column: bool = SettingsField( + title="Required Column", default=False ) - validate: str = SettingsField( - title="Validate", - default="" + validation_pattern: str = SettingsField( + title="Validation Regex Pattern", + default="^(.*)$" ) @@ -96,7 +96,7 @@ class RepresentationItemModel(BaseSettingsModel): def validate_extension(cls, value): for ext in value: if not ext.startswith("."): - raise BadRequestException("Extension must start with '.'") + raise BadRequestException(f"Extension must start with '.': {ext}") return value @@ -179,120 +179,120 @@ DEFAULT_CREATORS = { "name": "File Path", "type": "text", "default": "", - "required": True, - "validate": "^([a-z0-9#._\\/]*)$" + "required_column": True, + "validation_pattern": "^([a-z0-9#._\\/]*)$" }, { "name": "Folder Context", "type": "text", "default": "", - "required": True, - "validate": "^([a-zA-Z0-9_]*)$" + "required_column": True, + "validation_pattern": "^([a-zA-Z0-9_]*)$" }, { "name": "Task Name", "type": "text", "default": "", - "required": True, - "validate": "^(.*)$" + "required_column": True, + "validation_pattern": "^(.*)$" }, { "name": "Version", "type": "number", "default": 1, - "required": True, - "validate": "^(\\d{1,3})$" + "required_column": True, + "validation_pattern": "^(\\d{1,3})$" }, { "name": "Frame Start", "type": "number", "default": 0, - "required": True, - "validate": "^(\\d{1,8})$" + "required_column": True, + "validation_pattern": "^(\\d{1,8})$" }, { "name": "Frame End", "type": "number", "default": 0, - "required": True, - "validate": "^(\\d{1,8})$" + "required_column": True, + "validation_pattern": "^(\\d{1,8})$" }, { "name": "Handle Start", "type": "number", "default": 0, - "required": True, - "validate": "^(\\d)$" + "required_column": True, + "validation_pattern": "^(\\d)$" }, { "name": "Handle End", "type": "number", "default": 0, - "required": True, - "validate": "^(\\d)$" + "required_column": True, + "validation_pattern": "^(\\d)$" }, { "name": "FPS", "type": "decimal", "default": 0.0, - "required": True, - "validate": "^[0-9]*\\.[0-9]+$|^[0-9]+$" + "required_column": True, + "validation_pattern": "^[0-9]*\\.[0-9]+$|^[0-9]+$" }, { "name": "Thumbnail", "type": "text", "default": "", - "required": False, - "validate": "^([a-z0-9#._\\/]*)$" + "required_column": False, + "validation_pattern": "^([a-z0-9#._\\/]*)$" }, { "name": "Colorspace", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" }, { "name": "Version Comment", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" }, { "name": "Representation", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" }, { "name": "Product Type", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" }, { "name": "Variant", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" }, { "name": "Slate Exists", "type": "bool", "default": True, - "required": False, - "validate": "(True|False)" + "required_column": False, + "validation_pattern": "(True|False)" }, { "name": "Representation Tags", "type": "text", "default": "", - "required": False, - "validate": "^(.*)$" + "required_column": False, + "validation_pattern": "^(.*)$" } ] }, From b7bd29389240a518838fbb2124aae26da6cadea8 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Mon, 4 Mar 2024 17:19:51 +0100 Subject: [PATCH 05/21] Update addon and plugin files for Ayon Core hosts traypublisher. - Added import statement for `get_ayon_launcher_args`. - Renamed function call from `get_subset_name` to `get_product_name`. - Changed attribute name from `family` to `product_type` in class IngestCSV. - Updated comments and docstrings in the code. --- client/ayon_core/hosts/traypublisher/addon.py | 1 + .../plugins/create/create_csv_ingest.py | 7 +++---- .../server/settings/creator_plugins.py | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/addon.py b/client/ayon_core/hosts/traypublisher/addon.py index f3884aedfe..ae0705cee2 100644 --- a/client/ayon_core/hosts/traypublisher/addon.py +++ b/client/ayon_core/hosts/traypublisher/addon.py @@ -1,6 +1,7 @@ import os from pathlib import Path +from ayon_core.lib import get_ayon_launcher_args from ayon_core.lib.execute import run_detached_process from ayon_core.addon import ( click_wrap, diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 7c379784f7..193e439581 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -5,7 +5,7 @@ import clique from copy import deepcopy, copy from ayon_core.client import get_asset_by_name -from ayon_core.pipeline.create import get_subset_name +from ayon_core.pipeline.create import get_product_name from ayon_core.pipeline import CreatedInstance from ayon_core.lib import FileDef, BoolDef from ayon_core.lib.transcoding import ( @@ -23,7 +23,7 @@ class IngestCSV(TrayPublishCreator): icon = "fa.file" label = "CSV Ingest" - family = "csv_ingest_file" + product_type = "csv_ingest_file" identifier = "io.ayon_core.creators.traypublisher.csv_ingest" default_variants = ["Main"] @@ -41,7 +41,6 @@ configuration in project settings. columns_config = {} representations_config = {} - def create(self, subset_name, instance_data, pre_create_data): """Create an product from each row found in the CSV. @@ -118,7 +117,7 @@ configuration in project settings. version = product_data["version"] # create subset/product name - product_name = get_subset_name( + product_name = get_product_name( product_type, variant, task_name, diff --git a/server_addon/traypublisher/server/settings/creator_plugins.py b/server_addon/traypublisher/server/settings/creator_plugins.py index 9f9c31da98..82c4d37739 100644 --- a/server_addon/traypublisher/server/settings/creator_plugins.py +++ b/server_addon/traypublisher/server/settings/creator_plugins.py @@ -78,9 +78,11 @@ class ColumnConfigModel(BaseSettingsModel): class RepresentationItemModel(BaseSettingsModel): - """Allows to publish multiple video files in one go.
Name of matching - asset is parsed from file names ('asset.mov', 'asset_v001.mov', - 'my_asset_to_publish.mov')""" + """Allows to publish multiple video files in one go. + + Name of matching asset is parsed from file names + ('asset.mov', 'asset_v001.mov', 'my_asset_to_publish.mov') + """ name: str = SettingsField( title="Name", @@ -115,12 +117,12 @@ class RepresentationConfigModel(BaseSettingsModel): default_factory=list ) - representation: list[RepresentationItemModel] = SettingsField( - title="Representation", + representations: list[RepresentationItemModel] = SettingsField( + title="Representations", default_factory=list ) - @validator("representation") + @validator("representations") def validate_unique_outputs(cls, value): ensure_unique_names(value) return value @@ -141,9 +143,9 @@ class IngestCSVPluginModel(BaseSettingsModel): default_factory=ColumnConfigModel ) - representations_config: dict = SettingsField( + representations_config: RepresentationConfigModel = SettingsField( title="Representations config", - default_factory=dict + default_factory=RepresentationConfigModel ) From 102866da849ecb6e36d0acbbd1743fd6af2e5973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Tue, 5 Mar 2024 13:48:33 +0100 Subject: [PATCH 06/21] Update client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py Co-authored-by: Roy Nieterau --- .../plugins/publish/collect_csv_ingest_instance_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py index 0da3ebed81..1840dbb445 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py @@ -17,7 +17,6 @@ class CollectCSVIngestInstancesData( families = ["csv_ingest"] def process(self, instance): - self.log.info(f"Collecting {instance.name}") # expecting [(colorspace, repre_data), ...] prepared_repres_data_items = instance.data[ From 4cbeb1033aab65ecc2882e1f95e8ae909a504246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Tue, 5 Mar 2024 13:48:41 +0100 Subject: [PATCH 07/21] Update client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py Co-authored-by: Roy Nieterau --- .../plugins/publish/collect_csv_ingest_instance_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py index 1840dbb445..f76ef93c95 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py @@ -32,4 +32,3 @@ class CollectCSVIngestInstancesData( instance.data["representations"].append(repre_data) - self.log.debug(pformat(instance.data)) From ac4e3157bce395f953fd870317cdf426f9de85d7 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Mar 2024 14:43:48 +0100 Subject: [PATCH 08/21] ayon conversion fixes --- client/ayon_core/hosts/traypublisher/csv_publish.py | 10 +--------- .../plugins/create/create_csv_ingest.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/csv_publish.py b/client/ayon_core/hosts/traypublisher/csv_publish.py index f8eed2f2c5..6f2e335f89 100644 --- a/client/ayon_core/hosts/traypublisher/csv_publish.py +++ b/client/ayon_core/hosts/traypublisher/csv_publish.py @@ -35,14 +35,6 @@ def csvpublish( # setting host context into project host.set_project_name(project_name) - # add asset context to environment - # TODO: perhaps this can be done in a better way? - os.environ.update({ - "AVALON_PROJECT": project_name, - "AVALON_ASSET": asset_name, - "AVALON_TASK": task_name or "" - }) - # form precreate data with field values file_field = FileDefItem.from_paths([csv_filepath], False).pop().to_dict() precreate_data = { @@ -57,7 +49,7 @@ def csvpublish( ) create_context.create( - "io.openpype.creators.traypublisher.csv_ingest", + "io.ayon_core.creators.traypublisher.csv_ingest", "Main", asset_doc=asset_doc, task_name=task_name, diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 193e439581..dd2339392f 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -79,7 +79,7 @@ configuration in project settings. ) csv_instance = CreatedInstance( - self.family, subset_name, instance_data, self + self.product_type, subset_name, instance_data, self ) self._store_new_instance(csv_instance) @@ -106,6 +106,7 @@ configuration in project settings. """Create instances from csv data""" for asset_name, _data in csv_data_for_instances.items(): + project_name = self.create_context.get_current_project_name() asset_doc = _data["asset_doc"] products = _data["products"] @@ -118,10 +119,12 @@ configuration in project settings. # create subset/product name product_name = get_product_name( - product_type, - variant, - task_name, + project_name, asset_doc, + task_name, + self.host_name, + product_type, + variant ) # make sure frame start/end is inherited from csv columns @@ -419,7 +422,7 @@ configuration in project settings. # make sure csv file contains columns from following list required_columns = [ column["name"] for column in self.columns_config["columns"] - if column["required"] + if column["required_column"] ] # get data from csv file with open(csv_file_path, "r") as csv_file: From 09effd30fb2d22c519cd16b61b82dd46413454b4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Mar 2024 16:16:52 +0100 Subject: [PATCH 09/21] Update column names and add new fields for better data representation in CSV ingest and creator plugins. --- .../plugins/create/create_csv_ingest.py | 5 +- .../server/settings/creator_plugins.py | 66 +++++++++---------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index dd2339392f..00ff7e00ed 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -545,9 +545,9 @@ configuration in project settings. # Get optional columns thumbnail_path = self._get_row_value_with_validation( - "Thumbnail", row_data) + "Version Thumbnail", row_data) colorspace = self._get_row_value_with_validation( - "Colorspace", row_data) + "Representation Colorspace", row_data) comment = self._get_row_value_with_validation( "Version Comment", row_data) repre = self._get_row_value_with_validation( @@ -583,7 +583,6 @@ configuration in project settings. "handleEnd": int(handle_end), "fps": float(fps), } - return file_path, representation_data def _get_row_value_with_validation( diff --git a/server_addon/traypublisher/server/settings/creator_plugins.py b/server_addon/traypublisher/server/settings/creator_plugins.py index 82c4d37739..3a07a76e6f 100644 --- a/server_addon/traypublisher/server/settings/creator_plugins.py +++ b/server_addon/traypublisher/server/settings/creator_plugins.py @@ -198,6 +198,20 @@ DEFAULT_CREATORS = { "required_column": True, "validation_pattern": "^(.*)$" }, + { + "name": "Product Type", + "type": "text", + "default": "", + "required_column": False, + "validation_pattern": "^(.*)$" + }, + { + "name": "Variant", + "type": "text", + "default": "", + "required_column": False, + "validation_pattern": "^(.*)$" + }, { "name": "Version", "type": "number", @@ -205,6 +219,20 @@ DEFAULT_CREATORS = { "required_column": True, "validation_pattern": "^(\\d{1,3})$" }, + { + "name": "Version Comment", + "type": "text", + "default": "", + "required_column": False, + "validation_pattern": "^(.*)$" + }, + { + "name": "Version Thumbnail", + "type": "text", + "default": "", + "required_column": False, + "validation_pattern": "^([a-zA-Z0-9#._\\/]*)$" + }, { "name": "Frame Start", "type": "number", @@ -241,25 +269,11 @@ DEFAULT_CREATORS = { "validation_pattern": "^[0-9]*\\.[0-9]+$|^[0-9]+$" }, { - "name": "Thumbnail", - "type": "text", - "default": "", + "name": "Slate Exists", + "type": "bool", + "default": True, "required_column": False, - "validation_pattern": "^([a-z0-9#._\\/]*)$" - }, - { - "name": "Colorspace", - "type": "text", - "default": "", - "required_column": False, - "validation_pattern": "^(.*)$" - }, - { - "name": "Version Comment", - "type": "text", - "default": "", - "required_column": False, - "validation_pattern": "^(.*)$" + "validation_pattern": "(True|False)" }, { "name": "Representation", @@ -269,26 +283,12 @@ DEFAULT_CREATORS = { "validation_pattern": "^(.*)$" }, { - "name": "Product Type", + "name": "Representation Colorspace", "type": "text", "default": "", "required_column": False, "validation_pattern": "^(.*)$" }, - { - "name": "Variant", - "type": "text", - "default": "", - "required_column": False, - "validation_pattern": "^(.*)$" - }, - { - "name": "Slate Exists", - "type": "bool", - "default": True, - "required_column": False, - "validation_pattern": "(True|False)" - }, { "name": "Representation Tags", "type": "text", From 8ecb1fa3a542b49308bbfa3c35e11de4ebed5379 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Mar 2024 16:17:08 +0100 Subject: [PATCH 10/21] Refactor representation data handling for thumbnails and media types. - Refactored how thumbnail and media representation data is processed. - Added logic to handle different types of representations based on colorspaces. --- .../plugins/create/create_csv_ingest.py | 40 ++++++++++++++----- .../collect_csv_ingest_instance_data.py | 21 ++++++++-- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 00ff7e00ed..720ef5ddb3 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -246,15 +246,27 @@ configuration in project settings. "stagingDir": thumb_dir, "outputName": explicit_output_name, }) - new_instance["prepared_data_for_repres"].append( - ("_thumbnail_", thumbnail_repr_data) - ) + new_instance["prepared_data_for_repres"].append({ + "type": "thumbnail", + "colorspace": None, + "representation": thumbnail_repr_data, + }) + # also add thumbnailPath for ayon to integrate + if not new_instance.get("thumbnailPath"): + new_instance["thumbnailPath"] = ( + os.path.join(thumb_dir, thumb_file) + ) elif ( thumbnails and not multiple_thumbnails and not thumbnails_processed or not reviewable ): + """ + For case where we have only one thumbnail + and not reviewable medias. This needs to be processed + only once per instance. + """ if not thumbnails: continue # here we will use only one thumbnail for @@ -273,9 +285,17 @@ configuration in project settings. "files": thumb_file, "stagingDir": thumb_dir }) - new_instance["prepared_data_for_repres"].append( - ("_thumbnail_", thumbnail_repr_data) - ) + new_instance["prepared_data_for_repres"].append({ + "type": "thumbnail", + "colorspace": None, + "representation": thumbnail_repr_data, + }) + # also add thumbnailPath for ayon to integrate + if not new_instance.get("thumbnailPath"): + new_instance["thumbnailPath"] = ( + os.path.join(thumb_dir, thumb_file) + ) + thumbnails_processed = True # get representation data @@ -284,9 +304,11 @@ configuration in project settings. explicit_output_name ) - new_instance["prepared_data_for_repres"].append( - (repre_data["colorspace"], representation_data) - ) + new_instance["prepared_data_for_repres"].append({ + "type": "media", + "colorspace": repre_data["colorspace"], + "representation": representation_data, + }) def _get_refactor_thumbnail_path( self, staging_dir, relative_thumbnail_path): diff --git a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py index f76ef93c95..33536d0854 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py +++ b/client/ayon_core/hosts/traypublisher/plugins/publish/collect_csv_ingest_instance_data.py @@ -22,13 +22,26 @@ class CollectCSVIngestInstancesData( prepared_repres_data_items = instance.data[ "prepared_data_for_repres"] - for colorspace, repre_data in prepared_repres_data_items: - # only apply colorspace to those which are not marked as thumbnail - if colorspace != "_thumbnail_": + for prep_repre_data in prepared_repres_data_items: + type = prep_repre_data["type"] + colorspace = prep_repre_data["colorspace"] + repre_data = prep_repre_data["representation"] + + # thumbnails should be skipped + if type == "media": # colorspace name is passed from CSV column self.set_representation_colorspace( repre_data, instance.context, colorspace ) + elif type == "media" and colorspace is None: + # TODO: implement colorspace file rules file parsing + self.log.warning( + "Colorspace is not defined in csv for following" + f" representation: {pformat(repre_data)}" + ) + pass + elif type == "thumbnail": + # thumbnails should be skipped + pass instance.data["representations"].append(repre_data) - From 8bb9070a9e3d4a775360025197076a528f5c2af4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 5 Mar 2024 16:19:37 +0100 Subject: [PATCH 11/21] comment Improve handling of thumbnails by removing unnecessary iteration. --- .../hosts/traypublisher/plugins/create/create_csv_ingest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 720ef5ddb3..84ab5b72a1 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -272,6 +272,8 @@ configuration in project settings. # here we will use only one thumbnail for # all representations relative_thumbnail_path = repre_data["thumbnailPath"] + # popping last thumbnail from list since it is only one + # and we do not need to iterate again over it if not relative_thumbnail_path: relative_thumbnail_path = thumbnails.pop() thumb_dir, thumb_file = \ From 346db35546d67b38afcb9cdd09529720b6462c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Mon, 11 Mar 2024 12:17:20 +0100 Subject: [PATCH 12/21] Update client/ayon_core/hosts/traypublisher/csv_publish.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/hosts/traypublisher/csv_publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/csv_publish.py b/client/ayon_core/hosts/traypublisher/csv_publish.py index 6f2e335f89..c9fbbf917a 100644 --- a/client/ayon_core/hosts/traypublisher/csv_publish.py +++ b/client/ayon_core/hosts/traypublisher/csv_publish.py @@ -49,7 +49,7 @@ def csvpublish( ) create_context.create( - "io.ayon_core.creators.traypublisher.csv_ingest", + "io.ayon.creators.traypublisher.csv_ingest", "Main", asset_doc=asset_doc, task_name=task_name, From feffa4db66efa415683f3b0253de01ae773c2bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Mon, 11 Mar 2024 12:17:27 +0100 Subject: [PATCH 13/21] Update client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../hosts/traypublisher/plugins/create/create_csv_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 84ab5b72a1..4ec6ca302a 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -24,7 +24,7 @@ class IngestCSV(TrayPublishCreator): label = "CSV Ingest" product_type = "csv_ingest_file" - identifier = "io.ayon_core.creators.traypublisher.csv_ingest" + identifier = "io.ayon.creators.traypublisher.csv_ingest" default_variants = ["Main"] From 0252e8796259221b418c4e3aa6e6e951cffe2674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Mon, 11 Mar 2024 12:22:28 +0100 Subject: [PATCH 14/21] Update client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- .../hosts/traypublisher/plugins/create/create_csv_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 4ec6ca302a..31d0a022c5 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -141,7 +141,7 @@ configuration in project settings. version_comment = next( iter( repre_data["comment"] - for _, repre_data in product_data["representations"].items() # noqa: E501 + for repre_data in product_data["representations"].values() # noqa: E501 if repre_data["comment"] ), None From 638c68324139fbe365d36615ef21910a83f2fab2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 19 Mar 2024 10:19:50 +0100 Subject: [PATCH 15/21] Update CLI options and function parameters for CSV file ingestion and publishing. Refactor variable names for clarity in functions related to file handling and context creation. --- client/ayon_core/hosts/traypublisher/addon.py | 28 +++++++++---------- .../hosts/traypublisher/csv_publish.py | 16 +++++------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/addon.py b/client/ayon_core/hosts/traypublisher/addon.py index ae0705cee2..ee42784f98 100644 --- a/client/ayon_core/hosts/traypublisher/addon.py +++ b/client/ayon_core/hosts/traypublisher/addon.py @@ -62,25 +62,25 @@ def launch(): @cli_main.command() @click_wrap.option( - "--csv-filepath", + "--filepath", help="Full path to CSV file with data", type=str, required=True ) @click_wrap.option( - "--project-name", + "--project", help="Project name in which the context will be used", type=str, required=True ) @click_wrap.option( - "--asset-name", + "--folder", help="Asset name in which the context will be used", type=str, required=True ) @click_wrap.option( - "--task-name", + "--task", help="Task name under Asset in which the context will be used", type=str, required=False @@ -93,10 +93,10 @@ def launch(): required=False ) def ingestcsv( - csv_filepath, - project_name, - asset_name, - task_name, + filepath, + project, + folder, + task, ignore_validators ): """Ingest CSV file into project. @@ -107,13 +107,13 @@ def ingestcsv( from .csv_publish import csvpublish # use Path to check if csv_filepath exists - if not Path(csv_filepath).exists(): - raise FileNotFoundError(f"File {csv_filepath} does not exist.") + if not Path(filepath).exists(): + raise FileNotFoundError(f"File {filepath} does not exist.") csvpublish( - csv_filepath, - project_name, - asset_name, - task_name, + filepath, + project, + folder, + task, ignore_validators ) diff --git a/client/ayon_core/hosts/traypublisher/csv_publish.py b/client/ayon_core/hosts/traypublisher/csv_publish.py index c9fbbf917a..32c2b69371 100644 --- a/client/ayon_core/hosts/traypublisher/csv_publish.py +++ b/client/ayon_core/hosts/traypublisher/csv_publish.py @@ -3,7 +3,7 @@ import os import pyblish.api import pyblish.util -from ayon_core.client import get_asset_by_name +from ayon_api import get_folder_by_name from ayon_core.lib.attribute_definitions import FileDefItem from ayon_core.pipeline import install_host from ayon_core.pipeline.create import CreateContext @@ -12,18 +12,18 @@ from ayon_core.hosts.traypublisher.api import TrayPublisherHost def csvpublish( - csv_filepath, + filepath, project_name, - asset_name, + folder_name, task_name=None, ignore_validators=False ): """Publish CSV file. Args: - csv_filepath (str): Path to CSV file. + filepath (str): Path to CSV file. project_name (str): Project name. - asset_name (str): Asset name. + folder_name (str): Folder name. task_name (Optional[str]): Task name. ignore_validators (Optional[bool]): Option to ignore validators. """ @@ -36,16 +36,16 @@ def csvpublish( host.set_project_name(project_name) # form precreate data with field values - file_field = FileDefItem.from_paths([csv_filepath], False).pop().to_dict() + file_field = FileDefItem.from_paths([filepath], False).pop().to_dict() precreate_data = { "csv_filepath_data": file_field, } # create context initialization create_context = CreateContext(host, headless=True) - asset_doc = get_asset_by_name( + asset_doc = get_folder_by_name( project_name, - asset_name + folder_name=folder_name ) create_context.create( From 1c11e18314ff929f1fe1b9b7048a063552143088 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 19 Mar 2024 11:11:24 +0100 Subject: [PATCH 16/21] Update CSV ingest plugin to handle folder paths and task types. Improve error handling for missing assets and tasks. Refactor data processing for better organization and validation. --- .../plugins/create/create_csv_ingest.py | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 31d0a022c5..9d6f04ae99 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -4,14 +4,14 @@ import csv import clique from copy import deepcopy, copy -from ayon_core.client import get_asset_by_name +from ayon_api import get_folder_by_path, get_task_by_name from ayon_core.pipeline.create import get_product_name from ayon_core.pipeline import CreatedInstance from ayon_core.lib import FileDef, BoolDef from ayon_core.lib.transcoding import ( VIDEO_EXTENSIONS, IMAGE_EXTENSIONS ) - +from ayon_core.pipeline.create import CreatorError from ayon_core.hosts.traypublisher.api.plugin import ( TrayPublishCreator ) @@ -54,7 +54,7 @@ configuration in project settings. folder = csv_filepath_data.get("directory", "") if not os.path.exists(folder): - raise FileNotFoundError( + raise CreatorError( f"Directory '{folder}' does not exist." ) filename = csv_filepath_data.get("filenames", []) @@ -105,14 +105,14 @@ configuration in project settings. ): """Create instances from csv data""" - for asset_name, _data in csv_data_for_instances.items(): + for folder_path, prepared_data in csv_data_for_instances.items(): project_name = self.create_context.get_current_project_name() - asset_doc = _data["asset_doc"] - products = _data["products"] + products = prepared_data["products"] for instance_name, product_data in products.items(): # get important instance variables task_name = product_data["task_name"] + task_type = product_data["task_type"] variant = product_data["variant"] product_type = product_data["product_type"] version = product_data["version"] @@ -120,8 +120,8 @@ configuration in project settings. # create subset/product name product_name = get_product_name( project_name, - asset_doc, task_name, + task_type, self.host_name, product_type, variant @@ -155,7 +155,7 @@ configuration in project settings. # get representations from product data representations = product_data["representations"] - label = f"{asset_name}_{product_name}_v{version:>03}" + label = f"{folder_path}_{product_name}_v{version:>03}" families = ["csv_ingest"] if slate_exists: @@ -166,7 +166,7 @@ configuration in project settings. # make product data product_data = { "name": instance_name, - "asset": asset_name, + "folderPath": folder_path, "families": families, "label": label, "task": task_name, @@ -471,8 +471,10 @@ configuration in project settings. # get data from csv file for row in csv_reader: # Get required columns first - context_asset_name = self._get_row_value_with_validation( - "Folder Context", row) + # TODO: will need to be folder path in CSV + # TODO: `context_asset_name` is now `folder_path` + folder_path = self._get_row_value_with_validation( + "Folder Path", row) task_name = self._get_row_value_with_validation( "Task Name", row) version = self._get_row_value_with_validation( @@ -493,31 +495,40 @@ configuration in project settings. filename, representation_data = \ self._get_representation_row_data(row) + # TODO: batch query of all folder paths and task names + + # get folder entity from folder path + folder_entity = get_folder_by_path( + project_name, folder_path) + + # make sure asset exists + if not folder_entity: + raise CreatorError( + f"Asset '{folder_path}' not found." + ) + + # first get all tasks on the folder entity and then find + task_entity = get_task_by_name( + project_name, folder_entity["id"], task_name) + + # check if task name is valid task in asset doc + if not task_entity: + raise CreatorError( + f"Task '{task_name}' not found in asset doc." + ) + # get all csv data into one dict and make sure there are no # duplicates data are already validated and sorted under # correct existing asset also check if asset exists and if # task name is valid task in asset doc and representations # are distributed under products following variants - if context_asset_name not in csv_data: - asset_doc = get_asset_by_name( - project_name, context_asset_name) - - # make sure asset exists - if not asset_doc: - raise ValueError( - f"Asset '{context_asset_name}' not found." - ) - # check if task name is valid task in asset doc - if task_name not in asset_doc["data"]["tasks"]: - raise ValueError( - f"Task '{task_name}' not found in asset doc." - ) - - csv_data[context_asset_name] = { - "asset_doc": asset_doc, + if folder_path not in csv_data: + csv_data[folder_path] = { + "folder_entity": folder_entity, "products": { pre_product_name: { "task_name": task_name, + "task_type": task_entity["taskType"], "variant": variant, "product_type": product_type, "version": version, @@ -528,11 +539,11 @@ configuration in project settings. } } else: - asset_doc = csv_data[context_asset_name]["asset_doc"] - csv_products = csv_data[context_asset_name]["products"] + csv_products = csv_data[folder_path]["products"] if pre_product_name not in csv_products: csv_products[pre_product_name] = { "task_name": task_name, + "task_type": task_entity["taskType"], "variant": variant, "product_type": product_type, "version": version, From 175d299ccc5b20feea95242f156f7d15f6e0b132 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 19 Mar 2024 13:12:30 +0100 Subject: [PATCH 17/21] Refactor error handling to use custom CreatorError class. - Replaced KeyError, TypeError, NotADirectoryError, ValueError with CreatorError for consistency and better error management. --- .../plugins/create/create_csv_ingest.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 9d6f04ae99..1381059fbb 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -346,14 +346,14 @@ configuration in project settings. break if not repre_config_data: - raise KeyError( + raise CreatorError( f"Representation '{repre_name}' not found " "in config representation data." ) validate_extensions = repre_config_data["extensions"] if extension not in validate_extensions: - raise TypeError( + raise CreatorError( f"File extension '{extension}' not valid for " f"output '{validate_extensions}'." ) @@ -373,7 +373,7 @@ configuration in project settings. # check if dirname exists if not os.path.isdir(dirname): - raise NotADirectoryError( + raise CreatorError( f"Directory '{dirname}' does not exist." ) @@ -389,7 +389,7 @@ configuration in project settings. collections = collections[0] else: if is_sequence: - raise ValueError( + raise CreatorError( f"No collections found in directory '{dirname}'." ) @@ -463,7 +463,7 @@ configuration in project settings. # check if csv file contains all required columns if any(column not in all_columns for column in required_columns): - raise KeyError( + raise CreatorError( f"Missing required columns: {required_columns}" ) @@ -555,7 +555,7 @@ configuration in project settings. csv_representations = \ csv_products[pre_product_name]["representations"] if filename in csv_representations: - raise ValueError( + raise CreatorError( f"Duplicate filename '{filename}' in csv file." ) csv_representations[filename] = representation_data @@ -633,7 +633,7 @@ configuration in project settings. break if not column_data: - raise KeyError( + raise CreatorError( f"Column '{column_name}' not found in column config." ) @@ -643,7 +643,7 @@ configuration in project settings. # check if column value is not empty string and column is required if column_value == "" and column_required: - raise ValueError( + raise CreatorError( f"Value in column '{column_name}' is required." ) @@ -675,7 +675,7 @@ configuration in project settings. column_value is not None and not re.match(str(column_validation), str(column_value)) ): - raise ValueError( + raise CreatorError( f"Column '{column_name}' value '{column_value}' " f"does not match validation regex '{column_validation}' \n" f"Row data: {row_data} \n" @@ -719,7 +719,7 @@ configuration in project settings. Returns: list: list of attribute object instances """ - # Use same attributes as for instance attrobites + # Use same attributes as for instance attributes attr_defs = [ FileDef( "csv_filepath_data", From 9090706252175d1cfcb38203214a376f1fd5a84b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 19 Mar 2024 13:19:25 +0100 Subject: [PATCH 18/21] Refactor CSV file reading and data processing logic - Added import for StringIO - Refactored CSV file reading using StringIO - Improved handling of fieldnames and required columns detection --- .../plugins/create/create_csv_ingest.py | 196 +++++++++--------- 1 file changed, 102 insertions(+), 94 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py index 1381059fbb..8143e8b45b 100644 --- a/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py +++ b/client/ayon_core/hosts/traypublisher/plugins/create/create_csv_ingest.py @@ -2,6 +2,7 @@ import os import re import csv import clique +from io import StringIO from copy import deepcopy, copy from ayon_api import get_folder_by_path, get_task_by_name @@ -335,7 +336,7 @@ configuration in project settings. # get extension of file basename = os.path.basename(filepath) - _, extension = os.path.splitext(filepath) + extension = os.path.splitext(filepath)[-1].lower() # validate filepath is having correct extension based on output repre_name = repre_data["representationName"] @@ -448,100 +449,92 @@ configuration in project settings. column["name"] for column in self.columns_config["columns"] if column["required_column"] ] - # get data from csv file + + # read csv file with open(csv_file_path, "r") as csv_file: - csv_reader = csv.DictReader( - csv_file, delimiter=self.columns_config["csv_delimiter"]) + csv_content = csv_file.read() - # fix fieldnames - # sometimes someone can keep extra space at the start or end of - # the column name - all_columns = [ - " ".join(column.rsplit()) for column in csv_reader.fieldnames] - # return back fixed fieldnames - csv_reader.fieldnames = all_columns + # read csv file with DictReader + csv_reader = csv.DictReader( + StringIO(csv_content), + delimiter=self.columns_config["csv_delimiter"] + ) - # check if csv file contains all required columns - if any(column not in all_columns for column in required_columns): + # fix fieldnames + # sometimes someone can keep extra space at the start or end of + # the column name + all_columns = [ + " ".join(column.rsplit()) for column in csv_reader.fieldnames] + + # return back fixed fieldnames + csv_reader.fieldnames = all_columns + + # check if csv file contains all required columns + if any(column not in all_columns for column in required_columns): + raise CreatorError( + f"Missing required columns: {required_columns}" + ) + + csv_data = {} + # get data from csv file + for row in csv_reader: + # Get required columns first + # TODO: will need to be folder path in CSV + # TODO: `context_asset_name` is now `folder_path` + folder_path = self._get_row_value_with_validation( + "Folder Path", row) + task_name = self._get_row_value_with_validation( + "Task Name", row) + version = self._get_row_value_with_validation( + "Version", row) + + # Get optional columns + variant = self._get_row_value_with_validation( + "Variant", row) + product_type = self._get_row_value_with_validation( + "Product Type", row) + + pre_product_name = ( + f"{task_name}{variant}{product_type}" + f"{version}".replace(" ", "").lower() + ) + + # get representation data + filename, representation_data = \ + self._get_representation_row_data(row) + + # TODO: batch query of all folder paths and task names + + # get folder entity from folder path + folder_entity = get_folder_by_path( + project_name, folder_path) + + # make sure asset exists + if not folder_entity: raise CreatorError( - f"Missing required columns: {required_columns}" + f"Asset '{folder_path}' not found." ) - csv_data = {} - # get data from csv file - for row in csv_reader: - # Get required columns first - # TODO: will need to be folder path in CSV - # TODO: `context_asset_name` is now `folder_path` - folder_path = self._get_row_value_with_validation( - "Folder Path", row) - task_name = self._get_row_value_with_validation( - "Task Name", row) - version = self._get_row_value_with_validation( - "Version", row) + # first get all tasks on the folder entity and then find + task_entity = get_task_by_name( + project_name, folder_entity["id"], task_name) - # Get optional columns - variant = self._get_row_value_with_validation( - "Variant", row) - product_type = self._get_row_value_with_validation( - "Product Type", row) - - pre_product_name = ( - f"{task_name}{variant}{product_type}" - f"{version}".replace(" ", "").lower() + # check if task name is valid task in asset doc + if not task_entity: + raise CreatorError( + f"Task '{task_name}' not found in asset doc." ) - # get representation data - filename, representation_data = \ - self._get_representation_row_data(row) - - # TODO: batch query of all folder paths and task names - - # get folder entity from folder path - folder_entity = get_folder_by_path( - project_name, folder_path) - - # make sure asset exists - if not folder_entity: - raise CreatorError( - f"Asset '{folder_path}' not found." - ) - - # first get all tasks on the folder entity and then find - task_entity = get_task_by_name( - project_name, folder_entity["id"], task_name) - - # check if task name is valid task in asset doc - if not task_entity: - raise CreatorError( - f"Task '{task_name}' not found in asset doc." - ) - - # get all csv data into one dict and make sure there are no - # duplicates data are already validated and sorted under - # correct existing asset also check if asset exists and if - # task name is valid task in asset doc and representations - # are distributed under products following variants - if folder_path not in csv_data: - csv_data[folder_path] = { - "folder_entity": folder_entity, - "products": { - pre_product_name: { - "task_name": task_name, - "task_type": task_entity["taskType"], - "variant": variant, - "product_type": product_type, - "version": version, - "representations": { - filename: representation_data, - }, - } - } - } - else: - csv_products = csv_data[folder_path]["products"] - if pre_product_name not in csv_products: - csv_products[pre_product_name] = { + # get all csv data into one dict and make sure there are no + # duplicates data are already validated and sorted under + # correct existing asset also check if asset exists and if + # task name is valid task in asset doc and representations + # are distributed under products following variants + if folder_path not in csv_data: + csv_data[folder_path] = { + "folder_entity": folder_entity, + "products": { + pre_product_name: { "task_name": task_name, "task_type": task_entity["taskType"], "variant": variant, @@ -551,14 +544,29 @@ configuration in project settings. filename: representation_data, }, } - else: - csv_representations = \ - csv_products[pre_product_name]["representations"] - if filename in csv_representations: - raise CreatorError( - f"Duplicate filename '{filename}' in csv file." - ) - csv_representations[filename] = representation_data + } + } + else: + csv_products = csv_data[folder_path]["products"] + if pre_product_name not in csv_products: + csv_products[pre_product_name] = { + "task_name": task_name, + "task_type": task_entity["taskType"], + "variant": variant, + "product_type": product_type, + "version": version, + "representations": { + filename: representation_data, + }, + } + else: + csv_representations = \ + csv_products[pre_product_name]["representations"] + if filename in csv_representations: + raise CreatorError( + f"Duplicate filename '{filename}' in csv file." + ) + csv_representations[filename] = representation_data return csv_data From 6a01b8b6ac6a7f3701b6be7fdfcbdd073864c2a9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 19 Mar 2024 13:19:35 +0100 Subject: [PATCH 19/21] Update creator plugin field name from "Folder Context" to "Folder Path" and adjust validation pattern to allow slashes. --- server_addon/traypublisher/server/settings/creator_plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server_addon/traypublisher/server/settings/creator_plugins.py b/server_addon/traypublisher/server/settings/creator_plugins.py index 3a07a76e6f..1ff14002aa 100644 --- a/server_addon/traypublisher/server/settings/creator_plugins.py +++ b/server_addon/traypublisher/server/settings/creator_plugins.py @@ -185,11 +185,11 @@ DEFAULT_CREATORS = { "validation_pattern": "^([a-z0-9#._\\/]*)$" }, { - "name": "Folder Context", + "name": "Folder Path", "type": "text", "default": "", "required_column": True, - "validation_pattern": "^([a-zA-Z0-9_]*)$" + "validation_pattern": "^([a-zA-Z0-9_\\/]*)$" }, { "name": "Task Name", From 3791731b6ac2b42583f56ccc69501f39b6372f06 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 26 Mar 2024 14:45:30 +0100 Subject: [PATCH 20/21] Refactor option names and handle folder & task entities. - Renamed "--folder" to "--folder-path" for clarity. - Updated function parameters to use "folder_path" consistently. - Added error handling for non-existent folder or task paths. --- client/ayon_core/hosts/traypublisher/addon.py | 6 ++-- .../hosts/traypublisher/csv_publish.py | 32 +++++++++++++++---- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/client/ayon_core/hosts/traypublisher/addon.py b/client/ayon_core/hosts/traypublisher/addon.py index ee42784f98..3dd275f223 100644 --- a/client/ayon_core/hosts/traypublisher/addon.py +++ b/client/ayon_core/hosts/traypublisher/addon.py @@ -74,7 +74,7 @@ def launch(): required=True ) @click_wrap.option( - "--folder", + "--folder-path", help="Asset name in which the context will be used", type=str, required=True @@ -95,7 +95,7 @@ def launch(): def ingestcsv( filepath, project, - folder, + folder_path, task, ignore_validators ): @@ -113,7 +113,7 @@ def ingestcsv( csvpublish( filepath, project, - folder, + folder_path, task, ignore_validators ) diff --git a/client/ayon_core/hosts/traypublisher/csv_publish.py b/client/ayon_core/hosts/traypublisher/csv_publish.py index 32c2b69371..b43792a357 100644 --- a/client/ayon_core/hosts/traypublisher/csv_publish.py +++ b/client/ayon_core/hosts/traypublisher/csv_publish.py @@ -3,7 +3,7 @@ import os import pyblish.api import pyblish.util -from ayon_api import get_folder_by_name +from ayon_api import get_folder_by_path, get_task_by_name from ayon_core.lib.attribute_definitions import FileDefItem from ayon_core.pipeline import install_host from ayon_core.pipeline.create import CreateContext @@ -14,7 +14,7 @@ from ayon_core.hosts.traypublisher.api import TrayPublisherHost def csvpublish( filepath, project_name, - folder_name, + folder_path, task_name=None, ignore_validators=False ): @@ -23,7 +23,7 @@ def csvpublish( Args: filepath (str): Path to CSV file. project_name (str): Project name. - folder_name (str): Folder name. + folder_path (str): Folder path. task_name (Optional[str]): Task name. ignore_validators (Optional[bool]): Option to ignore validators. """ @@ -43,16 +43,34 @@ def csvpublish( # create context initialization create_context = CreateContext(host, headless=True) - asset_doc = get_folder_by_name( + folder_entity = get_folder_by_path( project_name, - folder_name=folder_name + folder_path=folder_path, ) + if not folder_entity: + ValueError( + f"Folder path '{folder_path}' doesn't " + f"exists at project '{project_name}'." + ) + + task_entity = get_task_by_name( + project_name, + folder_entity["id"], + task_name, + ) + + if not task_entity: + ValueError( + f"Task name '{task_name}' doesn't " + f"exists at folder '{folder_path}'." + ) + create_context.create( "io.ayon.creators.traypublisher.csv_ingest", "Main", - asset_doc=asset_doc, - task_name=task_name, + folder_entity=folder_entity, + task_entity=task_entity, pre_create_data=precreate_data, ) From efcf5148bd253968b79af43d68654eef66695e81 Mon Sep 17 00:00:00 2001 From: Roy Nieterau Date: Wed, 27 Mar 2024 14:47:31 +0100 Subject: [PATCH 21/21] Maya: Yeti - Implement writing and loading user variables with a yeti cache --- client/ayon_core/hosts/maya/api/yeti.py | 101 ++++++++++++++++++ .../maya/plugins/load/load_yeti_cache.py | 41 +++++++ .../plugins/publish/collect_yeti_cache.py | 20 +++- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 client/ayon_core/hosts/maya/api/yeti.py diff --git a/client/ayon_core/hosts/maya/api/yeti.py b/client/ayon_core/hosts/maya/api/yeti.py new file mode 100644 index 0000000000..1526c3a2f3 --- /dev/null +++ b/client/ayon_core/hosts/maya/api/yeti.py @@ -0,0 +1,101 @@ +from typing import List + +from maya import cmds + + +def get_yeti_user_variables(yeti_shape_node: str) -> List[str]: + """Get user defined yeti user variables for a `pgYetiMaya` shape node. + + Arguments: + yeti_shape_node (str): The `pgYetiMaya` shape node. + + Returns: + list: Attribute names (for a vector attribute it only lists the top + parent attribute, not the attribute per axis) + """ + + attrs = cmds.listAttr(yeti_shape_node, + userDefined=True, + string=("yetiVariableV_*", + "yetiVariableF_*")) or [] + valid_attrs = [] + for attr in attrs: + attr_type = cmds.attributeQuery(attr, node=yeti_shape_node, + attributeType=True) + if attr.startswith("yetiVariableV_") and attr_type == "double3": + # vector + valid_attrs.append(attr) + elif attr.startswith("yetiVariableF_") and attr_type == "double": + valid_attrs.append(attr) + + return valid_attrs + + +def create_yeti_variable(yeti_shape_node: str, + attr_name: str, + value=None, + force_value: bool = False) -> bool: + """Get user defined yeti user variables for a `pgYetiMaya` shape node. + + Arguments: + yeti_shape_node (str): The `pgYetiMaya` shape node. + attr_name (str): The fully qualified yeti variable name, e.g. + "yetiVariableF_myfloat" or "yetiVariableV_myvector" + value (object): The value to set (must match the type of the attribute) + When value is None it will ignored and not be set. + force_value (bool): Whether to set the value if the attribute already + exists or not. + + Returns: + bool: Whether the attribute value was set or not. + + """ + exists = cmds.attributeQuery(attr_name, node=yeti_shape_node, exists=True) + if not exists: + if attr_name.startswith("yetiVariableV_"): + _create_vector_yeti_user_variable(yeti_shape_node, attr_name) + if attr_name.startswith("yetiVariableF_"): + _create_float_yeti_user_variable(yeti_shape_node, attr_name) + + if value is not None and (not exists or force_value): + plug = "{}.{}".format(yeti_shape_node, attr_name) + if ( + isinstance(value, (list, tuple)) + and attr_name.startswith("yetiVariableV_") + ): + cmds.setAttr(plug, *value, type="double3") + else: + cmds.setAttr(plug, value) + + return True + return False + + +def _create_vector_yeti_user_variable(yeti_shape_node: str, attr_name: str): + if not attr_name.startswith("yetiVariableV_"): + raise ValueError("Must start with yetiVariableV_") + cmds.addAttr(yeti_shape_node, + longName=attr_name, + attributeType="double3", + cachedInternally=True, + keyable=True) + for axis in "XYZ": + cmds.addAttr(yeti_shape_node, + longName="{}{}".format(attr_name, axis), + attributeType="double", + parent=attr_name, + cachedInternally=True, + keyable=True) + + +def _create_float_yeti_user_variable(yeti_node: str, attr_name: str): + if not attr_name.startswith("yetiVariableF_"): + raise ValueError("Must start with yetiVariableF_") + + cmds.addAttr(yeti_node, + longName=attr_name, + attributeType="double", + cachedInternally=True, + softMinValue=0, + softMaxValue=100, + keyable=True) diff --git a/client/ayon_core/hosts/maya/plugins/load/load_yeti_cache.py b/client/ayon_core/hosts/maya/plugins/load/load_yeti_cache.py index a5cd04b0f4..06f74e5107 100644 --- a/client/ayon_core/hosts/maya/plugins/load/load_yeti_cache.py +++ b/client/ayon_core/hosts/maya/plugins/load/load_yeti_cache.py @@ -12,6 +12,7 @@ from ayon_core.pipeline import ( get_representation_path ) from ayon_core.hosts.maya.api import lib +from ayon_core.hosts.maya.api.yeti import create_yeti_variable from ayon_core.hosts.maya.api.pipeline import containerise from ayon_core.hosts.maya.api.plugin import get_load_color_for_product_type @@ -23,8 +24,19 @@ SKIP_UPDATE_ATTRS = { "viewportDensity", "viewportWidth", "viewportLength", + "renderDensity", + "renderWidth", + "renderLength", + "increaseRenderBounds" } +SKIP_ATTR_MESSAGE = ( + "Skipping updating %s.%s to %s because it " + "is considered a local overridable attribute. " + "Either set manually or the load the cache " + "anew." +) + def set_attribute(node, attr, value): """Wrapper of set attribute which ignores None values""" @@ -209,9 +221,31 @@ class YetiCacheLoader(load.LoaderPlugin): for attr, value in node_settings["attrs"].items(): if attr in SKIP_UPDATE_ATTRS: + self.log.info( + SKIP_ATTR_MESSAGE, yeti_node, attr, value + ) continue set_attribute(attr, value, yeti_node) + # Set up user defined attributes + user_variables = node_settings.get("user_variables", {}) + for attr, value in user_variables.items(): + was_value_set = create_yeti_variable( + yeti_shape_node=yeti_node, + attr_name=attr, + value=value, + # We do not want to update the + # value if it already exists so + # that any local overrides that + # may have been applied still + # persist + force_value=False + ) + if not was_value_set: + self.log.info( + SKIP_ATTR_MESSAGE, yeti_node, attr, value + ) + cmds.setAttr("{}.representation".format(container_node), repre_entity["id"], typ="string") @@ -332,6 +366,13 @@ class YetiCacheLoader(load.LoaderPlugin): for attr, value in attributes.items(): set_attribute(attr, value, yeti_node) + # Set up user defined attributes + user_variables = node_settings.get("user_variables", {}) + for attr, value in user_variables.items(): + create_yeti_variable(yeti_shape_node=yeti_node, + attr_name=attr, + value=value) + # Connect to the time node cmds.connectAttr("time1.outTime", "%s.currentTime" % yeti_node) diff --git a/client/ayon_core/hosts/maya/plugins/publish/collect_yeti_cache.py b/client/ayon_core/hosts/maya/plugins/publish/collect_yeti_cache.py index 067a7bc532..e1755e4212 100644 --- a/client/ayon_core/hosts/maya/plugins/publish/collect_yeti_cache.py +++ b/client/ayon_core/hosts/maya/plugins/publish/collect_yeti_cache.py @@ -3,6 +3,7 @@ from maya import cmds import pyblish.api from ayon_core.hosts.maya.api import lib +from ayon_core.hosts.maya.api.yeti import get_yeti_user_variables SETTINGS = { @@ -34,7 +35,7 @@ class CollectYetiCache(pyblish.api.InstancePlugin): - "increaseRenderBounds" - "imageSearchPath" - Other information is the name of the transform and it's Colorbleed ID + Other information is the name of the transform and its `cbId` """ order = pyblish.api.CollectorOrder + 0.45 @@ -54,6 +55,16 @@ class CollectYetiCache(pyblish.api.InstancePlugin): # Get specific node attributes attr_data = {} for attr in SETTINGS: + # Ignore non-existing attributes with a warning, e.g. cbId + # if they have not been generated yet + if not cmds.attributeQuery(attr, node=shape, exists=True): + self.log.warning( + "Attribute '{}' not found on Yeti node: {}".format( + attr, shape + ) + ) + continue + current = cmds.getAttr("%s.%s" % (shape, attr)) # change None to empty string as Maya doesn't support # NoneType in attributes @@ -61,6 +72,12 @@ class CollectYetiCache(pyblish.api.InstancePlugin): current = "" attr_data[attr] = current + # Get user variable attributes + user_variable_attrs = { + attr: lib.get_attribute("{}.{}".format(shape, attr)) + for attr in get_yeti_user_variables(shape) + } + # Get transform data parent = cmds.listRelatives(shape, parent=True)[0] transform_data = {"name": parent, "cbId": lib.get_id(parent)} @@ -70,6 +87,7 @@ class CollectYetiCache(pyblish.api.InstancePlugin): "name": shape, "cbId": lib.get_id(shape), "attrs": attr_data, + "user_variables": user_variable_attrs } settings["nodes"].append(shape_data)