From 7bd1f630e732ca0ea92ebbc1ecf8646b90de0c7f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 12:05:41 +0200 Subject: [PATCH 01/14] moved check of representations earlier --- .../plugins/publish/integrate_ftrack_instances.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py index b54db918a6..b9a486d9da 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py @@ -40,6 +40,13 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): def process(self, instance): self.log.debug("instance {}".format(instance)) + instance_repres = instance.data.get("representations") + if not instance_repres: + self.log.info(( + "Skipping instance. Does not have any representations {}" + ).format(str(instance))) + return + instance_version = instance.data.get("version") if instance_version is None: raise ValueError("Instance version not set") @@ -64,13 +71,6 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): ).format(family)) return - instance_repres = instance.data.get("representations") - if not instance_repres: - self.log.info(( - "Skipping instance. Does not have any representations {}" - ).format(str(instance))) - return - # Prepare FPS instance_fps = instance.data.get("fps") if instance_fps is None: From 66209b27cd0ad2efd90655ad352b66a2a043100a Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 12:05:56 +0200 Subject: [PATCH 02/14] added default asset type into integrate ftrack instances --- .../ftrack/plugins/publish/integrate_ftrack_instances.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py index b9a486d9da..5ea0469bce 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_instances.py @@ -60,8 +60,12 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): if not asset_type and family_low in self.family_mapping: asset_type = self.family_mapping[family_low] - self.log.debug(self.family_mapping) - self.log.debug(family_low) + if not asset_type: + asset_type = "upload" + + self.log.debug( + "Family: {}\nMapping: {}".format(family_low, self.family_mapping) + ) # Ignore this instance if neither "ftrackFamily" or a family mapping is # found. From b9dc19a046f061c9b495fe39fb812154e8cdaf5b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 12:14:37 +0200 Subject: [PATCH 03/14] prepared functions to separate process function --- .../plugins/publish/integrate_ftrack_api.py | 346 ++++++++++++++++++ 1 file changed, 346 insertions(+) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py index 6c25b9191e..8ea2d8411b 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py @@ -464,3 +464,349 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): for asset_version in used_asset_versions: if asset_version not in instance.data[asset_versions_key]: instance.data[asset_versions_key].append(asset_version) + + def _ensure_asset_types_exists(self, session, component_list): + """Make sure that all AssetType entities exists for integration. + + Returns: + dict: All asset types by short name. + """ + # Query existing asset types + asset_types = session.query("select id, short from AssetType").all() + # Stpore all existing short names + asset_type_shorts = {asset_type["short"] for asset_type in asset_types} + # Check which asset types are missing and store them + asset_type_names_by_missing_shorts = {} + default_short_name = "upload" + for data in component_list: + asset_type_data = data.get("assettype_data") or {} + asset_type_short = asset_type_data.get("short") + if not asset_type_short: + # Use default asset type name if not set and change the + # input data + asset_type_short = default_short_name + asset_type_data["short"] = asset_type_short + data["assettype_data"] = asset_type_data + + if ( + # Skip if short name exists + asset_type_short in asset_type_shorts + # Skip if short name was already added to missing types + # and asset type name is filled + # - if asset type name is missing then try use name from other + # data + or asset_type_names_by_missing_shorts.get(asset_type_short) + ): + continue + + asset_type_names_by_missing_shorts[asset_type_short] = ( + asset_type_data.get("name") + ) + + # Create missing asset types if there are any + if asset_type_names_by_missing_shorts: + self.log.info("Creating asset types with short names: {}".format( + ", ".join(asset_type_names_by_missing_shorts.keys()) + )) + for missing_short, type_name in asset_type_names_by_missing_shorts: + # Use short for name if name is not defined + if not type_name: + type_name = missing_short + # Use short name also for name + # - there is not other source for 'name' + session.create( + "AssetType", + { + "short": missing_short, + "name": type_name + } + ) + + # Commit creation + session.commit() + # Requery asset types + asset_types = session.query( + "select id, short from AssetType" + ).all() + + return {asset_type["short"]: asset_type for asset_type in asset_types} + + def _ensure_asset_exists( + self, session, asset_data, asset_type_id, parent_id + ): + asset_name = asset_data["name"] + asset_entity = self._query_asset( + session, asset_name, asset_type_id, parent_id + ) + if asset_entity is not None: + return asset_entity + + asset_data = { + "name": asset_name, + "type_id": asset_type_id, + "context_id": parent_id + } + self.log.info("Created new Asset with data: {}.".format(asset_data)) + session.create("Asset", asset_data) + session.commit() + return self._query_asset(session, asset_name, asset_type_id, parent_id) + + def _query_asset(self, session, asset_name, asset_type_id, parent_id): + return session.query( + ( + "select id from Asset" + " where name is \"{}\"" + " and type_id is \"{}\"" + " and context_id is \"{}\"" + ).format(asset_name, asset_type_id, parent_id) + ).first() + + def _ensure_asset_version_exists( + self, session, asset_version_data, asset_id, task_entity + ): + task_id = None + if task_entity: + task_id = task_entity["id"] + + # Try query asset version by criteria (asset id and version) + version = asset_version_data.get("version") or 0 + asset_version_entity = self._query_asset_version( + session, version, asset_id + ) + + # Prepare comment value + comment = asset_version_data.get("comment") or "" + if asset_version_entity is not None: + changed = False + if comment != asset_version_entity["comment"]: + asset_version_entity["comment"] = comment + changed = True + + if task_id != asset_version_entity["task_id"]: + asset_version_entity["task_id"] = task_id + changed = True + + if changed: + session.commit() + + else: + new_asset_version_data = { + "version": version, + "asset_id": asset_id + } + if task_id: + new_asset_version_data["task_id"] = task_id + + if comment: + new_asset_version_data["comment"] = comment + + self.log.info("Created new AssetVersion with data {}".format( + new_asset_version_data + )) + session.create("AssetVersion", new_asset_version_data) + session.commit() + asset_version_entity = self._query_asset_version( + session, version, asset_id + ) + + # Set custom attributes if there were any set + custom_attrs = asset_version_data.get("custom_attributes") or {} + for attr_key, attr_value in custom_attrs.items(): + if attr_key in asset_version_entity["custom_attributes"]: + try: + asset_version_entity["custom_attributes"][attr_key] = ( + attr_value + ) + session.commit() + continue + except Exception: + session.rollback() + session._configure_locations() + + self.log.warning( + ( + "Custom Attrubute \"{0}\" is not available for" + " AssetVersion <{1}>. Can't set it's value to: \"{2}\"" + ).format( + attr_key, asset_version_entity["id"], str(attr_value) + ) + ) + + return asset_version_entity + + def _query_asset_version(self, session, version, asset_id): + return session.query( + ( + "select id, task_id, comment from AssetVersion" + " where version is \"{}\" and asset_id is \"{}\"" + ).format(version, asset_id) + ).first() + + def create_component(self, session, asset_version_entity, data): + component_data = data.get("component_data") or {} + + if not component_data.get("name"): + component_data["name"] = "main" + + version_id = asset_version_entity["id"] + component_data["version_id"] = version_id + component_entity = session.query( + ( + "select id, name from Component where name is \"{}\"" + " and version_id is \"{}\"" + ).format(component_data["name"], version_id) + ).first() + + component_overwrite = data.get("component_overwrite", False) + location = data.get("component_location", session.pick_location()) + + # Overwrite existing component data if requested. + if component_entity and component_overwrite: + origin_location = session.query( + "Location where name is \"ftrack.origin\"" + ).one() + + # Removing existing members from location + components = list(component_entity.get("members", [])) + components += [component_entity] + for component in components: + for loc in component["component_locations"]: + if location["id"] == loc["location_id"]: + location.remove_component( + component, recursive=False + ) + + # Deleting existing members on component entity + for member in component_entity.get("members", []): + session.delete(member) + del(member) + + try: + session.commit() + except Exception: + tp, value, tb = sys.exc_info() + session.rollback() + session._configure_locations() + six.reraise(tp, value, tb) + + # Reset members in memory + if "members" in component_entity.keys(): + component_entity["members"] = [] + + # Add components to origin location + try: + collection = clique.parse(data["component_path"]) + except ValueError: + # Assume its a single file + # Changing file type + name, ext = os.path.splitext(data["component_path"]) + component_entity["file_type"] = ext + + origin_location.add_component( + component_entity, data["component_path"] + ) + else: + # Changing file type + component_entity["file_type"] = collection.format("{tail}") + + # Create member components for sequence. + for member_path in collection: + + size = 0 + try: + size = os.path.getsize(member_path) + except OSError: + pass + + name = collection.match(member_path).group("index") + + member_data = { + "name": name, + "container": component_entity, + "size": size, + "file_type": os.path.splitext(member_path)[-1] + } + + component = session.create( + "FileComponent", member_data + ) + origin_location.add_component( + component, member_path, recursive=False + ) + component_entity["members"].append(component) + + # Add components to location. + location.add_component( + component_entity, origin_location, recursive=True + ) + + data["component"] = component_entity + self.log.info( + ( + "Overwriting Component with path: {0}, data: {1}," + " location: {2}" + ).format( + data["component_path"], + component_data, + location + ) + ) + + # Extracting metadata, and adding after entity creation. This is + # due to a ftrack_api bug where you can't add metadata on creation. + component_metadata = component_data.pop("metadata", {}) + + # Create new component if none exists. + new_component = False + if not component_entity: + component_entity = asset_version_entity.create_component( + data["component_path"], + data=component_data, + location=location + ) + data["component"] = component_entity + self.log.info( + ( + "Created new Component with path: {0}, data: {1}," + " metadata: {2}, location: {3}" + ).format( + data["component_path"], + component_data, + component_metadata, + location + ) + ) + new_component = True + + # Adding metadata + existing_component_metadata = component_entity["metadata"] + existing_component_metadata.update(component_metadata) + component_entity["metadata"] = existing_component_metadata + + # if component_data['name'] = 'ftrackreview-mp4-mp4': + # assetversion_entity["thumbnail_id"] + + # Setting assetversion thumbnail + if data.get("thumbnail"): + asset_version_entity["thumbnail_id"] = component_entity["id"] + + # Inform user about no changes to the database. + if ( + component_entity + and not component_overwrite + and not new_component + ): + data["component"] = component_entity + self.log.info( + "Found existing component, and no request to overwrite. " + "Nothing has been changed." + ) + else: + # Commit changes. + try: + session.commit() + except Exception: + tp, value, tb = sys.exc_info() + session.rollback() + session._configure_locations() + six.reraise(tp, value, tb) From ad4fe059e8b90d4e2bda589c319b3ed2e5d94812 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 12:15:40 +0200 Subject: [PATCH 04/14] simplified process function of integrate ftrack api --- .../plugins/publish/integrate_ftrack_api.py | 468 ++++-------------- 1 file changed, 103 insertions(+), 365 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py index 8ea2d8411b..7bba93c7cd 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py @@ -1,3 +1,15 @@ +"""Integrate components into ftrack + +Requires: + context -> ftrackSession - connected ftrack.Session + instance -> ftrackComponentsList - list of components to integrate + +Provides: + instance -> ftrackIntegratedAssetVersionsData + # legacy + instance -> ftrackIntegratedAssetVersions +""" + import os import sys import six @@ -54,6 +66,97 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): self.log.debug(query) return query + def process(self, instance): + session = instance.context.data["ftrackSession"] + context = instance.context + component_list = instance.data.get("ftrackComponentsList") + if not component_list: + self.log.info( + "Instance don't have components to integrate to Ftrack." + " Skipping." + ) + return + + session = instance.context.data["ftrackSession"] + context = instance.context + + parent_entity = None + default_asset_name = None + # If instance has set "ftrackEntity" or "ftrackTask" then use them from + # instance. Even if they are set to None. If they are set to None it + # has a reason. (like has different context) + if "ftrackEntity" in instance.data or "ftrackTask" in instance.data: + task_entity = instance.data.get("ftrackTask") + parent_entity = instance.data.get("ftrackEntity") + + elif "ftrackEntity" in context.data or "ftrackTask" in context.data: + task_entity = context.data.get("ftrackTask") + parent_entity = context.data.get("ftrackEntity") + + if task_entity: + default_asset_name = task_entity["name"] + parent_entity = task_entity["parent"] + + if parent_entity is None: + self.log.info(( + "Skipping ftrack integration. Instance \"{}\" does not" + " have specified ftrack entities." + ).format(str(instance))) + return + + if not default_asset_name: + default_asset_name = parent_entity["name"] + + # Change status on task + self._set_task_status(instance, task_entity, session) + + # Prepare AssetTypes + asset_types_by_short = self._ensure_asset_types_exists( + session, component_list + ) + + used_asset_versions = [] + # Iterate over components and publish + for data in component_list: + self.log.debug("data: {}".format(data)) + + # AssetType + asset_type_short = data["assettype_data"]["short"] + asset_type_entity = asset_types_by_short[asset_type_short] + + # Asset + asset_data = data.get("asset_data") or {} + if "name" not in asset_data: + asset_data["name"] = default_asset_name + asset_entity = self._ensure_asset_exists( + session, + asset_data, + asset_type_entity["id"], + parent_entity["id"] + ) + + # Asset Version + asset_version_data = data.get("assetversion_data") or {} + asset_version_entity = self._ensure_asset_version_exists( + session, asset_version_data, asset_entity["id"], task_entity + ) + + # Component + self.create_component(session, asset_version_entity, data) + + + # Backwards compatibility + if asset_version_entity not in used_asset_versions: + used_asset_versions.append(asset_version_entity) + + asset_versions_key = "ftrackIntegratedAssetVersions" + if asset_versions_key not in instance.data: + instance.data[asset_versions_key] = [] + + for asset_version in used_asset_versions: + if asset_version not in instance.data[asset_versions_key]: + instance.data[asset_versions_key].append(asset_version) + def _set_task_status(self, instance, task_entity, session): project_entity = instance.context.data.get("ftrackProject") if not project_entity: @@ -100,371 +203,6 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): session._configure_locations() six.reraise(tp, value, tb) - def process(self, instance): - session = instance.context.data["ftrackSession"] - context = instance.context - - name = None - # If instance has set "ftrackEntity" or "ftrackTask" then use them from - # instance. Even if they are set to None. If they are set to None it - # has a reason. (like has different context) - if "ftrackEntity" in instance.data or "ftrackTask" in instance.data: - task = instance.data.get("ftrackTask") - parent = instance.data.get("ftrackEntity") - - elif "ftrackEntity" in context.data or "ftrackTask" in context.data: - task = context.data.get("ftrackTask") - parent = context.data.get("ftrackEntity") - - if task: - parent = task["parent"] - name = task - elif parent: - name = parent["name"] - - if not name: - self.log.info(( - "Skipping ftrack integration. Instance \"{}\" does not" - " have specified ftrack entities." - ).format(str(instance))) - return - - info_msg = ( - "Created new {entity_type} with data: {data}" - ", metadata: {metadata}." - ) - - used_asset_versions = [] - - self._set_task_status(instance, task, session) - - # Iterate over components and publish - for data in instance.data.get("ftrackComponentsList", []): - # AssetType - # Get existing entity. - assettype_data = {"short": "upload"} - assettype_data.update(data.get("assettype_data", {})) - self.log.debug("data: {}".format(data)) - - assettype_entity = session.query( - self.query("AssetType", assettype_data) - ).first() - - # Create a new entity if none exits. - if not assettype_entity: - assettype_entity = session.create("AssetType", assettype_data) - self.log.debug("Created new AssetType with data: {}".format( - assettype_data - )) - - # Asset - # Get existing entity. - asset_data = { - "name": name, - "type": assettype_entity, - "parent": parent, - } - asset_data.update(data.get("asset_data", {})) - - asset_entity = session.query( - self.query("Asset", asset_data) - ).first() - - self.log.info("asset entity: {}".format(asset_entity)) - - # Extracting metadata, and adding after entity creation. This is - # due to a ftrack_api bug where you can't add metadata on creation. - asset_metadata = asset_data.pop("metadata", {}) - - # Create a new entity if none exits. - if not asset_entity: - asset_entity = session.create("Asset", asset_data) - self.log.debug( - info_msg.format( - entity_type="Asset", - data=asset_data, - metadata=asset_metadata - ) - ) - try: - session.commit() - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) - - # Adding metadata - existing_asset_metadata = asset_entity["metadata"] - existing_asset_metadata.update(asset_metadata) - asset_entity["metadata"] = existing_asset_metadata - - # AssetVersion - # Get existing entity. - assetversion_data = { - "version": 0, - "asset": asset_entity, - } - _assetversion_data = data.get("assetversion_data", {}) - assetversion_cust_attrs = _assetversion_data.pop( - "custom_attributes", {} - ) - asset_version_comment = _assetversion_data.pop( - "comment", None - ) - assetversion_data.update(_assetversion_data) - - assetversion_entity = session.query( - self.query("AssetVersion", assetversion_data) - ).first() - - # Extracting metadata, and adding after entity creation. This is - # due to a ftrack_api bug where you can't add metadata on creation. - assetversion_metadata = assetversion_data.pop("metadata", {}) - - if task: - assetversion_data['task'] = task - - # Create a new entity if none exits. - if not assetversion_entity: - assetversion_entity = session.create( - "AssetVersion", assetversion_data - ) - self.log.debug( - info_msg.format( - entity_type="AssetVersion", - data=assetversion_data, - metadata=assetversion_metadata - ) - ) - try: - session.commit() - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) - - # Adding metadata - existing_assetversion_metadata = assetversion_entity["metadata"] - existing_assetversion_metadata.update(assetversion_metadata) - assetversion_entity["metadata"] = existing_assetversion_metadata - - # Add comment - if asset_version_comment: - assetversion_entity["comment"] = asset_version_comment - try: - session.commit() - except Exception: - session.rollback() - session._configure_locations() - self.log.warning(( - "Comment was not possible to set for AssetVersion" - "\"{0}\". Can't set it's value to: \"{1}\"" - ).format( - assetversion_entity["id"], str(asset_version_comment) - )) - - # Adding Custom Attributes - for attr, val in assetversion_cust_attrs.items(): - if attr in assetversion_entity["custom_attributes"]: - try: - assetversion_entity["custom_attributes"][attr] = val - session.commit() - continue - except Exception: - session.rollback() - session._configure_locations() - - self.log.warning(( - "Custom Attrubute \"{0}\"" - " is not available for AssetVersion <{1}>." - " Can't set it's value to: \"{2}\"" - ).format(attr, assetversion_entity["id"], str(val))) - - # Have to commit the version and asset, because location can't - # determine the final location without. - try: - session.commit() - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) - - # Component - # Get existing entity. - component_data = { - "name": "main", - "version": assetversion_entity - } - component_data.update(data.get("component_data", {})) - - component_entity = session.query( - self.query("Component", component_data) - ).first() - - component_overwrite = data.get("component_overwrite", False) - location = data.get("component_location", session.pick_location()) - - # Overwrite existing component data if requested. - if component_entity and component_overwrite: - - origin_location = session.query( - "Location where name is \"ftrack.origin\"" - ).one() - - # Removing existing members from location - components = list(component_entity.get("members", [])) - components += [component_entity] - for component in components: - for loc in component["component_locations"]: - if location["id"] == loc["location_id"]: - location.remove_component( - component, recursive=False - ) - - # Deleting existing members on component entity - for member in component_entity.get("members", []): - session.delete(member) - del(member) - - try: - session.commit() - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) - - # Reset members in memory - if "members" in component_entity.keys(): - component_entity["members"] = [] - - # Add components to origin location - try: - collection = clique.parse(data["component_path"]) - except ValueError: - # Assume its a single file - # Changing file type - name, ext = os.path.splitext(data["component_path"]) - component_entity["file_type"] = ext - - origin_location.add_component( - component_entity, data["component_path"] - ) - else: - # Changing file type - component_entity["file_type"] = collection.format("{tail}") - - # Create member components for sequence. - for member_path in collection: - - size = 0 - try: - size = os.path.getsize(member_path) - except OSError: - pass - - name = collection.match(member_path).group("index") - - member_data = { - "name": name, - "container": component_entity, - "size": size, - "file_type": os.path.splitext(member_path)[-1] - } - - component = session.create( - "FileComponent", member_data - ) - origin_location.add_component( - component, member_path, recursive=False - ) - component_entity["members"].append(component) - - # Add components to location. - location.add_component( - component_entity, origin_location, recursive=True - ) - - data["component"] = component_entity - msg = "Overwriting Component with path: {0}, data: {1}, " - msg += "location: {2}" - self.log.info( - msg.format( - data["component_path"], - component_data, - location - ) - ) - - # Extracting metadata, and adding after entity creation. This is - # due to a ftrack_api bug where you can't add metadata on creation. - component_metadata = component_data.pop("metadata", {}) - - # Create new component if none exists. - new_component = False - if not component_entity: - component_entity = assetversion_entity.create_component( - data["component_path"], - data=component_data, - location=location - ) - data["component"] = component_entity - msg = "Created new Component with path: {0}, data: {1}" - msg += ", metadata: {2}, location: {3}" - self.log.info( - msg.format( - data["component_path"], - component_data, - component_metadata, - location - ) - ) - new_component = True - - # Adding metadata - existing_component_metadata = component_entity["metadata"] - existing_component_metadata.update(component_metadata) - component_entity["metadata"] = existing_component_metadata - - # if component_data['name'] = 'ftrackreview-mp4-mp4': - # assetversion_entity["thumbnail_id"] - - # Setting assetversion thumbnail - if data.get("thumbnail", False): - assetversion_entity["thumbnail_id"] = component_entity["id"] - - # Inform user about no changes to the database. - if (component_entity and not component_overwrite and - not new_component): - data["component"] = component_entity - self.log.info( - "Found existing component, and no request to overwrite. " - "Nothing has been changed." - ) - else: - # Commit changes. - try: - session.commit() - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) - - if assetversion_entity not in used_asset_versions: - used_asset_versions.append(assetversion_entity) - - asset_versions_key = "ftrackIntegratedAssetVersions" - if asset_versions_key not in instance.data: - instance.data[asset_versions_key] = [] - - for asset_version in used_asset_versions: - if asset_version not in instance.data[asset_versions_key]: - instance.data[asset_versions_key].append(asset_version) - def _ensure_asset_types_exists(self, session, component_list): """Make sure that all AssetType entities exists for integration. From 89d29a1d87ffe21e5d6d6d187ebb6e268bf289a1 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 12:16:03 +0200 Subject: [PATCH 05/14] store more data about asset versions in integrate ftrack api --- .../plugins/publish/integrate_ftrack_api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py index 7bba93c7cd..7ebf807f55 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_api.py @@ -115,6 +115,7 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): session, component_list ) + asset_versions_data_by_id = {} used_asset_versions = [] # Iterate over components and publish for data in component_list: @@ -144,11 +145,27 @@ class IntegrateFtrackApi(pyblish.api.InstancePlugin): # Component self.create_component(session, asset_version_entity, data) + # Store asset version and components items that were + version_id = asset_version_entity["id"] + if version_id not in asset_versions_data_by_id: + asset_versions_data_by_id[version_id] = { + "asset_version": asset_version_entity, + "component_items": [] + } + + asset_versions_data_by_id[version_id]["component_items"].append( + data + ) # Backwards compatibility if asset_version_entity not in used_asset_versions: used_asset_versions.append(asset_version_entity) + instance.data["ftrackIntegratedAssetVersionsData"] = ( + asset_versions_data_by_id + ) + + # Backwards compatibility asset_versions_key = "ftrackIntegratedAssetVersions" if asset_versions_key not in instance.data: instance.data[asset_versions_key] = [] From 4da7f7c1cce65434d385bf8cad5dc678226c62e4 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 13:48:21 +0200 Subject: [PATCH 06/14] changed integrate ftrack note to b able add published paths into asset version comment --- .../plugins/publish/integrate_ftrack_note.py | 196 +++++++++--------- .../defaults/project_settings/ftrack.json | 2 +- .../defaults/project_settings/global.json | 2 +- .../schema_project_ftrack.json | 8 +- 4 files changed, 111 insertions(+), 97 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index acd295854d..c165e99918 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -15,10 +15,112 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): # Can be set in presets: # - Allows only `intent` and `comment` keys + note_template = None + # Backwards compatibility note_with_intent_template = "{intent}: {comment}" # - note label must exist in Ftrack note_labels = [] + def process(self, instance): + # Check if there are any integrated AssetVersion entities + asset_versions_key = "ftrackIntegratedAssetVersionsData" + asset_versions_data_by_id = instance.data.get(asset_versions_key) + if not asset_versions_data_by_id: + self.log.info("There are any integrated AssetVersions") + return + + comment = (instance.context.data.get("comment") or "").strip() + if not comment: + self.log.info("Comment is not set.") + else: + self.log.debug("Comment is set to `{}`".format(comment)) + + session = instance.context.data["ftrackSession"] + + intent = instance.context.data.get("intent") + if intent and isinstance(intent, dict): + intent_val = intent.get("value") + intent_label = intent.get("label") + else: + intent_val = intent_label = intent + + final_intent_label = None + if intent_val: + final_intent_label = self.get_intent_label(session, intent_val) + if final_intent_label is None: + final_intent_label = intent_label + + # if intent label is set then format comment + # - it is possible that intent_label is equal to "" (empty string) + if final_intent_label: + self.log.debug( + "Intent label is set to `{}`.".format(final_intent_label) + ) + + elif intent_val: + self.log.debug(( + "Intent is set to `{}` and was not added" + " to comment because label is set to `{}`." + ).format(intent_val, final_intent_label)) + + else: + self.log.debug("Intent is not set.") + + user = session.query( + "User where username is \"{}\"".format(session.api_user) + ).first() + if not user: + self.log.warning( + "Was not able to query current User {}".format( + session.api_user + ) + ) + + labels = [] + if self.note_labels: + all_labels = session.query("select id, name from NoteLabel").all() + labels_by_low_name = {lab["name"].lower(): lab for lab in all_labels} + for _label in self.note_labels: + label = labels_by_low_name.get(_label.lower()) + if not label: + self.log.warning( + "Note Label `{}` was not found.".format(_label) + ) + continue + + labels.append(label) + + for asset_version_data in asset_versions_data_by_id.values(): + asset_version = asset_version_data["asset_version"] + component_items = asset_version_data["component_items"] + + published_paths = set() + for component_item in component_items: + published_paths.add(component_item["component_path"]) + + # Backwards compatibility for older settings using + # attribute 'note_with_intent_template' + template = self.note_template + if template is None: + template = self.note_with_intent_template + comment = template.format(**{ + "intent": final_intent_label, + "comment": comment, + "published_paths": "\n".join(sorted(published_paths)) + }) + asset_version.create_note(comment, author=user, labels=labels) + + try: + session.commit() + self.log.debug("Note added to AssetVersion \"{}\"".format( + str(asset_version) + )) + except Exception: + tp, value, tb = sys.exc_info() + session.rollback() + session._configure_locations() + six.reraise(tp, value, tb) + def get_intent_label(self, session, intent_value): if not intent_value: return @@ -45,12 +147,7 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): if not items: return - if sys.version_info[0] < 3: - string_type = basestring - else: - string_type = str - - if isinstance(items, string_type): + if isinstance(items, six.string_types): items = json.loads(items) intent_label = None @@ -60,90 +157,3 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): break return intent_label - - def process(self, instance): - comment = (instance.context.data.get("comment") or "").strip() - if not comment: - self.log.info("Comment is not set.") - return - - self.log.debug("Comment is set to `{}`".format(comment)) - - session = instance.context.data["ftrackSession"] - - intent = instance.context.data.get("intent") - if intent and isinstance(intent, dict): - intent_val = intent.get("value") - intent_label = intent.get("label") - else: - intent_val = intent_label = intent - - final_label = None - if intent_val: - final_label = self.get_intent_label(session, intent_val) - if final_label is None: - final_label = intent_label - - # if intent label is set then format comment - # - it is possible that intent_label is equal to "" (empty string) - if final_label: - msg = "Intent label is set to `{}`.".format(final_label) - comment = self.note_with_intent_template.format(**{ - "intent": final_label, - "comment": comment - }) - - elif intent_val: - msg = ( - "Intent is set to `{}` and was not added" - " to comment because label is set to `{}`." - ).format(intent_val, final_label) - - else: - msg = "Intent is not set." - - self.log.debug(msg) - - asset_versions_key = "ftrackIntegratedAssetVersions" - asset_versions = instance.data.get(asset_versions_key) - if not asset_versions: - self.log.info("There are any integrated AssetVersions") - return - - user = session.query( - "User where username is \"{}\"".format(session.api_user) - ).first() - if not user: - self.log.warning( - "Was not able to query current User {}".format( - session.api_user - ) - ) - - labels = [] - if self.note_labels: - all_labels = session.query("NoteLabel").all() - labels_by_low_name = {lab["name"].lower(): lab for lab in all_labels} - for _label in self.note_labels: - label = labels_by_low_name.get(_label.lower()) - if not label: - self.log.warning( - "Note Label `{}` was not found.".format(_label) - ) - continue - - labels.append(label) - - for asset_version in asset_versions: - asset_version.create_note(comment, author=user, labels=labels) - - try: - session.commit() - self.log.debug("Note added to AssetVersion \"{}\"".format( - str(asset_version) - )) - except Exception: - tp, value, tb = sys.exc_info() - session.rollback() - session._configure_locations() - six.reraise(tp, value, tb) diff --git a/openpype/settings/defaults/project_settings/ftrack.json b/openpype/settings/defaults/project_settings/ftrack.json index ca1cfe1e12..9b350ec88d 100644 --- a/openpype/settings/defaults/project_settings/ftrack.json +++ b/openpype/settings/defaults/project_settings/ftrack.json @@ -354,7 +354,7 @@ }, "IntegrateFtrackNote": { "enabled": true, - "note_with_intent_template": "{intent}: {comment}", + "note_template": "{intent}: {comment}", "note_labels": [] }, "ValidateFtrackAttributes": { diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index ed28d357f2..4c94eee254 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -190,7 +190,7 @@ "tasks": [], "template_name": "simpleUnrealTexture" }, - { + { "families": [ "staticMesh", "skeletalMesh" diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json index fb384882c6..2b62d67c98 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json @@ -738,10 +738,14 @@ "key": "enabled", "label": "Enabled" }, + { + "type": "label", + "label": "Template may contain formatting keys {intent}, {comment} and {published_paths}." + }, { "type": "text", - "key": "note_with_intent_template", - "label": "Note with intent template" + "key": "note_template", + "label": "Note template" }, { "type": "list", From 8167a65f597b6526e69a66a5591f86c2b5c8aa54 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 15:26:53 +0200 Subject: [PATCH 07/14] added more checks --- .../plugins/publish/integrate_ftrack_note.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index c165e99918..a77b6d6674 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -47,8 +47,9 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): final_intent_label = None if intent_val: final_intent_label = self.get_intent_label(session, intent_val) - if final_intent_label is None: - final_intent_label = intent_label + + if final_intent_label is None: + final_intent_label = intent_label # if intent label is set then format comment # - it is possible that intent_label is equal to "" (empty string) @@ -103,11 +104,18 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): template = self.note_template if template is None: template = self.note_with_intent_template - comment = template.format(**{ + format_data = { "intent": final_intent_label, "comment": comment, "published_paths": "\n".join(sorted(published_paths)) - }) + } + comment = template.format(**format_data) + if not comment: + self.log.info(( + "Note for AssetVersion {} would be empty. Skipping." + "\nTemplate: {}\nData: {}" + ).format(asset_version["id"], template, format_data)) + continue asset_version.create_note(comment, author=user, labels=labels) try: From fe39d0a300fd1ad76390f152cd8502a6f798c244 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 15:39:50 +0200 Subject: [PATCH 08/14] add host name to possible keys in template --- .../modules/ftrack/plugins/publish/integrate_ftrack_note.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index a77b6d6674..8609e8bca6 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -29,6 +29,7 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): self.log.info("There are any integrated AssetVersions") return + host_name = instance.context.data["hostName"] comment = (instance.context.data.get("comment") or "").strip() if not comment: self.log.info("Comment is not set.") @@ -107,7 +108,8 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): format_data = { "intent": final_intent_label, "comment": comment, - "published_paths": "\n".join(sorted(published_paths)) + "host_name": host_name, + "published_paths": "\n".join(sorted(published_paths)), } comment = template.format(**format_data) if not comment: From 04b8eaf2998eb3269800df0d8fa4985a4a6a8df0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 15:44:30 +0200 Subject: [PATCH 09/14] added new key into settings label --- .../entities/schemas/projects_schema/schema_project_ftrack.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json index 2b62d67c98..b3c094e398 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json @@ -740,7 +740,7 @@ }, { "type": "label", - "label": "Template may contain formatting keys {intent}, {comment} and {published_paths}." + "label": "Template may contain formatting keys {intent}, {comment}, {host_name} and {published_paths}." }, { "type": "text", From 00406a737610dc4623ff78edda1e7369ddbfc3f0 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 16:13:50 +0200 Subject: [PATCH 10/14] added more information abou app - name and label --- .../plugins/publish/collect_app_name.py | 13 +++++++++ .../plugins/publish/collect_app_name.py | 13 +++++++++ openpype/plugins/publish/collect_host_name.py | 28 +++++++++++++------ 3 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py create mode 100644 openpype/hosts/traypublisher/plugins/publish/collect_app_name.py diff --git a/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py b/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py new file mode 100644 index 0000000000..857f3dca20 --- /dev/null +++ b/openpype/hosts/standalonepublisher/plugins/publish/collect_app_name.py @@ -0,0 +1,13 @@ +import pyblish.api + + +class CollectSAAppName(pyblish.api.ContextPlugin): + """Collect app name and label.""" + + label = "Collect App Name/Label" + order = pyblish.api.CollectorOrder - 0.5 + hosts = ["standalonepublisher"] + + def process(self, context): + context.data["appName"] = "standalone publisher" + context.data["appLabel"] = "Standalone publisher" diff --git a/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py b/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py new file mode 100644 index 0000000000..e38d10e70f --- /dev/null +++ b/openpype/hosts/traypublisher/plugins/publish/collect_app_name.py @@ -0,0 +1,13 @@ +import pyblish.api + + +class CollectTrayPublisherAppName(pyblish.api.ContextPlugin): + """Collect app name and label.""" + + label = "Collect App Name/Label" + order = pyblish.api.CollectorOrder - 0.5 + hosts = ["traypublisher"] + + def process(self, context): + context.data["appName"] = "tray publisher" + context.data["appLabel"] = "Tray publisher" diff --git a/openpype/plugins/publish/collect_host_name.py b/openpype/plugins/publish/collect_host_name.py index b731e3ed26..d64af4d049 100644 --- a/openpype/plugins/publish/collect_host_name.py +++ b/openpype/plugins/publish/collect_host_name.py @@ -18,20 +18,30 @@ class CollectHostName(pyblish.api.ContextPlugin): def process(self, context): host_name = context.data.get("hostName") + app_name = context.data.get("appName") + app_label = context.data.get("appLabel") # Don't override value if is already set - if host_name: + if host_name and app_name and app_label: return - # Use AVALON_APP as first if available it is the same as host name - # - only if is not defined use AVALON_APP_NAME (e.g. on Farm) and - # set it back to AVALON_APP env variable - host_name = os.environ.get("AVALON_APP") + # Use AVALON_APP to get host name if available if not host_name: + host_name = os.environ.get("AVALON_APP") + + # Use AVALON_APP_NAME to get full app name + if not app_name: app_name = os.environ.get("AVALON_APP_NAME") - if app_name: - app_manager = ApplicationManager() - app = app_manager.applications.get(app_name) - if app: + + # Fill missing values based on app full name + if (not host_name or not app_label) and app_name: + app_manager = ApplicationManager() + app = app_manager.applications.get(app_name) + if app: + if not host_name: host_name = app.host_name + if not app_label: + app_label = app.full_label context.data["hostName"] = host_name + context.data["appName"] = app_name + context.data["appLabel"] = app_label From e68a7cbf7bb9c4362bd5c64ad617ec38db9c633b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 16:15:40 +0200 Subject: [PATCH 11/14] added app name and lable keys into note template --- .../modules/ftrack/plugins/publish/integrate_ftrack_note.py | 4 ++++ .../schemas/projects_schema/schema_project_ftrack.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index 8609e8bca6..8220b8e6ca 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -30,6 +30,8 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): return host_name = instance.context.data["hostName"] + app_name = instance.context.data["appName"] + app_label = instance.context.data["appLabel"] comment = (instance.context.data.get("comment") or "").strip() if not comment: self.log.info("Comment is not set.") @@ -109,6 +111,8 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): "intent": final_intent_label, "comment": comment, "host_name": host_name, + "app_name": app_name, + "app_label": app_label, "published_paths": "\n".join(sorted(published_paths)), } comment = template.format(**format_data) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json index b3c094e398..0d7faac2ba 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json @@ -740,7 +740,7 @@ }, { "type": "label", - "label": "Template may contain formatting keys {intent}, {comment}, {host_name} and {published_paths}." + "label": "Template may contain formatting keys intent, comment, host_name, app_name, app_label and published_paths." }, { "type": "text", From 2a3460e0fa57408356a2af0ebfcba33ad83f4132 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 16:15:55 +0200 Subject: [PATCH 12/14] changed note template input into multiline input --- .../schemas/projects_schema/schema_project_ftrack.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json index 0d7faac2ba..0ed2fb3536 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_ftrack.json @@ -745,7 +745,8 @@ { "type": "text", "key": "note_template", - "label": "Note template" + "label": "Note template", + "multiline": true }, { "type": "list", From 2e9b7325e47c30a75850c048669c2e04d8bbf3de Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 16:25:27 +0200 Subject: [PATCH 13/14] change new line char with br/ html tag --- .../modules/ftrack/plugins/publish/integrate_ftrack_note.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index 8220b8e6ca..3a9f904d00 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -113,7 +113,7 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): "host_name": host_name, "app_name": app_name, "app_label": app_label, - "published_paths": "\n".join(sorted(published_paths)), + "published_paths": "
".join(sorted(published_paths)), } comment = template.format(**format_data) if not comment: From dff1a51f96a1237071e2fe2ed3cac400c7a3bf6e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Wed, 6 Apr 2022 16:33:05 +0200 Subject: [PATCH 14/14] removed outdated log --- .../modules/ftrack/plugins/publish/integrate_ftrack_note.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py index 3a9f904d00..56a7a89e16 100644 --- a/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py +++ b/openpype/modules/ftrack/plugins/publish/integrate_ftrack_note.py @@ -61,12 +61,6 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): "Intent label is set to `{}`.".format(final_intent_label) ) - elif intent_val: - self.log.debug(( - "Intent is set to `{}` and was not added" - " to comment because label is set to `{}`." - ).format(intent_val, final_intent_label)) - else: self.log.debug("Intent is not set.")