TrayPublisher & StandalonePublisher: Specify version (#5142)

* modified simple creator plugin to be able handle version control

* added 'allow_version_control' to simple creators

* don't remove 'create_context' from pyblish context during publishing

* implemented validator for existing version override

* actually fill version on collected instances

* version can be again changed from standalone publisher

* added comment to collector

* make sure the version is set always to int

* removed unused import

* disable validator if is disabled

* fix filtered instances loop

openpype/hosts/standalonepublisher/plugins/publish/collect_context.py

@@ -222,7 +222,6 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin):
                 "label": subset,
                 "name": subset,
                 "family": in_data["family"],
-                # "version": in_data.get("version", 1),
                 "frameStart": in_data.get("representations", [None])[0].get(
                     "frameStart", None
                 ),
@@ -232,6 +231,14 @@ class CollectContextDataSAPublish(pyblish.api.ContextPlugin):
                 "families": instance_families
             }
         )
+        # Fill version only if 'use_next_available_version' is disabled
+        #   and version is filled in instance data
+        version = in_data.get("version")
+        use_next_available_version = in_data.get(
+            "use_next_available_version", True)
+        if not use_next_available_version and version is not None:
+            instance.data["version"] = version
+
         self.log.info("collected instance: {}".format(pformat(instance.data)))
         self.log.info("parsing data: {}".format(pformat(in_data)))
 

openpype/hosts/traypublisher/api/plugin.py

@@ -1,4 +1,14 @@
-from openpype.lib.attribute_definitions import FileDef
+from openpype.client import (
+    get_assets,
+    get_subsets,
+    get_last_versions,
+)
+from openpype.lib.attribute_definitions import (
+    FileDef,
+    BoolDef,
+    NumberDef,
+    UISeparatorDef,
+)
 from openpype.lib.transcoding import IMAGE_EXTENSIONS, VIDEO_EXTENSIONS
 from openpype.pipeline.create import (
     Creator,
@@ -94,6 +104,7 @@ class TrayPublishCreator(Creator):
 class SettingsCreator(TrayPublishCreator):
     create_allow_context_change = True
     create_allow_thumbnail = True
+    allow_version_control = False
 
     extensions = []
 
@@ -101,8 +112,18 @@ class SettingsCreator(TrayPublishCreator):
         # Pass precreate data to creator attributes
         thumbnail_path = pre_create_data.pop(PRE_CREATE_THUMBNAIL_KEY, None)
 
+        # Fill 'version_to_use' if version control is enabled
+        if self.allow_version_control:
+            asset_name = data["asset"]
+            subset_docs_by_asset_id = self._prepare_next_versions(
+                [asset_name], [subset_name])
+            version = subset_docs_by_asset_id[asset_name].get(subset_name)
+            pre_create_data["version_to_use"] = version
+            data["_previous_last_version"] = version
+
         data["creator_attributes"] = pre_create_data
         data["settings_creator"] = True
+
         # Create new instance
         new_instance = CreatedInstance(self.family, subset_name, data, self)
 
@@ -111,7 +132,158 @@ class SettingsCreator(TrayPublishCreator):
         if thumbnail_path:
             self.set_instance_thumbnail_path(new_instance.id, thumbnail_path)
 
+    def _prepare_next_versions(self, asset_names, subset_names):
+        """Prepare next versions for given asset and subset names.
+
+        Todos:
+            Expect combination of subset names by asset name to avoid
+                unnecessary server calls for unused subsets.
+
+        Args:
+            asset_names (Iterable[str]): Asset names.
+            subset_names (Iterable[str]): Subset names.
+
+        Returns:
+            dict[str, dict[str, int]]: Last versions by asset
+                and subset names.
+        """
+
+        # Prepare all versions for all combinations to '1'
+        subset_docs_by_asset_id = {
+            asset_name: {
+                subset_name: 1
+                for subset_name in subset_names
+            }
+            for asset_name in asset_names
+        }
+        if not asset_names or not subset_names:
+            return subset_docs_by_asset_id
+
+        asset_docs = get_assets(
+            self.project_name,
+            asset_names=asset_names,
+            fields=["_id", "name"]
+        )
+        asset_names_by_id = {
+            asset_doc["_id"]: asset_doc["name"]
+            for asset_doc in asset_docs
+        }
+        subset_docs = list(get_subsets(
+            self.project_name,
+            asset_ids=asset_names_by_id.keys(),
+            subset_names=subset_names,
+            fields=["_id", "name", "parent"]
+        ))
+
+        subset_ids = {subset_doc["_id"] for subset_doc in subset_docs}
+        last_versions = get_last_versions(
+            self.project_name,
+            subset_ids,
+            fields=["name", "parent"])
+
+        for subset_doc in subset_docs:
+            asset_id = subset_doc["parent"]
+            asset_name = asset_names_by_id[asset_id]
+            subset_name = subset_doc["name"]
+            subset_id = subset_doc["_id"]
+            last_version = last_versions.get(subset_id)
+            version = 0
+            if last_version is not None:
+                version = last_version["name"]
+            subset_docs_by_asset_id[asset_name][subset_name] += version
+        return subset_docs_by_asset_id
+
+    def _fill_next_versions(self, instances_data):
+        """Fill next version for instances.
+
+        Instances have also stored previous next version to be able to
+        recognize if user did enter different version. If version was
+        not changed by user, or user set it to '0' the next version will be
+        updated by current database state.
+        """
+
+        filtered_instance_data = []
+        for instance in instances_data:
+            previous_last_version = instance.get("_previous_last_version")
+            creator_attributes = instance["creator_attributes"]
+            use_next_version = creator_attributes.get(
+                "use_next_version", True)
+            version = creator_attributes.get("version_to_use", 0)
+            if (
+                use_next_version
+                or version == 0
+                or version == previous_last_version
+            ):
+                filtered_instance_data.append(instance)
+
+        asset_names = {
+            instance["asset"]
+            for instance in filtered_instance_data}
+        subset_names = {
+            instance["subset"]
+            for instance in filtered_instance_data}
+        subset_docs_by_asset_id = self._prepare_next_versions(
+            asset_names, subset_names
+        )
+        for instance in filtered_instance_data:
+            asset_name = instance["asset"]
+            subset_name = instance["subset"]
+            version = subset_docs_by_asset_id[asset_name][subset_name]
+            instance["creator_attributes"]["version_to_use"] = version
+            instance["_previous_last_version"] = version
+
+    def collect_instances(self):
+        """Collect instances from host.
+
+        Overriden to be able to manage version control attributes. If version
+        control is disabled, the attributes will be removed from instances,
+        and next versions are filled if is version control enabled.
+        """
+
+        instances_by_identifier = cache_and_get_instances(
+            self, SHARED_DATA_KEY, list_instances
+        )
+        instances = instances_by_identifier[self.identifier]
+        if not instances:
+            return
+
+        if self.allow_version_control:
+            self._fill_next_versions(instances)
+
+        for instance_data in instances:
+            # Make sure that there are not data related to version control
+            #   if plugin does not support it
+            if not self.allow_version_control:
+                instance_data.pop("_previous_last_version", None)
+                creator_attributes = instance_data["creator_attributes"]
+                creator_attributes.pop("version_to_use", None)
+                creator_attributes.pop("use_next_version", None)
+
+            instance = CreatedInstance.from_existing(instance_data, self)
+            self._add_instance_to_context(instance)
+
     def get_instance_attr_defs(self):
+        defs = self.get_pre_create_attr_defs()
+        if self.allow_version_control:
+            defs += [
+                UISeparatorDef(),
+                BoolDef(
+                    "use_next_version",
+                    default=True,
+                    label="Use next version",
+                ),
+                NumberDef(
+                    "version_to_use",
+                    default=1,
+                    minimum=0,
+                    maximum=999,
+                    label="Version to use",
+                )
+            ]
+        return defs
+
+    def get_pre_create_attr_defs(self):
+        # Use same attributes as for instance attributes
         return [
             FileDef(
                 "representation_files",
@@ -132,10 +304,6 @@ class SettingsCreator(TrayPublishCreator):
             )
         ]
 
-    def get_pre_create_attr_defs(self):
-        # Use same attributes as for instance attrobites
-        return self.get_instance_attr_defs()
-
     @classmethod
     def from_settings(cls, item_data):
         identifier = item_data["identifier"]
@@ -155,6 +323,8 @@ class SettingsCreator(TrayPublishCreator):
                 "extensions": item_data["extensions"],
                 "allow_sequences": item_data["allow_sequences"],
                 "allow_multiple_items": item_data["allow_multiple_items"],
-                "default_variants": item_data["default_variants"]
+                "allow_version_control": item_data.get(
+                    "allow_version_control", False),
+                "default_variants": item_data["default_variants"],
             }
         )

openpype/hosts/traypublisher/plugins/publish/collect_simple_instances.py

@@ -47,6 +47,8 @@ class CollectSettingsSimpleInstances(pyblish.api.InstancePlugin):
             "Created temp staging directory for instance {}. {}"
         ).format(instance_label, tmp_folder))
 
+        self._fill_version(instance, instance_label)
+
         # Store filepaths for validation of their existence
         source_filepaths = []
         # Make sure there are no representations with same name
@@ -93,6 +95,28 @@ class CollectSettingsSimpleInstances(pyblish.api.InstancePlugin):
             )
         )
 
+    def _fill_version(self, instance, instance_label):
+        """Fill instance version under which will be instance integrated.
+
+        Instance must have set 'use_next_version' to 'False'
+        and 'version_to_use' to version to use.
+
+        Args:
+            instance (pyblish.api.Instance): Instance to fill version for.
+            instance_label (str): Label of instance to fill version for.
+        """
+
+        creator_attributes = instance.data["creator_attributes"]
+        use_next_version = creator_attributes.get("use_next_version", True)
+        # If 'version_to_use' is '0' it means that next version should be used
+        version_to_use = creator_attributes.get("version_to_use", 0)
+        if use_next_version or not version_to_use:
+            return
+        instance.data["version"] = version_to_use
+        self.log.debug(
+            "Version for instance \"{}\" was set to \"{}\"".format(
+                instance_label, version_to_use))
+
     def _create_main_representations(
         self,
         instance,

openpype/hosts/traypublisher/plugins/publish/help/validate_existing_version.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<root>
+<error id="main">
+<title>Version already exists</title>
+<description>
+## Version already exists
+
+Version {version} you have set on instance '{subset_name}' under '{asset_name}' already exists. This validation is enabled by default to prevent accidental override of existing versions.
+
+### How to repair?
+- Click on 'Repair' action -> this will change version to next available.
+- Disable validation on the instance if you are sure you want to override the version.
+- Reset publishing and manually change the version number.
+</description>
+</error>
+</root>

openpype/hosts/traypublisher/plugins/publish/validate_existing_version.py

@@ -0,0 +1,57 @@
+import pyblish.api
+
+from openpype.pipeline.publish import (
+    ValidateContentsOrder,
+    PublishXmlValidationError,
+    OptionalPyblishPluginMixin,
+    RepairAction,
+)
+
+
+class ValidateExistingVersion(
+    OptionalPyblishPluginMixin,
+    pyblish.api.InstancePlugin
+):
+    label = "Validate Existing Version"
+    order = ValidateContentsOrder
+
+    hosts = ["traypublisher"]
+
+    actions = [RepairAction]
+
+    settings_category = "traypublisher"
+    optional = True
+
+    def process(self, instance):
+        if not self.is_active(instance.data):
+            return
+
+        version = instance.data.get("version")
+        if version is None:
+            return
+
+        last_version = instance.data.get("latestVersion")
+        if last_version is None or last_version < version:
+            return
+
+        subset_name = instance.data["subset"]
+        msg = "Version {} already exists for subset {}.".format(
+            version, subset_name)
+
+        formatting_data = {
+            "subset_name": subset_name,
+            "asset_name": instance.data["asset"],
+            "version": version
+        }
+        raise PublishXmlValidationError(
+            self, msg, formatting_data=formatting_data)
+
+    @classmethod
+    def repair(cls, instance):
+        create_context = instance.context.data["create_context"]
+        created_instance = create_context.get_instance_by_id(
+            instance.data["instance_id"])
+        creator_attributes = created_instance["creator_attributes"]
+        # Disable version override
+        creator_attributes["use_next_version"] = True
+        create_context.save_changes()

openpype/pipeline/create/context.py

@@ -1441,6 +1441,19 @@ class CreateContext:
         """Access to global publish attributes."""
         return self._publish_attributes
 
+    def get_instance_by_id(self, instance_id):
+        """Receive instance by id.
+
+        Args:
+            instance_id (str): Instance id.
+
+        Returns:
+            Union[CreatedInstance, None]: Instance or None if instance with
+                given id is not available.
+        """
+
+        return self._instances_by_id.get(instance_id)
+
     def get_sorted_creators(self, identifiers=None):
         """Sorted creators by 'order' attribute.
 

openpype/plugins/publish/collect_from_create_context.py

@@ -16,7 +16,7 @@ class CollectFromCreateContext(pyblish.api.ContextPlugin):
     order = pyblish.api.CollectorOrder - 0.5
 
     def process(self, context):
-        create_context = context.data.pop("create_context", None)
+        create_context = context.data.get("create_context")
         if not create_context:
             host = registered_host()
             if isinstance(host, IPublishHost):

openpype/settings/defaults/project_settings/traypublisher.json

@@ -23,6 +23,7 @@
             "detailed_description": "Workfiles are full scenes from any application that are directly edited by artists. They represent a state of work on a task at a given point and are usually not directly referenced into other scenes.",
             "allow_sequences": false,
             "allow_multiple_items": false,
+            "allow_version_control": false,
             "extensions": [
                 ".ma",
                 ".mb",
@@ -57,6 +58,7 @@
             "detailed_description": "Models should only contain geometry data, without any extras like cameras, locators or bones.\n\nKeep in mind that models published from tray publisher are not validated for correctness. ",
             "allow_sequences": false,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".ma",
                 ".mb",
@@ -82,6 +84,7 @@
             "detailed_description": "Alembic or bgeo cache of animated data",
             "allow_sequences": true,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".abc",
                 ".bgeo",
@@ -105,6 +108,7 @@
             "detailed_description": "Any type of image seqeuence coming from outside of the studio. Usually camera footage, but could also be animatics used for reference.",
             "allow_sequences": true,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".exr",
                 ".png",
@@ -127,6 +131,7 @@
             "detailed_description": "Sequence or single file renders",
             "allow_sequences": true,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".exr",
                 ".png",
@@ -150,6 +155,7 @@
             "detailed_description": "Ideally this should be only camera itself with baked animation, however, it can technically also include helper geometry.",
             "allow_sequences": false,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".abc",
                 ".ma",
@@ -174,6 +180,7 @@
             "detailed_description": "Any image data can be published as image family. References, textures, concept art, matte paints. This is a fallback 2d family for everything that doesn't fit more specific family.",
             "allow_sequences": false,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".exr",
                 ".jpg",
@@ -197,6 +204,7 @@
             "detailed_description": "Hierarchical data structure for the efficient storage and manipulation of sparse volumetric data discretized on three-dimensional grids",
             "allow_sequences": true,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": [
                 ".vdb"
             ]
@@ -215,6 +223,7 @@
             "detailed_description": "Script exported from matchmoving application to be later processed into a tracked camera with additional data",
             "allow_sequences": false,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": []
         },
         {
@@ -227,6 +236,7 @@
             "detailed_description": "CG rigged character or prop. Rig should be clean of any extra data and directly loadable into it's respective application\t",
             "allow_sequences": false,
             "allow_multiple_items": false,
+            "allow_version_control": false,
             "extensions": [
                 ".ma",
                 ".blend",
@@ -244,6 +254,7 @@
             "detailed_description": "Texture files with Unreal Engine naming conventions",
             "allow_sequences": false,
             "allow_multiple_items": true,
+            "allow_version_control": false,
             "extensions": []
         }
     ],
@@ -322,6 +333,11 @@
             "enabled": true,
             "optional": true,
             "active": true
+        },
+        "ValidateExistingVersion": {
+            "enabled": true,
+            "optional": true,
+            "active": true
         }
     }
 }

openpype/settings/entities/schemas/projects_schema/schema_project_traypublisher.json

@@ -85,6 +85,12 @@
                         "label": "Allow multiple items",
                         "type": "boolean"
                     },
+                    {
+                        "type": "boolean",
+                        "key": "allow_version_control",
+                        "label": "Allow version control",
+                        "default": false
+                    },
                     {
                         "type": "list",
                         "key": "extensions",
@@ -346,6 +352,10 @@
                         {
                             "key": "ValidateFrameRange",
                             "label": "Validate frame range"
+                        },
+                        {
+                            "key": "ValidateExistingVersion",
+                            "label": "Validate Existing Version"
                         }
                     ]
                 }

openpype/tools/standalonepublish/widgets/widget_family.py

@@ -128,7 +128,8 @@ class FamilyWidget(QtWidgets.QWidget):
             'family_preset_key': key,
             'family': family,
             'subset': self.input_result.text(),
-            'version': self.version_spinbox.value()
+            'version': self.version_spinbox.value(),
+            'use_next_available_version': self.version_checkbox.isChecked(),
         }
         return data