From 631942327edc0469834c5cbb2e328a4caae0586e Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Tue, 4 Feb 2020 10:43:51 +0000 Subject: [PATCH 001/118] Creation and publishing of a rig in Blender --- pype/blender/plugin.py | 7 +++ pype/plugins/blender/create/create_rig.py | 32 +++++++++++++ pype/plugins/blender/publish/collect_rig.py | 53 +++++++++++++++++++++ pype/plugins/blender/publish/extract_rig.py | 47 ++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 pype/plugins/blender/create/create_rig.py create mode 100644 pype/plugins/blender/publish/collect_rig.py create mode 100644 pype/plugins/blender/publish/extract_rig.py diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index ad5a259785..eaa429c989 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -17,6 +17,13 @@ def model_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: name = f"{namespace}:{name}" return name +def rig_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: + """Return a consistent name for a rig asset.""" + name = f"{asset}_{subset}" + if namespace: + name = f"{namespace}:{name}" + return name + class AssetLoader(api.Loader): """A basic AssetLoader for Blender diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py new file mode 100644 index 0000000000..01eb524eef --- /dev/null +++ b/pype/plugins/blender/create/create_rig.py @@ -0,0 +1,32 @@ +"""Create a rig asset.""" + +import bpy + +from avalon import api +from avalon.blender import Creator, lib + + +class CreateRig(Creator): + """Artist-friendly rig with controls to direct motion""" + + name = "rigMain" + label = "Rig" + family = "rig" + icon = "cube" + + def process(self): + import pype.blender + + asset = self.data["asset"] + subset = self.data["subset"] + name = pype.blender.plugin.rig_name(asset, subset) + collection = bpy.data.collections.new(name=name) + bpy.context.scene.collection.children.link(collection) + self.data['task'] = api.Session.get('AVALON_TASK') + lib.imprint(collection, self.data) + + if (self.options or {}).get("useSelection"): + for obj in lib.get_selection(): + collection.objects.link(obj) + + return collection diff --git a/pype/plugins/blender/publish/collect_rig.py b/pype/plugins/blender/publish/collect_rig.py new file mode 100644 index 0000000000..a4b30541f6 --- /dev/null +++ b/pype/plugins/blender/publish/collect_rig.py @@ -0,0 +1,53 @@ +import typing +from typing import Generator + +import bpy + +import avalon.api +import pyblish.api +from avalon.blender.pipeline import AVALON_PROPERTY + + +class CollectRig(pyblish.api.ContextPlugin): + """Collect the data of a rig.""" + + hosts = ["blender"] + label = "Collect Rig" + order = pyblish.api.CollectorOrder + + @staticmethod + def get_rig_collections() -> Generator: + """Return all 'rig' collections. + + Check if the family is 'rig' and if it doesn't have the + representation set. If the representation is set, it is a loaded rig + and we don't want to publish it. + """ + for collection in bpy.data.collections: + avalon_prop = collection.get(AVALON_PROPERTY) or dict() + if (avalon_prop.get('family') == 'rig' + and not avalon_prop.get('representation')): + yield collection + + def process(self, context): + """Collect the rigs from the current Blender scene.""" + collections = self.get_rig_collections() + for collection in collections: + avalon_prop = collection[AVALON_PROPERTY] + asset = avalon_prop['asset'] + family = avalon_prop['family'] + subset = avalon_prop['subset'] + task = avalon_prop['task'] + name = f"{asset}_{subset}" + instance = context.create_instance( + name=name, + family=family, + families=[family], + subset=subset, + asset=asset, + task=task, + ) + members = list(collection.objects) + members.append(collection) + instance[:] = members + self.log.debug(instance.data) diff --git a/pype/plugins/blender/publish/extract_rig.py b/pype/plugins/blender/publish/extract_rig.py new file mode 100644 index 0000000000..8a3c83d07c --- /dev/null +++ b/pype/plugins/blender/publish/extract_rig.py @@ -0,0 +1,47 @@ +import os +import avalon.blender.workio + +import pype.api + + +class ExtractRig(pype.api.Extractor): + """Extract as rig.""" + + label = "Rig" + hosts = ["blender"] + families = ["rig"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.blend" + filepath = os.path.join(stagingdir, filename) + + # Perform extraction + self.log.info("Performing extraction..") + + # Just save the file to a temporary location. At least for now it's no + # problem to have (possibly) extra stuff in the file. + avalon.blender.workio.save_file(filepath, copy=True) + # + # # Store reference for integration + # if "files" not in instance.data: + # instance.data["files"] = list() + # + # # instance.data["files"].append(filename) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'blend', + 'ext': 'blend', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + + self.log.info("Extracted instance '%s' to: %s", instance.name, representation) From fd6bdc9aa5bb48a3ca252e493281d26f45120470 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Tue, 4 Feb 2020 10:45:50 +0000 Subject: [PATCH 002/118] Loading and removal of rigs from Blender scenes --- pype/blender/plugin.py | 20 ++ pype/plugins/blender/load/load_rig.py | 337 ++++++++++++++++++++++++++ 2 files changed, 357 insertions(+) create mode 100644 pype/plugins/blender/load/load_rig.py diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index eaa429c989..c85e6df990 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -24,6 +24,26 @@ def rig_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: name = f"{namespace}:{name}" return name +def create_blender_context( obj: Optional[bpy.types.Object] = None ): + """Create a new Blender context. If an object is passed as + parameter, it is set as selected and active. + """ + for win in bpy.context.window_manager.windows: + for area in win.screen.areas: + if area.type == 'VIEW_3D': + for region in area.regions: + if region.type == 'WINDOW': + override_context = { + 'window': win, + 'screen': win.screen, + 'area': area, + 'region': region, + 'scene': bpy.context.scene, + 'active_object': obj, + 'selected_objects': [obj] + } + return override_context + raise Exception( "Could not create a custom Blender context." ) class AssetLoader(api.Loader): """A basic AssetLoader for Blender diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py new file mode 100644 index 0000000000..f3c9e49f53 --- /dev/null +++ b/pype/plugins/blender/load/load_rig.py @@ -0,0 +1,337 @@ +"""Load a rig asset in Blender.""" + +import logging +from pathlib import Path +from pprint import pformat +from typing import Dict, List, Optional + +import avalon.blender.pipeline +import bpy +import pype.blender +from avalon import api + +logger = logging.getLogger("pype").getChild("blender").getChild("load_model") + +class BlendRigLoader(pype.blender.AssetLoader): + """Load rigs from a .blend file. + + Because they come from a .blend file we can simply link the collection that + contains the model. There is no further need to 'containerise' it. + + Warning: + Loading the same asset more then once is not properly supported at the + moment. + """ + + families = ["rig"] + representations = ["blend"] + + label = "Link Rig" + icon = "code-fork" + color = "orange" + + @staticmethod + def _get_lib_collection(name: str, libpath: Path) -> Optional[bpy.types.Collection]: + """Find the collection(s) with name, loaded from libpath. + + Note: + It is assumed that only 1 matching collection is found. + """ + for collection in bpy.data.collections: + if collection.name != name: + continue + if collection.library is None: + continue + if not collection.library.filepath: + continue + collection_lib_path = str(Path(bpy.path.abspath(collection.library.filepath)).resolve()) + normalized_libpath = str(Path(bpy.path.abspath(str(libpath))).resolve()) + if collection_lib_path == normalized_libpath: + return collection + return None + + @staticmethod + def _collection_contains_object( + collection: bpy.types.Collection, object: bpy.types.Object + ) -> bool: + """Check if the collection contains the object.""" + for obj in collection.objects: + if obj == object: + return True + return False + + def process_asset( + self, context: dict, name: str, namespace: Optional[str] = None, + options: Optional[Dict] = None + ) -> Optional[List]: + """ + Arguments: + name: Use pre-defined name + namespace: Use pre-defined namespace + context: Full parenthood of representation to load + options: Additional settings dictionary + """ + + libpath = self.fname + asset = context["asset"]["name"] + subset = context["subset"]["name"] + lib_container = pype.blender.plugin.rig_name(asset, subset) + container_name = pype.blender.plugin.rig_name( + asset, subset, namespace + ) + relative = bpy.context.preferences.filepaths.use_relative_paths + + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + container = bpy.data.collections[lib_container] + container.name = container_name + avalon.blender.pipeline.containerise_existing( + container, + name, + namespace, + context, + self.__class__.__name__, + ) + + override_context = pype.blender.plugin.create_blender_context() + bpy.ops.object.collection_instance_add( override_context, name = container_name + "_CON" ) + + override_context = pype.blender.plugin.create_blender_context( bpy.data.objects[container_name + "_CON"] ) + bpy.ops.object.make_override_library( override_context ) + bpy.ops.object.delete( override_context ) + + container_metadata = container.get( 'avalon' ) + + object_names_list = [] + + for c in bpy.data.collections: + + if c.name == container_name + "_CON" and c.library is None: + + for obj in c.objects: + + scene.collection.objects.link( obj ) + c.objects.unlink( obj ) + + if not obj.get("avalon"): + obj["avalon"] = dict() + + avalon_info = obj["avalon"] + avalon_info.update( { "container_name": container_name } ) + + object_names_list.append( obj.name ) + + bpy.data.collections.remove( c ) + + container_metadata["objects"] = object_names_list + + bpy.ops.object.select_all( action = 'DESELECT' ) + + nodes = list(container.objects) + nodes.append(container) + self[:] = nodes + return nodes + + def load(self, + context: dict, + name: Optional[str] = None, + namespace: Optional[str] = None, + options: Optional[Dict] = None) -> Optional[bpy.types.Collection]: + """Load asset via database + + Arguments: + context: Full parenthood of representation to load + name: Use pre-defined name + namespace: Use pre-defined namespace + options: Additional settings dictionary + """ + # TODO (jasper): make it possible to add the asset several times by + # just re-using the collection + assert Path(self.fname).exists(), f"{self.fname} doesn't exist." + + self.process_asset( + context=context, + name=name, + namespace=namespace, + options=options, + ) + + # Only containerise if anything was loaded by the Loader. + nodes = self[:] + if not nodes: + return None + + # Only containerise if it's not already a collection from a .blend file. + representation = context["representation"]["name"] + if representation != "blend": + from avalon.blender.pipeline import containerise + return containerise( + name=name, + namespace=namespace, + nodes=nodes, + context=context, + loader=self.__class__.__name__, + ) + + asset = context["asset"]["name"] + subset = context["subset"]["name"] + instance_name = pype.blender.plugin.rig_name(asset, subset, namespace) + + return self._get_instance_collection(instance_name, nodes) + + def update(self, container: Dict, representation: Dict): + """Update the loaded asset. + + This will remove all objects of the current collection, load the new + ones and add them to the collection. + If the objects of the collection are used in another collection they + will not be removed, only unlinked. Normally this should not be the + case though. + + Warning: + No nested collections are supported at the moment! + """ + + collection = bpy.data.collections.get( + container["objectName"] + ) + libpath = Path(api.get_representation_path(representation)) + extension = libpath.suffix.lower() + + logger.info( + "Container: %s\nRepresentation: %s", + pformat(container, indent=2), + pformat(representation, indent=2), + ) + + assert collection, ( + f"The asset is not loaded: {container['objectName']}" + ) + assert not (collection.children), ( + "Nested collections are not supported." + ) + assert libpath, ( + "No existing library file found for {container['objectName']}" + ) + assert libpath.is_file(), ( + f"The file doesn't exist: {libpath}" + ) + assert extension in pype.blender.plugin.VALID_EXTENSIONS, ( + f"Unsupported file: {libpath}" + ) + collection_libpath = ( + self._get_library_from_container(collection).filepath + ) + print( collection_libpath ) + normalized_collection_libpath = ( + str(Path(bpy.path.abspath(collection_libpath)).resolve()) + ) + print( normalized_collection_libpath ) + normalized_libpath = ( + str(Path(bpy.path.abspath(str(libpath))).resolve()) + ) + print( normalized_libpath ) + logger.debug( + "normalized_collection_libpath:\n %s\nnormalized_libpath:\n %s", + normalized_collection_libpath, + normalized_libpath, + ) + if normalized_collection_libpath == normalized_libpath: + logger.info("Library already loaded, not updating...") + return + # Let Blender's garbage collection take care of removing the library + # itself after removing the objects. + objects_to_remove = set() + collection_objects = list() + collection_objects[:] = collection.objects + for obj in collection_objects: + # Unlink every object + collection.objects.unlink(obj) + remove_obj = True + for coll in [ + coll for coll in bpy.data.collections + if coll != collection + ]: + if ( + coll.objects and + self._collection_contains_object(coll, obj) + ): + remove_obj = False + if remove_obj: + objects_to_remove.add(obj) + + for obj in objects_to_remove: + # Only delete objects that are not used elsewhere + bpy.data.objects.remove(obj) + + instance_empties = [ + obj for obj in collection.users_dupli_group + if obj.name in collection.name + ] + if instance_empties: + instance_empty = instance_empties[0] + container_name = instance_empty["avalon"]["container_name"] + + relative = bpy.context.preferences.filepaths.use_relative_paths + with bpy.data.libraries.load( + str(libpath), link=True, relative=relative + ) as (_, data_to): + data_to.collections = [container_name] + + new_collection = self._get_lib_collection(container_name, libpath) + if new_collection is None: + raise ValueError( + "A matching collection '{container_name}' " + "should have been found in: {libpath}" + ) + + for obj in new_collection.objects: + collection.objects.link(obj) + bpy.data.collections.remove(new_collection) + # Update the representation on the collection + avalon_prop = collection[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_prop["representation"] = str(representation["_id"]) + + def remove(self, container: Dict) -> bool: + """Remove an existing container from a Blender scene. + + Arguments: + container (avalon-core:container-1.0): Container to remove, + from `host.ls()`. + + Returns: + bool: Whether the container was deleted. + + Warning: + No nested collections are supported at the moment! + """ + + print( container["objectName"] ) + + collection = bpy.data.collections.get( + container["objectName"] + ) + if not collection: + return False + assert not (collection.children), ( + "Nested collections are not supported." + ) + instance_objects = list(collection.objects) + + data = collection.get( "avalon" ) + object_names = data["objects"] + + for obj in instance_objects: + bpy.data.objects.remove(obj) + + for name in object_names: + bpy.data.objects.remove( bpy.data.objects[name] ) + + bpy.data.collections.remove(collection) + + return True From e88fc5cb4891f0de74822551258687d5e29226ce Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Tue, 4 Feb 2020 11:49:38 +0000 Subject: [PATCH 003/118] We now use references to objects in the metadata instead of names --- pype/plugins/blender/load/load_rig.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index f3c9e49f53..70cf6e781a 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -107,7 +107,7 @@ class BlendRigLoader(pype.blender.AssetLoader): container_metadata = container.get( 'avalon' ) - object_names_list = [] + objects_list = [] for c in bpy.data.collections: @@ -124,11 +124,11 @@ class BlendRigLoader(pype.blender.AssetLoader): avalon_info = obj["avalon"] avalon_info.update( { "container_name": container_name } ) - object_names_list.append( obj.name ) + objects_list.append( obj ) bpy.data.collections.remove( c ) - container_metadata["objects"] = object_names_list + container_metadata["objects"] = objects_list bpy.ops.object.select_all( action = 'DESELECT' ) @@ -324,13 +324,13 @@ class BlendRigLoader(pype.blender.AssetLoader): instance_objects = list(collection.objects) data = collection.get( "avalon" ) - object_names = data["objects"] + objects = data["objects"] for obj in instance_objects: bpy.data.objects.remove(obj) - for name in object_names: - bpy.data.objects.remove( bpy.data.objects[name] ) + for obj in objects: + bpy.data.objects.remove( obj ) bpy.data.collections.remove(collection) From c123ed757ced7a0fc7bd9b82ce4824e03561573d Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Tue, 4 Feb 2020 15:31:44 +0000 Subject: [PATCH 004/118] Improved handling of rigs using avalon container for metadata only --- pype/plugins/blender/load/load_rig.py | 46 +++++++++++---------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 70cf6e781a..b348f99728 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -81,12 +81,7 @@ class BlendRigLoader(pype.blender.AssetLoader): ) relative = bpy.context.preferences.filepaths.use_relative_paths - with bpy.data.libraries.load( - libpath, link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene + bpy.data.collections.new( lib_container ) container = bpy.data.collections[lib_container] container.name = container_name @@ -98,35 +93,34 @@ class BlendRigLoader(pype.blender.AssetLoader): self.__class__.__name__, ) - override_context = pype.blender.plugin.create_blender_context() - bpy.ops.object.collection_instance_add( override_context, name = container_name + "_CON" ) - - override_context = pype.blender.plugin.create_blender_context( bpy.data.objects[container_name + "_CON"] ) - bpy.ops.object.make_override_library( override_context ) - bpy.ops.object.delete( override_context ) - container_metadata = container.get( 'avalon' ) objects_list = [] - for c in bpy.data.collections: + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (data_from, data_to): - if c.name == container_name + "_CON" and c.library is None: + data_to.collections = [lib_container] - for obj in c.objects: + scene = bpy.context.scene - scene.collection.objects.link( obj ) - c.objects.unlink( obj ) + models = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] + armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] - if not obj.get("avalon"): - obj["avalon"] = dict() + for obj in models + armatures: - avalon_info = obj["avalon"] - avalon_info.update( { "container_name": container_name } ) + scene.collection.objects.link( obj ) - objects_list.append( obj ) + obj = obj.make_local() - bpy.data.collections.remove( c ) + if not obj.get("avalon"): + + obj["avalon"] = dict() + + avalon_info = obj["avalon"] + avalon_info.update( { "container_name": container_name } ) + objects_list.append( obj ) container_metadata["objects"] = objects_list @@ -321,14 +315,10 @@ class BlendRigLoader(pype.blender.AssetLoader): assert not (collection.children), ( "Nested collections are not supported." ) - instance_objects = list(collection.objects) data = collection.get( "avalon" ) objects = data["objects"] - for obj in instance_objects: - bpy.data.objects.remove(obj) - for obj in objects: bpy.data.objects.remove( obj ) From 88b9ceccab25d1785ab73849d83b8ba7c21e8f98 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Tue, 4 Feb 2020 15:45:17 +0000 Subject: [PATCH 005/118] More comments for clarity --- pype/plugins/blender/load/load_rig.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index b348f99728..75aa515c66 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -105,10 +105,13 @@ class BlendRigLoader(pype.blender.AssetLoader): scene = bpy.context.scene - models = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] + meshes = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] - for obj in models + armatures: + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in meshes + armatures: scene.collection.objects.link( obj ) @@ -122,6 +125,7 @@ class BlendRigLoader(pype.blender.AssetLoader): avalon_info.update( { "container_name": container_name } ) objects_list.append( obj ) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list bpy.ops.object.select_all( action = 'DESELECT' ) From fc1779387149902705034f9d2101e7cf5a1acf72 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Wed, 5 Feb 2020 15:20:15 +0000 Subject: [PATCH 006/118] Implemented update for rigs --- pype/plugins/blender/load/load_rig.py | 124 +++++++++++++------------- 1 file changed, 64 insertions(+), 60 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 75aa515c66..294366c41b 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -93,14 +93,14 @@ class BlendRigLoader(pype.blender.AssetLoader): self.__class__.__name__, ) - container_metadata = container.get( 'avalon' ) + container_metadata = container.get( avalon.blender.pipeline.AVALON_PROPERTY ) - objects_list = [] + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container with bpy.data.libraries.load( libpath, link=True, relative=relative ) as (data_from, data_to): - data_to.collections = [lib_container] scene = bpy.context.scene @@ -108,6 +108,8 @@ class BlendRigLoader(pype.blender.AssetLoader): meshes = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] + objects_list = [] + # Link meshes first, then armatures. # The armature is unparented for all the non-local meshes, # when it is made local. @@ -117,17 +119,19 @@ class BlendRigLoader(pype.blender.AssetLoader): obj = obj.make_local() - if not obj.get("avalon"): + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): - obj["avalon"] = dict() + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj["avalon"] + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] avalon_info.update( { "container_name": container_name } ) objects_list.append( obj ) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list + bpy.data.collections.remove( bpy.data.collections[lib_container] ) + bpy.ops.object.select_all( action = 'DESELECT' ) nodes = list(container.objects) @@ -198,6 +202,7 @@ class BlendRigLoader(pype.blender.AssetLoader): collection = bpy.data.collections.get( container["objectName"] ) + libpath = Path(api.get_representation_path(representation)) extension = libpath.suffix.lower() @@ -222,18 +227,14 @@ class BlendRigLoader(pype.blender.AssetLoader): assert extension in pype.blender.plugin.VALID_EXTENSIONS, ( f"Unsupported file: {libpath}" ) - collection_libpath = ( - self._get_library_from_container(collection).filepath - ) - print( collection_libpath ) + + collection_libpath = container["libpath"] normalized_collection_libpath = ( str(Path(bpy.path.abspath(collection_libpath)).resolve()) ) - print( normalized_collection_libpath ) normalized_libpath = ( str(Path(bpy.path.abspath(str(libpath))).resolve()) ) - print( normalized_libpath ) logger.debug( "normalized_collection_libpath:\n %s\nnormalized_libpath:\n %s", normalized_collection_libpath, @@ -242,58 +243,63 @@ class BlendRigLoader(pype.blender.AssetLoader): if normalized_collection_libpath == normalized_libpath: logger.info("Library already loaded, not updating...") return - # Let Blender's garbage collection take care of removing the library - # itself after removing the objects. - objects_to_remove = set() - collection_objects = list() - collection_objects[:] = collection.objects - for obj in collection_objects: - # Unlink every object - collection.objects.unlink(obj) - remove_obj = True - for coll in [ - coll for coll in bpy.data.collections - if coll != collection - ]: - if ( - coll.objects and - self._collection_contains_object(coll, obj) - ): - remove_obj = False - if remove_obj: - objects_to_remove.add(obj) - for obj in objects_to_remove: - # Only delete objects that are not used elsewhere - bpy.data.objects.remove(obj) + # Get the armature of the rig + armatures = [ obj for obj in container["objects"] if obj.type == 'ARMATURE' ] + assert( len( armatures ) == 1 ) - instance_empties = [ - obj for obj in collection.users_dupli_group - if obj.name in collection.name - ] - if instance_empties: - instance_empty = instance_empties[0] - container_name = instance_empty["avalon"]["container_name"] + action = armatures[0].animation_data.action + + for obj in container["objects"]: + bpy.data.objects.remove( obj ) + + lib_container = container["lib_container"] relative = bpy.context.preferences.filepaths.use_relative_paths with bpy.data.libraries.load( str(libpath), link=True, relative=relative ) as (_, data_to): - data_to.collections = [container_name] + data_to.collections = [lib_container] - new_collection = self._get_lib_collection(container_name, libpath) - if new_collection is None: - raise ValueError( - "A matching collection '{container_name}' " - "should have been found in: {libpath}" - ) + scene = bpy.context.scene - for obj in new_collection.objects: - collection.objects.link(obj) - bpy.data.collections.remove(new_collection) - # Update the representation on the collection - avalon_prop = collection[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_prop["representation"] = str(representation["_id"]) + meshes = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] + armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] + objects_list = [] + + assert( len( armatures ) == 1 ) + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in meshes + armatures: + + scene.collection.objects.link( obj ) + + obj = obj.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update( { "container_name": collection.name } ) + objects_list.append( obj ) + + if obj.type == 'ARMATURE' and action is not None: + + obj.animation_data.action = action + + collection_metadata = collection.get(avalon.blender.pipeline.AVALON_PROPERTY) + + # Save the list of objects in the metadata container + collection_metadata["objects"] = objects_list + collection_metadata["libpath"] = str(libpath) + collection_metadata["representation"] = str(representation["_id"]) + + bpy.data.collections.remove( bpy.data.collections[lib_container] ) + + bpy.ops.object.select_all( action = 'DESELECT' ) def remove(self, container: Dict) -> bool: """Remove an existing container from a Blender scene. @@ -309,8 +315,6 @@ class BlendRigLoader(pype.blender.AssetLoader): No nested collections are supported at the moment! """ - print( container["objectName"] ) - collection = bpy.data.collections.get( container["objectName"] ) @@ -320,12 +324,12 @@ class BlendRigLoader(pype.blender.AssetLoader): "Nested collections are not supported." ) - data = collection.get( "avalon" ) - objects = data["objects"] + collection_metadata = collection.get( avalon.blender.pipeline.AVALON_PROPERTY ) + objects = collection_metadata["objects"] for obj in objects: bpy.data.objects.remove( obj ) - bpy.data.collections.remove(collection) + bpy.data.collections.remove( collection ) return True From 4bd4c6e811f52a4cfc74551d53b3a9fde2e857a3 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Wed, 5 Feb 2020 16:49:27 +0000 Subject: [PATCH 007/118] The data in the objects is made local as well. Not having this would cause problems with the keyframing of shape keys and custom data. --- pype/plugins/blender/load/load_rig.py | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 294366c41b..fa0e1c52b2 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -119,6 +119,8 @@ class BlendRigLoader(pype.blender.AssetLoader): obj = obj.make_local() + obj.data.make_local() + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() @@ -228,7 +230,9 @@ class BlendRigLoader(pype.blender.AssetLoader): f"Unsupported file: {libpath}" ) - collection_libpath = container["libpath"] + collection_metadata = collection.get(avalon.blender.pipeline.AVALON_PROPERTY) + + collection_libpath = collection_metadata["libpath"] normalized_collection_libpath = ( str(Path(bpy.path.abspath(collection_libpath)).resolve()) ) @@ -245,15 +249,19 @@ class BlendRigLoader(pype.blender.AssetLoader): return # Get the armature of the rig - armatures = [ obj for obj in container["objects"] if obj.type == 'ARMATURE' ] + armatures = [ obj for obj in collection_metadata["objects"] if obj.type == 'ARMATURE' ] assert( len( armatures ) == 1 ) action = armatures[0].animation_data.action - for obj in container["objects"]: - bpy.data.objects.remove( obj ) + for obj in collection_metadata["objects"]: - lib_container = container["lib_container"] + if obj.type == 'ARMATURE': + bpy.data.armatures.remove( obj.data ) + elif obj.type == 'MESH': + bpy.data.meshes.remove( obj.data ) + + lib_container = collection_metadata["lib_container"] relative = bpy.context.preferences.filepaths.use_relative_paths with bpy.data.libraries.load( @@ -278,6 +286,8 @@ class BlendRigLoader(pype.blender.AssetLoader): obj = obj.make_local() + obj.data.make_local() + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() @@ -290,8 +300,6 @@ class BlendRigLoader(pype.blender.AssetLoader): obj.animation_data.action = action - collection_metadata = collection.get(avalon.blender.pipeline.AVALON_PROPERTY) - # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list collection_metadata["libpath"] = str(libpath) @@ -328,7 +336,11 @@ class BlendRigLoader(pype.blender.AssetLoader): objects = collection_metadata["objects"] for obj in objects: - bpy.data.objects.remove( obj ) + + if obj.type == 'ARMATURE': + bpy.data.armatures.remove( obj.data ) + elif obj.type == 'MESH': + bpy.data.meshes.remove( obj.data ) bpy.data.collections.remove( collection ) From 6f6482a58b4aa328af07ccaae9c2e66ccf7ecc7f Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 7 Feb 2020 11:39:14 +0000 Subject: [PATCH 008/118] Rig is linked in a collection, instead of the generic scene collection --- pype/plugins/blender/load/load_rig.py | 79 ++++++++++++++++----------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index fa0e1c52b2..7ea131a54c 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -12,6 +12,7 @@ from avalon import api logger = logging.getLogger("pype").getChild("blender").getChild("load_model") + class BlendRigLoader(pype.blender.AssetLoader): """Load rigs from a .blend file. @@ -44,8 +45,10 @@ class BlendRigLoader(pype.blender.AssetLoader): continue if not collection.library.filepath: continue - collection_lib_path = str(Path(bpy.path.abspath(collection.library.filepath)).resolve()) - normalized_libpath = str(Path(bpy.path.abspath(str(libpath))).resolve()) + collection_lib_path = str( + Path(bpy.path.abspath(collection.library.filepath)).resolve()) + normalized_libpath = str( + Path(bpy.path.abspath(str(libpath))).resolve()) if collection_lib_path == normalized_libpath: return collection return None @@ -81,7 +84,7 @@ class BlendRigLoader(pype.blender.AssetLoader): ) relative = bpy.context.preferences.filepaths.use_relative_paths - bpy.data.collections.new( lib_container ) + bpy.data.collections.new(lib_container) container = bpy.data.collections[lib_container] container.name = container_name @@ -93,7 +96,8 @@ class BlendRigLoader(pype.blender.AssetLoader): self.__class__.__name__, ) - container_metadata = container.get( avalon.blender.pipeline.AVALON_PROPERTY ) + container_metadata = container.get( + avalon.blender.pipeline.AVALON_PROPERTY) container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container @@ -105,8 +109,13 @@ class BlendRigLoader(pype.blender.AssetLoader): scene = bpy.context.scene - meshes = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] - armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] + scene.collection.children.link(bpy.data.collections[lib_container]) + + rig_container = scene.collection.children[lib_container].make_local() + + meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] + armatures = [ + obj for obj in rig_container.objects if obj.type == 'ARMATURE'] objects_list = [] @@ -115,8 +124,6 @@ class BlendRigLoader(pype.blender.AssetLoader): # when it is made local. for obj in meshes + armatures: - scene.collection.objects.link( obj ) - obj = obj.make_local() obj.data.make_local() @@ -126,15 +133,13 @@ class BlendRigLoader(pype.blender.AssetLoader): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update( { "container_name": container_name } ) - objects_list.append( obj ) + avalon_info.update({"container_name": container_name}) + objects_list.append(obj) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list - bpy.data.collections.remove( bpy.data.collections[lib_container] ) - - bpy.ops.object.select_all( action = 'DESELECT' ) + bpy.ops.object.select_all(action='DESELECT') nodes = list(container.objects) nodes.append(container) @@ -230,7 +235,8 @@ class BlendRigLoader(pype.blender.AssetLoader): f"Unsupported file: {libpath}" ) - collection_metadata = collection.get(avalon.blender.pipeline.AVALON_PROPERTY) + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) collection_libpath = collection_metadata["libpath"] normalized_collection_libpath = ( @@ -249,20 +255,23 @@ class BlendRigLoader(pype.blender.AssetLoader): return # Get the armature of the rig - armatures = [ obj for obj in collection_metadata["objects"] if obj.type == 'ARMATURE' ] - assert( len( armatures ) == 1 ) + armatures = [obj for obj in collection_metadata["objects"] + if obj.type == 'ARMATURE'] + assert(len(armatures) == 1) action = armatures[0].animation_data.action for obj in collection_metadata["objects"]: if obj.type == 'ARMATURE': - bpy.data.armatures.remove( obj.data ) + bpy.data.armatures.remove(obj.data) elif obj.type == 'MESH': - bpy.data.meshes.remove( obj.data ) + bpy.data.meshes.remove(obj.data) lib_container = collection_metadata["lib_container"] + bpy.data.collections.remove(bpy.data.collections[lib_container]) + relative = bpy.context.preferences.filepaths.use_relative_paths with bpy.data.libraries.load( str(libpath), link=True, relative=relative @@ -271,19 +280,22 @@ class BlendRigLoader(pype.blender.AssetLoader): scene = bpy.context.scene - meshes = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'MESH' ] - armatures = [ obj for obj in bpy.data.collections[lib_container].objects if obj.type == 'ARMATURE' ] + scene.collection.children.link(bpy.data.collections[lib_container]) + + rig_container = scene.collection.children[lib_container].make_local() + + meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] + armatures = [ + obj for obj in rig_container.objects if obj.type == 'ARMATURE'] objects_list = [] - assert( len( armatures ) == 1 ) + assert(len(armatures) == 1) # Link meshes first, then armatures. # The armature is unparented for all the non-local meshes, # when it is made local. for obj in meshes + armatures: - scene.collection.objects.link( obj ) - obj = obj.make_local() obj.data.make_local() @@ -293,8 +305,8 @@ class BlendRigLoader(pype.blender.AssetLoader): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update( { "container_name": collection.name } ) - objects_list.append( obj ) + avalon_info.update({"container_name": collection.name}) + objects_list.append(obj) if obj.type == 'ARMATURE' and action is not None: @@ -305,9 +317,7 @@ class BlendRigLoader(pype.blender.AssetLoader): collection_metadata["libpath"] = str(libpath) collection_metadata["representation"] = str(representation["_id"]) - bpy.data.collections.remove( bpy.data.collections[lib_container] ) - - bpy.ops.object.select_all( action = 'DESELECT' ) + bpy.ops.object.select_all(action='DESELECT') def remove(self, container: Dict) -> bool: """Remove an existing container from a Blender scene. @@ -332,16 +342,19 @@ class BlendRigLoader(pype.blender.AssetLoader): "Nested collections are not supported." ) - collection_metadata = collection.get( avalon.blender.pipeline.AVALON_PROPERTY ) + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] for obj in objects: if obj.type == 'ARMATURE': - bpy.data.armatures.remove( obj.data ) + bpy.data.armatures.remove(obj.data) elif obj.type == 'MESH': - bpy.data.meshes.remove( obj.data ) - - bpy.data.collections.remove( collection ) + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + bpy.data.collections.remove(collection) return True From 65104882db7a44ba3d5e213aae9f7cee0c572c93 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 7 Feb 2020 15:09:21 +0000 Subject: [PATCH 009/118] Changed handling of models for consistency --- pype/plugins/blender/load/load_model.py | 196 ++++++++++++------------ pype/plugins/blender/load/load_rig.py | 39 +---- 2 files changed, 98 insertions(+), 137 deletions(-) diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index bd6db17650..bb9f2250be 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -12,7 +12,6 @@ from avalon import api logger = logging.getLogger("pype").getChild("blender").getChild("load_model") - class BlendModelLoader(pype.blender.AssetLoader): """Load models from a .blend file. @@ -31,36 +30,6 @@ class BlendModelLoader(pype.blender.AssetLoader): icon = "code-fork" color = "orange" - @staticmethod - def _get_lib_collection(name: str, libpath: Path) -> Optional[bpy.types.Collection]: - """Find the collection(s) with name, loaded from libpath. - - Note: - It is assumed that only 1 matching collection is found. - """ - for collection in bpy.data.collections: - if collection.name != name: - continue - if collection.library is None: - continue - if not collection.library.filepath: - continue - collection_lib_path = str(Path(bpy.path.abspath(collection.library.filepath)).resolve()) - normalized_libpath = str(Path(bpy.path.abspath(str(libpath))).resolve()) - if collection_lib_path == normalized_libpath: - return collection - return None - - @staticmethod - def _collection_contains_object( - collection: bpy.types.Collection, object: bpy.types.Object - ) -> bool: - """Check if the collection contains the object.""" - for obj in collection.objects: - if obj == object: - return True - return False - def process_asset( self, context: dict, name: str, namespace: Optional[str] = None, options: Optional[Dict] = None @@ -82,25 +51,8 @@ class BlendModelLoader(pype.blender.AssetLoader): ) relative = bpy.context.preferences.filepaths.use_relative_paths - with bpy.data.libraries.load( - libpath, link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene - instance_empty = bpy.data.objects.new( - container_name, None - ) - if not instance_empty.get("avalon"): - instance_empty["avalon"] = dict() - avalon_info = instance_empty["avalon"] - avalon_info.update({"container_name": container_name}) - scene.collection.objects.link(instance_empty) - instance_empty.instance_type = 'COLLECTION' - container = bpy.data.collections[lib_container] + container = bpy.data.collections.new(lib_container) container.name = container_name - instance_empty.instance_collection = container - container.make_local() avalon.blender.pipeline.containerise_existing( container, name, @@ -109,9 +61,47 @@ class BlendModelLoader(pype.blender.AssetLoader): self.__class__.__name__, ) + container_metadata = container.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container + + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + rig_container = scene.collection.children[lib_container].make_local() + + objects_list = [] + + for obj in rig_container.objects: + + obj = obj.make_local() + + obj.data.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": container_name}) + + objects_list.append(obj) + + # Save the list of objects in the metadata container + container_metadata["objects"] = objects_list + + bpy.ops.object.select_all(action='DESELECT') + nodes = list(container.objects) nodes.append(container) - nodes.append(instance_empty) self[:] = nodes return nodes @@ -154,9 +144,11 @@ class BlendModelLoader(pype.blender.AssetLoader): assert extension in pype.blender.plugin.VALID_EXTENSIONS, ( f"Unsupported file: {libpath}" ) - collection_libpath = ( - self._get_library_from_container(collection).filepath - ) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + collection_libpath = collection_metadata["libpath"] normalized_collection_libpath = ( str(Path(bpy.path.abspath(collection_libpath)).resolve()) ) @@ -171,58 +163,52 @@ class BlendModelLoader(pype.blender.AssetLoader): if normalized_collection_libpath == normalized_libpath: logger.info("Library already loaded, not updating...") return - # Let Blender's garbage collection take care of removing the library - # itself after removing the objects. - objects_to_remove = set() - collection_objects = list() - collection_objects[:] = collection.objects - for obj in collection_objects: - # Unlink every object - collection.objects.unlink(obj) - remove_obj = True - for coll in [ - coll for coll in bpy.data.collections - if coll != collection - ]: - if ( - coll.objects and - self._collection_contains_object(coll, obj) - ): - remove_obj = False - if remove_obj: - objects_to_remove.add(obj) - for obj in objects_to_remove: - # Only delete objects that are not used elsewhere - bpy.data.objects.remove(obj) + for obj in collection_metadata["objects"]: - instance_empties = [ - obj for obj in collection.users_dupli_group - if obj.name in collection.name - ] - if instance_empties: - instance_empty = instance_empties[0] - container_name = instance_empty["avalon"]["container_name"] + bpy.data.meshes.remove(obj.data) + + lib_container = collection_metadata["lib_container"] + + bpy.data.collections.remove(bpy.data.collections[lib_container]) relative = bpy.context.preferences.filepaths.use_relative_paths with bpy.data.libraries.load( str(libpath), link=True, relative=relative ) as (_, data_to): - data_to.collections = [container_name] + data_to.collections = [lib_container] - new_collection = self._get_lib_collection(container_name, libpath) - if new_collection is None: - raise ValueError( - "A matching collection '{container_name}' " - "should have been found in: {libpath}" - ) + scene = bpy.context.scene - for obj in new_collection.objects: - collection.objects.link(obj) - bpy.data.collections.remove(new_collection) - # Update the representation on the collection - avalon_prop = collection[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_prop["representation"] = str(representation["_id"]) + scene.collection.children.link(bpy.data.collections[lib_container]) + + rig_container = scene.collection.children[lib_container].make_local() + + objects_list = [] + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in rig_container.objects: + + obj = obj.make_local() + + obj.data.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": collection.name}) + objects_list.append(obj) + + # Save the list of objects in the metadata container + collection_metadata["objects"] = objects_list + collection_metadata["libpath"] = str(libpath) + collection_metadata["representation"] = str(representation["_id"]) + + bpy.ops.object.select_all(action='DESELECT') def remove(self, container: Dict) -> bool: """Remove an existing container from a Blender scene. @@ -245,10 +231,17 @@ class BlendModelLoader(pype.blender.AssetLoader): assert not (collection.children), ( "Nested collections are not supported." ) - instance_parents = list(collection.users_dupli_group) - instance_objects = list(collection.objects) - for obj in instance_objects + instance_parents: - bpy.data.objects.remove(obj) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] + + for obj in objects: + + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) bpy.data.collections.remove(collection) return True @@ -281,7 +274,8 @@ class CacheModelLoader(pype.blender.AssetLoader): context: Full parenthood of representation to load options: Additional settings dictionary """ - raise NotImplementedError("Loading of Alembic files is not yet implemented.") + raise NotImplementedError( + "Loading of Alembic files is not yet implemented.") # TODO (jasper): implement Alembic import. libpath = self.fname diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 7ea131a54c..8593440624 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -31,38 +31,6 @@ class BlendRigLoader(pype.blender.AssetLoader): icon = "code-fork" color = "orange" - @staticmethod - def _get_lib_collection(name: str, libpath: Path) -> Optional[bpy.types.Collection]: - """Find the collection(s) with name, loaded from libpath. - - Note: - It is assumed that only 1 matching collection is found. - """ - for collection in bpy.data.collections: - if collection.name != name: - continue - if collection.library is None: - continue - if not collection.library.filepath: - continue - collection_lib_path = str( - Path(bpy.path.abspath(collection.library.filepath)).resolve()) - normalized_libpath = str( - Path(bpy.path.abspath(str(libpath))).resolve()) - if collection_lib_path == normalized_libpath: - return collection - return None - - @staticmethod - def _collection_contains_object( - collection: bpy.types.Collection, object: bpy.types.Object - ) -> bool: - """Check if the collection contains the object.""" - for obj in collection.objects: - if obj == object: - return True - return False - def process_asset( self, context: dict, name: str, namespace: Optional[str] = None, options: Optional[Dict] = None @@ -84,9 +52,7 @@ class BlendRigLoader(pype.blender.AssetLoader): ) relative = bpy.context.preferences.filepaths.use_relative_paths - bpy.data.collections.new(lib_container) - - container = bpy.data.collections[lib_container] + container = bpy.data.collections.new(lib_container) container.name = container_name avalon.blender.pipeline.containerise_existing( container, @@ -104,7 +70,7 @@ class BlendRigLoader(pype.blender.AssetLoader): with bpy.data.libraries.load( libpath, link=True, relative=relative - ) as (data_from, data_to): + ) as (_, data_to): data_to.collections = [lib_container] scene = bpy.context.scene @@ -134,6 +100,7 @@ class BlendRigLoader(pype.blender.AssetLoader): avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": container_name}) + objects_list.append(obj) # Save the list of objects in the metadata container From 951dcfca3e7f9c02e8d878fea4d693f950b7fada Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Mon, 10 Feb 2020 14:45:50 +0000 Subject: [PATCH 010/118] Code optimization --- pype/blender/plugin.py | 13 ++---- pype/plugins/blender/create/create_model.py | 2 +- pype/plugins/blender/create/create_rig.py | 4 +- pype/plugins/blender/load/load_model.py | 15 +++--- pype/plugins/blender/load/load_rig.py | 51 +-------------------- 5 files changed, 16 insertions(+), 69 deletions(-) diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index c85e6df990..b441714c0d 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -10,15 +10,8 @@ from avalon import api VALID_EXTENSIONS = [".blend"] -def model_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: - """Return a consistent name for a model asset.""" - name = f"{asset}_{subset}" - if namespace: - name = f"{namespace}:{name}" - return name - -def rig_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: - """Return a consistent name for a rig asset.""" +def asset_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: + """Return a consistent name for an asset.""" name = f"{asset}_{subset}" if namespace: name = f"{namespace}:{name}" @@ -149,7 +142,7 @@ class AssetLoader(api.Loader): asset = context["asset"]["name"] subset = context["subset"]["name"] - instance_name = model_name(asset, subset, namespace) + instance_name = asset_name(asset, subset, namespace) return self._get_instance_collection(instance_name, nodes) diff --git a/pype/plugins/blender/create/create_model.py b/pype/plugins/blender/create/create_model.py index 7301073f05..a3b2ffc55b 100644 --- a/pype/plugins/blender/create/create_model.py +++ b/pype/plugins/blender/create/create_model.py @@ -19,7 +19,7 @@ class CreateModel(Creator): asset = self.data["asset"] subset = self.data["subset"] - name = pype.blender.plugin.model_name(asset, subset) + name = pype.blender.plugin.asset_name(asset, subset) collection = bpy.data.collections.new(name=name) bpy.context.scene.collection.children.link(collection) self.data['task'] = api.Session.get('AVALON_TASK') diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py index 01eb524eef..5d83fafdd3 100644 --- a/pype/plugins/blender/create/create_rig.py +++ b/pype/plugins/blender/create/create_rig.py @@ -12,14 +12,14 @@ class CreateRig(Creator): name = "rigMain" label = "Rig" family = "rig" - icon = "cube" + icon = "wheelchair" def process(self): import pype.blender asset = self.data["asset"] subset = self.data["subset"] - name = pype.blender.plugin.rig_name(asset, subset) + name = pype.blender.plugin.asset_name(asset, subset) collection = bpy.data.collections.new(name=name) bpy.context.scene.collection.children.link(collection) self.data['task'] = api.Session.get('AVALON_TASK') diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index bb9f2250be..cde4109a7c 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -12,6 +12,7 @@ from avalon import api logger = logging.getLogger("pype").getChild("blender").getChild("load_model") + class BlendModelLoader(pype.blender.AssetLoader): """Load models from a .blend file. @@ -45,8 +46,8 @@ class BlendModelLoader(pype.blender.AssetLoader): libpath = self.fname asset = context["asset"]["name"] subset = context["subset"]["name"] - lib_container = pype.blender.plugin.model_name(asset, subset) - container_name = pype.blender.plugin.model_name( + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( asset, subset, namespace ) relative = bpy.context.preferences.filepaths.use_relative_paths @@ -76,11 +77,11 @@ class BlendModelLoader(pype.blender.AssetLoader): scene.collection.children.link(bpy.data.collections[lib_container]) - rig_container = scene.collection.children[lib_container].make_local() + model_container = scene.collection.children[lib_container].make_local() objects_list = [] - for obj in rig_container.objects: + for obj in model_container.objects: obj = obj.make_local() @@ -182,14 +183,14 @@ class BlendModelLoader(pype.blender.AssetLoader): scene.collection.children.link(bpy.data.collections[lib_container]) - rig_container = scene.collection.children[lib_container].make_local() + model_container = scene.collection.children[lib_container].make_local() objects_list = [] # Link meshes first, then armatures. # The armature is unparented for all the non-local meshes, # when it is made local. - for obj in rig_container.objects: + for obj in model_container.objects: obj = obj.make_local() @@ -283,7 +284,7 @@ class CacheModelLoader(pype.blender.AssetLoader): subset = context["subset"]["name"] # TODO (jasper): evaluate use of namespace which is 'alien' to Blender. lib_container = container_name = ( - pype.blender.plugin.model_name(asset, subset, namespace) + pype.blender.plugin.asset_name(asset, subset, namespace) ) relative = bpy.context.preferences.filepaths.use_relative_paths diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 8593440624..361850c51b 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -46,8 +46,8 @@ class BlendRigLoader(pype.blender.AssetLoader): libpath = self.fname asset = context["asset"]["name"] subset = context["subset"]["name"] - lib_container = pype.blender.plugin.rig_name(asset, subset) - container_name = pype.blender.plugin.rig_name( + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( asset, subset, namespace ) relative = bpy.context.preferences.filepaths.use_relative_paths @@ -113,53 +113,6 @@ class BlendRigLoader(pype.blender.AssetLoader): self[:] = nodes return nodes - def load(self, - context: dict, - name: Optional[str] = None, - namespace: Optional[str] = None, - options: Optional[Dict] = None) -> Optional[bpy.types.Collection]: - """Load asset via database - - Arguments: - context: Full parenthood of representation to load - name: Use pre-defined name - namespace: Use pre-defined namespace - options: Additional settings dictionary - """ - # TODO (jasper): make it possible to add the asset several times by - # just re-using the collection - assert Path(self.fname).exists(), f"{self.fname} doesn't exist." - - self.process_asset( - context=context, - name=name, - namespace=namespace, - options=options, - ) - - # Only containerise if anything was loaded by the Loader. - nodes = self[:] - if not nodes: - return None - - # Only containerise if it's not already a collection from a .blend file. - representation = context["representation"]["name"] - if representation != "blend": - from avalon.blender.pipeline import containerise - return containerise( - name=name, - namespace=namespace, - nodes=nodes, - context=context, - loader=self.__class__.__name__, - ) - - asset = context["asset"]["name"] - subset = context["subset"]["name"] - instance_name = pype.blender.plugin.rig_name(asset, subset, namespace) - - return self._get_instance_collection(instance_name, nodes) - def update(self, container: Dict, representation: Dict): """Update the loaded asset. From 2b6e90ffe00a44a7f9e972389a50c3c8b20fb972 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Mon, 10 Feb 2020 14:55:04 +0000 Subject: [PATCH 011/118] Creation, loading, and management of animations --- .../blender/create/create_animation.py | 30 ++ pype/plugins/blender/load/load_animation.py | 274 ++++++++++++++++++ .../blender/publish/collect_animation.py | 53 ++++ .../blender/publish/extract_animation.py | 47 +++ 4 files changed, 404 insertions(+) create mode 100644 pype/plugins/blender/create/create_animation.py create mode 100644 pype/plugins/blender/load/load_animation.py create mode 100644 pype/plugins/blender/publish/collect_animation.py create mode 100644 pype/plugins/blender/publish/extract_animation.py diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py new file mode 100644 index 0000000000..cfe569f918 --- /dev/null +++ b/pype/plugins/blender/create/create_animation.py @@ -0,0 +1,30 @@ +import bpy + +from avalon import api +from avalon.blender import Creator, lib + + +class CreateAnimation(Creator): + """Animation output for character rigs""" + + name = "animationMain" + label = "Animation" + family = "animation" + icon = "male" + + def process(self): + import pype.blender + + asset = self.data["asset"] + subset = self.data["subset"] + name = pype.blender.plugin.asset_name(asset, subset) + collection = bpy.data.collections.new(name=name) + bpy.context.scene.collection.children.link(collection) + self.data['task'] = api.Session.get('AVALON_TASK') + lib.imprint(collection, self.data) + + if (self.options or {}).get("useSelection"): + for obj in lib.get_selection(): + collection.objects.link(obj) + + return collection diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py new file mode 100644 index 0000000000..5b527e1717 --- /dev/null +++ b/pype/plugins/blender/load/load_animation.py @@ -0,0 +1,274 @@ +"""Load an animation in Blender.""" + +import logging +from pathlib import Path +from pprint import pformat +from typing import Dict, List, Optional + +import avalon.blender.pipeline +import bpy +import pype.blender +from avalon import api + +logger = logging.getLogger("pype").getChild("blender").getChild("load_model") + + +class BlendAnimationLoader(pype.blender.AssetLoader): + """Load animations from a .blend file. + + Because they come from a .blend file we can simply link the collection that + contains the model. There is no further need to 'containerise' it. + + Warning: + Loading the same asset more then once is not properly supported at the + moment. + """ + + families = ["animation"] + representations = ["blend"] + + label = "Link Animation" + icon = "code-fork" + color = "orange" + + def process_asset( + self, context: dict, name: str, namespace: Optional[str] = None, + options: Optional[Dict] = None + ) -> Optional[List]: + """ + Arguments: + name: Use pre-defined name + namespace: Use pre-defined namespace + context: Full parenthood of representation to load + options: Additional settings dictionary + """ + + libpath = self.fname + asset = context["asset"]["name"] + subset = context["subset"]["name"] + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( + asset, subset, namespace + ) + relative = bpy.context.preferences.filepaths.use_relative_paths + + container = bpy.data.collections.new(lib_container) + container.name = container_name + avalon.blender.pipeline.containerise_existing( + container, + name, + namespace, + context, + self.__class__.__name__, + ) + + container_metadata = container.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container + + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + animation_container = scene.collection.children[lib_container].make_local() + + meshes = [obj for obj in animation_container.objects if obj.type == 'MESH'] + armatures = [ + obj for obj in animation_container.objects if obj.type == 'ARMATURE'] + + objects_list = [] + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in meshes + armatures: + + obj = obj.make_local() + + obj.data.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": container_name}) + + objects_list.append(obj) + + # Save the list of objects in the metadata container + container_metadata["objects"] = objects_list + + bpy.ops.object.select_all(action='DESELECT') + + nodes = list(container.objects) + nodes.append(container) + self[:] = nodes + return nodes + + def update(self, container: Dict, representation: Dict): + """Update the loaded asset. + + This will remove all objects of the current collection, load the new + ones and add them to the collection. + If the objects of the collection are used in another collection they + will not be removed, only unlinked. Normally this should not be the + case though. + + Warning: + No nested collections are supported at the moment! + """ + + collection = bpy.data.collections.get( + container["objectName"] + ) + + libpath = Path(api.get_representation_path(representation)) + extension = libpath.suffix.lower() + + logger.info( + "Container: %s\nRepresentation: %s", + pformat(container, indent=2), + pformat(representation, indent=2), + ) + + assert collection, ( + f"The asset is not loaded: {container['objectName']}" + ) + assert not (collection.children), ( + "Nested collections are not supported." + ) + assert libpath, ( + "No existing library file found for {container['objectName']}" + ) + assert libpath.is_file(), ( + f"The file doesn't exist: {libpath}" + ) + assert extension in pype.blender.plugin.VALID_EXTENSIONS, ( + f"Unsupported file: {libpath}" + ) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + collection_libpath = collection_metadata["libpath"] + normalized_collection_libpath = ( + str(Path(bpy.path.abspath(collection_libpath)).resolve()) + ) + normalized_libpath = ( + str(Path(bpy.path.abspath(str(libpath))).resolve()) + ) + logger.debug( + "normalized_collection_libpath:\n %s\nnormalized_libpath:\n %s", + normalized_collection_libpath, + normalized_libpath, + ) + if normalized_collection_libpath == normalized_libpath: + logger.info("Library already loaded, not updating...") + return + + # Get the armature of the rig + armatures = [obj for obj in collection_metadata["objects"] + if obj.type == 'ARMATURE'] + assert(len(armatures) == 1) + + for obj in collection_metadata["objects"]: + + if obj.type == 'ARMATURE': + bpy.data.armatures.remove(obj.data) + elif obj.type == 'MESH': + bpy.data.meshes.remove(obj.data) + + lib_container = collection_metadata["lib_container"] + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + + relative = bpy.context.preferences.filepaths.use_relative_paths + with bpy.data.libraries.load( + str(libpath), link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + animation_container = scene.collection.children[lib_container].make_local() + + meshes = [obj for obj in animation_container.objects if obj.type == 'MESH'] + armatures = [ + obj for obj in animation_container.objects if obj.type == 'ARMATURE'] + objects_list = [] + + assert(len(armatures) == 1) + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in meshes + armatures: + + obj = obj.make_local() + + obj.data.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": collection.name}) + objects_list.append(obj) + + # Save the list of objects in the metadata container + collection_metadata["objects"] = objects_list + collection_metadata["libpath"] = str(libpath) + collection_metadata["representation"] = str(representation["_id"]) + + bpy.ops.object.select_all(action='DESELECT') + + def remove(self, container: Dict) -> bool: + """Remove an existing container from a Blender scene. + + Arguments: + container (avalon-core:container-1.0): Container to remove, + from `host.ls()`. + + Returns: + bool: Whether the container was deleted. + + Warning: + No nested collections are supported at the moment! + """ + + collection = bpy.data.collections.get( + container["objectName"] + ) + if not collection: + return False + assert not (collection.children), ( + "Nested collections are not supported." + ) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] + + for obj in objects: + + if obj.type == 'ARMATURE': + bpy.data.armatures.remove(obj.data) + elif obj.type == 'MESH': + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + bpy.data.collections.remove(collection) + + return True diff --git a/pype/plugins/blender/publish/collect_animation.py b/pype/plugins/blender/publish/collect_animation.py new file mode 100644 index 0000000000..9bc0b02227 --- /dev/null +++ b/pype/plugins/blender/publish/collect_animation.py @@ -0,0 +1,53 @@ +import typing +from typing import Generator + +import bpy + +import avalon.api +import pyblish.api +from avalon.blender.pipeline import AVALON_PROPERTY + + +class CollectAnimation(pyblish.api.ContextPlugin): + """Collect the data of an animation.""" + + hosts = ["blender"] + label = "Collect Animation" + order = pyblish.api.CollectorOrder + + @staticmethod + def get_animation_collections() -> Generator: + """Return all 'animation' collections. + + Check if the family is 'animation' and if it doesn't have the + representation set. If the representation is set, it is a loaded rig + and we don't want to publish it. + """ + for collection in bpy.data.collections: + avalon_prop = collection.get(AVALON_PROPERTY) or dict() + if (avalon_prop.get('family') == 'animation' + and not avalon_prop.get('representation')): + yield collection + + def process(self, context): + """Collect the animations from the current Blender scene.""" + collections = self.get_animation_collections() + for collection in collections: + avalon_prop = collection[AVALON_PROPERTY] + asset = avalon_prop['asset'] + family = avalon_prop['family'] + subset = avalon_prop['subset'] + task = avalon_prop['task'] + name = f"{asset}_{subset}" + instance = context.create_instance( + name=name, + family=family, + families=[family], + subset=subset, + asset=asset, + task=task, + ) + members = list(collection.objects) + members.append(collection) + instance[:] = members + self.log.debug(instance.data) diff --git a/pype/plugins/blender/publish/extract_animation.py b/pype/plugins/blender/publish/extract_animation.py new file mode 100644 index 0000000000..dbfe29af83 --- /dev/null +++ b/pype/plugins/blender/publish/extract_animation.py @@ -0,0 +1,47 @@ +import os +import avalon.blender.workio + +import pype.api + + +class ExtractAnimation(pype.api.Extractor): + """Extract as animation.""" + + label = "Animation" + hosts = ["blender"] + families = ["animation"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.blend" + filepath = os.path.join(stagingdir, filename) + + # Perform extraction + self.log.info("Performing extraction..") + + # Just save the file to a temporary location. At least for now it's no + # problem to have (possibly) extra stuff in the file. + avalon.blender.workio.save_file(filepath, copy=True) + # + # # Store reference for integration + # if "files" not in instance.data: + # instance.data["files"] = list() + # + # # instance.data["files"].append(filename) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'blend', + 'ext': 'blend', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + + self.log.info("Extracted instance '%s' to: %s", instance.name, representation) From 0a561382f2dcb716fc2c311c7b94ee712383ff26 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Wed, 12 Feb 2020 12:53:48 +0000 Subject: [PATCH 012/118] Fixed a problem where loaded assets were collected for publishing --- pype/plugins/blender/load/load_animation.py | 4 ++++ pype/plugins/blender/load/load_model.py | 4 ++++ pype/plugins/blender/load/load_rig.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py index 5b527e1717..58a0e94665 100644 --- a/pype/plugins/blender/load/load_animation.py +++ b/pype/plugins/blender/load/load_animation.py @@ -103,6 +103,8 @@ class BlendAnimationLoader(pype.blender.AssetLoader): objects_list.append(obj) + animation_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -226,6 +228,8 @@ class BlendAnimationLoader(pype.blender.AssetLoader): avalon_info.update({"container_name": collection.name}) objects_list.append(obj) + animation_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list collection_metadata["libpath"] = str(libpath) diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index cde4109a7c..40d6c3434c 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -96,6 +96,8 @@ class BlendModelLoader(pype.blender.AssetLoader): objects_list.append(obj) + model_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -204,6 +206,8 @@ class BlendModelLoader(pype.blender.AssetLoader): avalon_info.update({"container_name": collection.name}) objects_list.append(obj) + model_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list collection_metadata["libpath"] = str(libpath) diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 361850c51b..c19717cd82 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -103,6 +103,8 @@ class BlendRigLoader(pype.blender.AssetLoader): objects_list.append(obj) + rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -232,6 +234,8 @@ class BlendRigLoader(pype.blender.AssetLoader): obj.animation_data.action = action + rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list collection_metadata["libpath"] = str(libpath) From 186b29340f33b56b774f8adf066a4794126a49d0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 14:13:19 +0100 Subject: [PATCH 013/118] added master version implementation to outdated check --- pype/lib.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/lib.py b/pype/lib.py index 2235efa2f4..796fe4f11f 100644 --- a/pype/lib.py +++ b/pype/lib.py @@ -173,6 +173,8 @@ def is_latest(representation): """ version = io.find_one({"_id": representation['parent']}) + if version["type"] == "master_version": + return True # Get highest version under the parent highest_version = io.find_one({ From 6ff31dd9a232414b1537126b81caa669f4aea076 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 17:04:12 +0100 Subject: [PATCH 014/118] integrate_new also stores anatomy data to published_representations --- pype/plugins/global/publish/integrate_new.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index a2343ce8a9..18e492796a 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -255,6 +255,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if 'transfers' not in instance.data: instance.data['transfers'] = [] + published_representations = {} for idx, repre in enumerate(instance.data["representations"]): # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) @@ -448,6 +449,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): self.log.debug("__ destination_list: {}".format(destination_list)) instance.data['destination_list'] = destination_list representations.append(representation) + published_representations[repre_id] = { + "representation": representation, + "anatomy_data": template_data + } self.log.debug("__ representations: {}".format(representations)) # Remove old representations if there are any (before insertion of new) @@ -462,7 +467,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): self.log.debug("__ represNAME: {}".format(rep['name'])) self.log.debug("__ represPATH: {}".format(rep['published_path'])) io.insert_many(representations) - instance.data["published_representations"] = representations + instance.data["published_representations"] = ( + published_representations + ) # self.log.debug("Representation: {}".format(representations)) self.log.info("Registered {} items".format(len(representations))) From ceac303221fdc96d66d16e2137ed44dc9e384bbc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 17:04:50 +0100 Subject: [PATCH 015/118] integrate thumbnails do not raise error but log warnings --- pype/plugins/global/publish/integrate_thumbnail.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/integrate_thumbnail.py b/pype/plugins/global/publish/integrate_thumbnail.py index b623fa9072..5361c8aadb 100644 --- a/pype/plugins/global/publish/integrate_thumbnail.py +++ b/pype/plugins/global/publish/integrate_thumbnail.py @@ -21,14 +21,16 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): def process(self, instance): if not os.environ.get("AVALON_THUMBNAIL_ROOT"): - self.log.info("AVALON_THUMBNAIL_ROOT is not set." - " Skipping thumbnail integration.") + self.log.warning( + "AVALON_THUMBNAIL_ROOT is not set." + " Skipping thumbnail integration." + ) return published_repres = instance.data.get("published_representations") if not published_repres: self.log.debug( - "There are not published representation ids on the instance." + "There are not published representations on the instance." ) return @@ -36,10 +38,11 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): anatomy = instance.context.data["anatomy"] if "publish" not in anatomy.templates: - raise AssertionError("Anatomy does not have set publish key!") + self.warning("Anatomy does not have set publish key!") + return if "thumbnail" not in anatomy.templates["publish"]: - raise AssertionError(( + self.warning(( "There is not set \"thumbnail\" template for project \"{}\"" ).format(project_name)) From 20d6893e1dbaa19fcd9282ffccfc039896016222 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 17:05:48 +0100 Subject: [PATCH 016/118] integrate thumbnail uses new anatomy feature --- pype/plugins/global/publish/integrate_thumbnail.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pype/plugins/global/publish/integrate_thumbnail.py b/pype/plugins/global/publish/integrate_thumbnail.py index 5361c8aadb..78929713da 100644 --- a/pype/plugins/global/publish/integrate_thumbnail.py +++ b/pype/plugins/global/publish/integrate_thumbnail.py @@ -92,15 +92,9 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): }) anatomy_filled = anatomy.format(template_data) - final_path = anatomy_filled.get("publish", {}).get("thumbnail") - if not final_path: - raise AssertionError(( - "Anatomy template was not filled with entered data" - "\nTemplate: {} " - "\nData: {}" - ).format(thumbnail_template, str(template_data))) + template_filled = anatomy_filled["publish"]["thumbnail"] - dst_full_path = os.path.normpath(final_path) + dst_full_path = os.path.normpath(str(template_filled)) self.log.debug( "Copying file .. {} -> {}".format(src_full_path, dst_full_path) ) @@ -118,13 +112,14 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): template_data.pop("_id") template_data.pop("thumbnail_root") + repre_context = template_filled.used_values thumbnail_entity = { "_id": thumbnail_id, "type": "thumbnail", "schema": "pype:thumbnail-1.0", "data": { "template": thumbnail_template, - "template_data": template_data + "template_data": repre_context } } # Create thumbnail entity From edf48c01491568f4291de59d47370200cca32ac2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 17:06:57 +0100 Subject: [PATCH 017/118] added required keys for anatomy data to thumbnail context --- .../global/publish/integrate_thumbnail.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/integrate_thumbnail.py b/pype/plugins/global/publish/integrate_thumbnail.py index 78929713da..75755ccb64 100644 --- a/pype/plugins/global/publish/integrate_thumbnail.py +++ b/pype/plugins/global/publish/integrate_thumbnail.py @@ -18,6 +18,10 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder + 0.01 families = ["review"] + required_context_keys = [ + "project", "asset", "task", "subset", "version" + ] + def process(self, instance): if not os.environ.get("AVALON_THUMBNAIL_ROOT"): @@ -45,10 +49,7 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): self.warning(( "There is not set \"thumbnail\" template for project \"{}\"" ).format(project_name)) - - thumbnail_template = anatomy.templates["publish"]["thumbnail"] - - io.install() + return thumb_repre = None for repre in published_repres: @@ -62,6 +63,10 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): ) return + io.install() + + thumbnail_template = anatomy.templates["publish"]["thumbnail"] + version = io.find_one({"_id": thumb_repre["parent"]}) if not version: raise AssertionError( @@ -83,7 +88,7 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): thumbnail_id = ObjectId() # Prepare anatomy template fill data - template_data = copy.deepcopy(thumb_repre["context"]) + template_data = copy.deepcopy(thumb_repre_anatomy_data) template_data.update({ "_id": str(thumbnail_id), "thumbnail_root": os.environ.get("AVALON_THUMBNAIL_ROOT"), @@ -113,6 +118,12 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): template_data.pop("thumbnail_root") repre_context = template_filled.used_values + for key in self.required_context_keys: + value = template_data.get(key) + if not value: + continue + repre_context[key] = template_data[key] + thumbnail_entity = { "_id": thumbnail_id, "type": "thumbnail", From 4a3bf303d4170dceba4331ae9fdfe070f0ad5436 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 17:07:18 +0100 Subject: [PATCH 018/118] integrate thumbnails use new structure of published representations --- pype/plugins/global/publish/integrate_thumbnail.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_thumbnail.py b/pype/plugins/global/publish/integrate_thumbnail.py index 75755ccb64..0bb34eab58 100644 --- a/pype/plugins/global/publish/integrate_thumbnail.py +++ b/pype/plugins/global/publish/integrate_thumbnail.py @@ -52,9 +52,12 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): return thumb_repre = None - for repre in published_repres: + thumb_repre_anatomy_data = None + for repre_info in published_repres.values(): + repre = repre_info["representation"] if repre["name"].lower() == "thumbnail": thumb_repre = repre + thumb_repre_anatomy_data = repre_info["anatomy_data"] break if not thumb_repre: From a3e847bc21ad6498c7f10494115e765ff1e48214 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 18:29:05 +0100 Subject: [PATCH 019/118] store more information into published repres --- pype/plugins/global/publish/integrate_new.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 18e492796a..fe2bcbff33 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -234,6 +234,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): bulk_writes ) + version = io.find_one({"_id": version_id}) + existing_repres = list(io.find({ "parent": version_id, "type": "archived_representation" @@ -451,7 +453,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): representations.append(representation) published_representations[repre_id] = { "representation": representation, - "anatomy_data": template_data + "anatomy_data": template_data, + # TODO prabably should store subset and version to instance + "subset_entity": subset, + "version_entity": version } self.log.debug("__ representations: {}".format(representations)) From 2abe39ef9d8f6e75aa92a3d145f909aeba4d8c16 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 18:29:19 +0100 Subject: [PATCH 020/118] initial commit for instegrate master version --- .../publish/integrate_master_version.py | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 pype/plugins/global/publish/integrate_master_version.py diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py new file mode 100644 index 0000000000..efd01dd07c --- /dev/null +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -0,0 +1,129 @@ +import os +import logging +import shutil + +import errno +import pyblish.api +from avalon import api, io +from avalon.vendor import filelink + + +log = logging.getLogger(__name__) + + +class IntegrateMasterVersion(pyblish.api.InstancePlugin): + label = "Integrate Master Version" + # Must happen after IntegrateNew + order = pyblish.api.IntegratorOrder + 0.1 + + ignored_representation_names = [] + + def process(self, instance): + published_repres = instance.data.get("published_representations") + if not published_repres: + self.log.debug( + "There are not published representations on the instance." + ) + return + + project_name = api.Session["AVALON_PROJECT"] + + # TODO raise error if master not set? + anatomy = instance.context.data["anatomy"] + if "publish" not in anatomy.templates: + self.warning("Anatomy does not have set publish key!") + return + + if "master" not in anatomy.templates["publish"]: + self.warning(( + "There is not set \"master\" template for project \"{}\"" + ).format(project_name)) + return + + version_entity = None + + filtered_repre_ids = [] + for repre_id, repre_info in published_repres.items(): + repre = repre_info["representation"] + if version_entity is None: + version_entity = repre_info.get("version_entity") + + if repre["name"].lower() in self.ignored_representation_names: + filtered_repre_ids.append(repre_id) + + for repre_id in filtered_repre_ids: + published_repres.pop(repre_id, None) + + if not published_repres: + self.log.debug( + "All published representations were filtered by name." + ) + return + + if version_entity is None: + version_entity = ( + self.version_from_representations(published_repres) + ) + + if not version_entity: + self.log.warning("Can't find origin version in database.") + return + + cur_master_version, cur_master_repres = ( + self.current_master_ents(version_entity) + ) + + cur_master_repres_by_name = { + repre["name"].lower(): repre for repre in cur_master_repres + } + + if cur_master_version: + cur_master_version_id = cur_master_version["_id"] + else: + cur_master_version_id = io.ObjectId() + + new_master_version = { + "_id": cur_master_version_id, + "version_id": version_entity["_id"], + "parent": version_entity["parent"], + "type": "master_version", + "schema": "pype:master_version-1.0" + } + + repres_to_replace = {} + for repre_id, repre_info in published_repres.items(): + repre = repre_info["representation"] + repre_name_low = repre["name"].lower() + if repre_name_low in cur_master_repres_by_name: + repres_to_replace[repre_id] = ( + cur_master_repres_by_name.pop(repre_name_low) + ) + + if cur_master_version: + io.replace_one( + {"_id": new_master_version["_id"]}, + new_master_version + ) + else: + io.insert_one(new_master_version) + + def version_from_representations(self, repres): + for repre in repres: + version = io.find_one({"_id": repre["parent"]}) + if version: + return version + + def current_master_ents(self, version): + master_version = io.find_one({ + "parent": version["parent"], + "type": "master_version" + }) + + if not master_version: + return (None, []) + + master_repres = list(io.find({ + "parent": master_version["_id"], + "type": "representation" + })) + return (master_version, master_repres) From c06a4c337beb85df6e3a2bb18538ccb8a36c3f35 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 20 Feb 2020 18:59:30 +0100 Subject: [PATCH 021/118] initial master version schema --- schema/master_version-1.0.json | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 schema/master_version-1.0.json diff --git a/schema/master_version-1.0.json b/schema/master_version-1.0.json new file mode 100644 index 0000000000..173a076537 --- /dev/null +++ b/schema/master_version-1.0.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "title": "pype:master_version-1.0", + "description": "Master version of asset", + + "type": "object", + + "additionalProperties": true, + + "required": [ + "schema", + "type", + "parent" + ], + + "properties": { + "_id": { + "description": "Document's id (database will create it's if not entered)", + "type": "ObjectId", + "example": "592c33475f8c1b064c4d1696" + }, + "schema": { + "description": "The schema associated with this document", + "type": "string", + "enum": ["avalon-core:master_version-3.0", "pype:master_version-3.0"], + "example": "pype:master_version-3.0" + }, + "type": { + "description": "The type of document", + "type": "string", + "enum": ["master_version"], + "example": "master_version" + }, + "parent": { + "description": "Unique identifier to parent document", + "type": "ObjectId", + "example": "592c33475f8c1b064c4d1696" + } + } +} From 34438fcc42a47076ae7bc089eebb99fa02c081f0 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 21 Feb 2020 19:17:29 +0100 Subject: [PATCH 022/118] seems to look like it may work once --- .../publish/integrate_master_version.py | 260 ++++++++++++++++-- 1 file changed, 231 insertions(+), 29 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index efd01dd07c..6991978a24 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -1,10 +1,10 @@ import os +import copy import logging -import shutil -import errno +from pymongo import InsertOne, ReplaceOne import pyblish.api -from avalon import api, io +from avalon import api, io, pipeline from avalon.vendor import filelink @@ -40,13 +40,15 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ).format(project_name)) return - version_entity = None + master_template = anatomy.templates["publish"]["master"] + + src_version_entity = None filtered_repre_ids = [] for repre_id, repre_info in published_repres.items(): repre = repre_info["representation"] - if version_entity is None: - version_entity = repre_info.get("version_entity") + if src_version_entity is None: + src_version_entity = repre_info.get("version_entity") if repre["name"].lower() in self.ignored_representation_names: filtered_repre_ids.append(repre_id) @@ -60,52 +62,252 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ) return - if version_entity is None: - version_entity = ( + if src_version_entity is None: + src_version_entity = ( self.version_from_representations(published_repres) ) - if not version_entity: + if not src_version_entity: self.log.warning("Can't find origin version in database.") return - cur_master_version, cur_master_repres = ( - self.current_master_ents(version_entity) + old_version, old_repres = ( + self.current_master_ents(src_version_entity) ) - cur_master_repres_by_name = { - repre["name"].lower(): repre for repre in cur_master_repres + old_repres_by_name = { + repre["name"].lower(): repre for repre in old_repres } - if cur_master_version: - cur_master_version_id = cur_master_version["_id"] + if old_version: + new_version_id = old_version["_id"] else: - cur_master_version_id = io.ObjectId() + new_version_id = io.ObjectId() new_master_version = { - "_id": cur_master_version_id, - "version_id": version_entity["_id"], - "parent": version_entity["parent"], + "_id": new_version_id, + "version_id": src_version_entity["_id"], + "parent": src_version_entity["parent"], "type": "master_version", "schema": "pype:master_version-1.0" } - repres_to_replace = {} + bulk_writes = [] + + if old_version: + bulk_writes.append( + ReplaceOne( + {"_id": new_master_version["_id"]}, + new_master_version + ) + ) + else: + bulk_writes.append( + InsertOne(new_master_version) + ) + + # Separate old representations into `to replace` and `to delete` + old_repres_to_replace = {} + old_repres_to_delete = {} for repre_id, repre_info in published_repres.items(): repre = repre_info["representation"] repre_name_low = repre["name"].lower() - if repre_name_low in cur_master_repres_by_name: - repres_to_replace[repre_id] = ( - cur_master_repres_by_name.pop(repre_name_low) + if repre_name_low in old_repres_by_name: + old_repres_to_replace[repre_name_low] = ( + old_repres_by_name.pop(repre_name_low) + ) + else: + old_repres_to_delete[repre_name_low] = ( + old_repres_by_name.pop(repre_name_low) ) - if cur_master_version: - io.replace_one( - {"_id": new_master_version["_id"]}, - new_master_version + archived_repres = list(io.find({ + # Check what is type of archived representation + "type": "archived_repsentation", + "parent": new_version_id + })) + archived_repres_by_name = {} + for repre in archived_repres: + repre_name_low = repre["name"].lower() + archived_repres_by_name[repre_name_low] = repre + + self.delete_repre_files(old_repres) + + for repre_id, repre_info in published_repres.items(): + repre = copy.deepcopy(repre_info["representation"]) + repre_name_low = repre["name"].lower() + + repre["parent"] = new_master_version["_id"] + # TODO change repre data and context (new anatomy) + # TODO hardlink files + + # Replace current representation + if repre_name_low in old_repres_to_replace: + old_repre = old_repres_to_replace.pop(repre_name_low) + repre["_id"] = old_repre["_id"] + bulk_writes.append( + ReplaceOne( + {"_id": old_repre["_id"]}, + repre + ) + ) + + # Unarchive representation + elif repre_name_low in archived_repres_by_name: + archived_repre = archived_repres_by_name.pop(repre_name_low) + old_id = archived_repre["old_id"] + repre["_id"] = old_id + bulk_writes.append( + ReplaceOne( + {"old_id": old_id}, + repre + ) + ) + + # Create representation + else: + repre["_id"] = io.ObjectId() + bulk_writes.append( + InsertOne(repre) + ) + + # Archive not replaced old representations + for repre_name_low, repre in old_repres_to_delete.items(): + # TODO delete their files + + # Replace archived representation (This is backup) + # - should not happen to have both repre and archived repre + if repre_name_low in archived_repres_by_name: + archived_repre = archived_repres_by_name.pop(repre_name_low) + repre["old_id"] = repre["_id"] + repre["_id"] = archived_repre["_id"] + repre["type"] = archived_repre["type"] + bulk_writes.append( + ReplaceOne( + {"_id": archived_repre["_id"]}, + repre + ) + ) + + else: + repre["old_id"] = repre["_id"] + repre["_id"] = io.ObjectId() + repre["type"] = "archived_representation" + bulk_writes.append( + InsertOne(repre) + ) + + if bulk_writes: + pass + + def delete_repre_files(self, repres): + if not repres: + return + + frame_splitter = "_-_FRAME_-_" + files_to_delete = [] + for repre in repres: + is_sequence = False + if "frame" in repre["context"]: + repre["context"]["frame"] = frame_splitter + is_sequence = True + + template = repre["data"]["template"] + context = repre["context"] + context["root"] = api.registered_root() + path = pipeline.format_template_with_optional_keys( + context, template ) - else: - io.insert_one(new_master_version) + path = os.path.normpath(path) + if not is_sequence: + if os.path.exists(path): + files_to_delete.append(path) + continue + + dirpath = os.path.dirname(path) + file_start = None + file_end = None + file_items = path.split(frame_splitter) + if len(file_items) == 0: + continue + elif len(file_items) == 1: + if path.startswith(frame_splitter): + file_end = file_items[0] + else: + file_start = file_items[1] + + elif len(file_items) == 2: + file_start, file_end = file_items + + else: + raise ValueError(( + "Representation template has `frame` key " + "more than once inside." + )) + + for file_name in os.listdir(dirpath): + check_name = str(file_name) + if file_start and not check_name.startswith(file_start): + continue + check_name.replace(file_start, "") + + if file_end and not check_name.endswith(file_end): + continue + check_name.replace(file_end, "") + + # File does not have frame + if not check_name: + continue + + files_to_delete.append(os.path.join(dirpath, file_name)) + + renamed_files = [] + failed = False + for file_path in files_to_delete: + # TODO too robust for testing - should be easier in future + _rename_path = file_path + ".BACKUP" + rename_path = None + max_index = 200 + cur_index = 1 + while True: + if max_index >= cur_index: + raise Exception(( + "Max while loop index reached! Can't make backup" + " for previous master version." + )) + break + + if not os.path.exists(_rename_path): + rename_path = _rename_path + break + + try: + os.remove(_rename_path) + except Exception: + _rename_path = file_path + ".BACKUP{}".format( + str(cur_index) + ) + cur_index += 1 + + try: + args = (file_path, rename_path) + os.rename(*args) + renamed_files.append(args) + except Exception: + failed = True + break + + if failed: + for dst_name, src_name in renamed_files: + os.rename(src_name, dst_name) + + raise AssertionError(( + "Could not create master version because it is not possible" + " to replace current master files." + )) + + for _, renamed_path in renamed_files: + os.remove(renamed_path) def version_from_representations(self, repres): for repre in repres: From 36c35dbf8433be8600eb9041c0d9e4d9c2fc8953 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 14:06:41 +0100 Subject: [PATCH 023/118] store all published files per representation --- pype/plugins/global/publish/integrate_new.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index fe2bcbff33..8ef027bb93 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -259,6 +259,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): published_representations = {} for idx, repre in enumerate(instance.data["representations"]): + published_files = [] + # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) if intent is not None: @@ -364,16 +366,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): self.log.debug("source: {}".format(src)) instance.data["transfers"].append([src, dst]) + published_files.append(dst) + # for adding first frame into db if not dst_start_frame: dst_start_frame = dst_padding - dst = "{0}{1}{2}".format( - dst_head, - dst_start_frame, - dst_tail).replace("..", ".") - repre['published_path'] = self.unc_convert(dst) - else: # Single file # _______ @@ -402,9 +400,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): instance.data["transfers"].append([src, dst]) - repre['published_path'] = self.unc_convert(dst) + published_files.append(dst) + self.log.debug("__ dst: {}".format(dst)) + repre["publishedFiles"] = published_files + for key in self.db_representation_context_keys: value = template_data.get(key) if not value: @@ -454,6 +455,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): published_representations[repre_id] = { "representation": representation, "anatomy_data": template_data, + "published_files": published_files, # TODO prabably should store subset and version to instance "subset_entity": subset, "version_entity": version @@ -470,7 +472,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): self.log.debug("__ representations: {}".format(representations)) for rep in instance.data["representations"]: self.log.debug("__ represNAME: {}".format(rep['name'])) - self.log.debug("__ represPATH: {}".format(rep['published_path'])) + self.log.debug("__ represPATH:\n{}".format( + ",\n".join(rep['publishedFiles']) + )) io.insert_many(representations) instance.data["published_representations"] = ( published_representations From 2a128b0956f23a03cc844b8d8e7fbf379c7ac7bd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 14:06:58 +0100 Subject: [PATCH 024/118] added first version of file mapping --- .../publish/integrate_master_version.py | 67 +++++++++++++++++-- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 6991978a24..a93226ae18 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -1,6 +1,7 @@ import os import copy import logging +import clique from pymongo import InsertOne, ReplaceOne import pyblish.api @@ -40,8 +41,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ).format(project_name)) return - master_template = anatomy.templates["publish"]["master"] - src_version_entity = None filtered_repre_ids = [] @@ -133,13 +132,21 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): self.delete_repre_files(old_repres) - for repre_id, repre_info in published_repres.items(): - repre = copy.deepcopy(repre_info["representation"]) - repre_name_low = repre["name"].lower() + master_template = anatomy.templates["publish"]["master"] + src_to_dst_file_paths = [] + for repre_id, repre_info in published_repres.items(): + + # Skip if new repre does not have published repre files + published_files = repre_info["published_files"] + if len(published_files) == 0: + continue + + # Prepare new repre + repre = copy.deepcopy(repre_info["representation"]) repre["parent"] = new_master_version["_id"] - # TODO change repre data and context (new anatomy) - # TODO hardlink files + + repre_name_low = repre["name"].lower() # Replace current representation if repre_name_low in old_repres_to_replace: @@ -171,6 +178,52 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): InsertOne(repre) ) + # TODO change repre data and context (new anatomy) + # TODO hardlink files + + # Prepare anatomy data + anatomy_data = repre_info["anatomy_data"] + anatomy_data.pop("version", None) + + if len(published_files) == 1: + anatomy_filled = anatomy.format(anatomy_data) + template_filled = anatomy_filled["publish"]["master"] + src_to_dst_file_paths.append( + (published_files[0], template_filled) + ) + continue + + collections, remainders = clique.assemble(published_files) + if remainders or not collections or len(collections) > 1: + raise Exception(( + "Integrity error. Files of published representation" + " is combination of frame collections and single files." + )) + + src_col = collections[0] + + # Get filled path to repre context + anatomy_filled = anatomy.format(anatomy_data) + template_filled = anatomy_filled["publish"]["master"] + + # Get head and tail for collection + frame_splitter = "_-_FRAME_SPLIT_-_" + anatomy_data["frame"] = frame_splitter + _anatomy_filled = anatomy.format(anatomy_data) + _template_filled = _anatomy_filled["publish"]["master"] + head, tail = _template_filled.split(frame_splitter) + padding = ( + anatomy.templates["render"]["padding"] + ) + + dst_col = clique.Collection(head=head, padding=padding, tail=tail) + dst_col.indexes.clear() + dst_col.indexes.update(src_col.indexes) + for src_file, dst_file in zip(src_col, dst_col): + src_to_dst_file_paths.append( + (src_file, dst_file) + ) + # Archive not replaced old representations for repre_name_low, repre in old_repres_to_delete.items(): # TODO delete their files From 1a3463c78c5f10f013766991142bbd94708077d4 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 16:40:42 +0100 Subject: [PATCH 025/118] representation context and data are replaced with new data --- .../publish/integrate_master_version.py | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index a93226ae18..b508404d77 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -2,6 +2,7 @@ import os import copy import logging import clique +import errno from pymongo import InsertOne, ReplaceOne import pyblish.api @@ -18,6 +19,10 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): order = pyblish.api.IntegratorOrder + 0.1 ignored_representation_names = [] + db_representation_context_keys = [ + "project", "asset", "task", "subset", "representation", + "family", "hierarchy", "task", "username" + ] def process(self, instance): published_repres = instance.data.get("published_representations") @@ -142,9 +147,34 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): if len(published_files) == 0: continue + # Prepare anatomy data + anatomy_data = repre_info["anatomy_data"] + anatomy_data.pop("version", None) + + # Get filled path to repre context + anatomy_filled = anatomy.format(anatomy_data) + template_filled = anatomy_filled["publish"]["master"] + + repre_data = { + "path": str(template_filled), + "template": master_template + } + repre_context = template_filled.used_values + for key in self.db_representation_context_keys: + if ( + key in repre_context or + key not in anatomy_data + ): + continue + + repre_context[key] = anatomy_data[key] + + # TODO change repre data and context (new anatomy) # Prepare new repre repre = copy.deepcopy(repre_info["representation"]) repre["parent"] = new_master_version["_id"] + repre["context"] = repre_context + repre["data"] = repre_data repre_name_low = repre["name"].lower() @@ -178,16 +208,8 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): InsertOne(repre) ) - # TODO change repre data and context (new anatomy) # TODO hardlink files - - # Prepare anatomy data - anatomy_data = repre_info["anatomy_data"] - anatomy_data.pop("version", None) - if len(published_files) == 1: - anatomy_filled = anatomy.format(anatomy_data) - template_filled = anatomy_filled["publish"]["master"] src_to_dst_file_paths.append( (published_files[0], template_filled) ) @@ -202,10 +224,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): src_col = collections[0] - # Get filled path to repre context - anatomy_filled = anatomy.format(anatomy_data) - template_filled = anatomy_filled["publish"]["master"] - # Get head and tail for collection frame_splitter = "_-_FRAME_SPLIT_-_" anatomy_data["frame"] = frame_splitter From 84dceb42afda2753a74f9e86b5b1aa10aa748b0a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 16:41:04 +0100 Subject: [PATCH 026/118] added reate hardlink and path root checker --- .../publish/integrate_master_version.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index b508404d77..1ec0bd00dd 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -242,6 +242,10 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): (src_file, dst_file) ) + # TODO should we *only* create hardlinks? + for src_path, dst_path in src_to_dst_file_paths: + self.create_hardlink(src_path, dst_path) + # Archive not replaced old representations for repre_name_low, repre in old_repres_to_delete.items(): # TODO delete their files @@ -271,6 +275,83 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): if bulk_writes: pass + def create_hardlink(self, src_path, dst_path): + dst_path = self.path_root_check(dst_path) + src_path = self.path_root_check(src_path) + + dirname = os.path.dirname(dst_path) + + try: + os.makedirs(dirname) + except OSError as exc: + if exc.errno != errno.EEXIST: + self.log.error("An unexpected error occurred.", exc_info=True) + raise + + filelink.create(src_path, dst_path, filelink.HARDLINK) + + def path_root_check(self, path): + normalized_path = os.path.normpath(path) + forward_slash_path = normalized_path.replace("\\", "/") + + drive, _path = os.path.splitdrive(normalized_path) + if os.path.exists(drive + "/"): + self.log.debug( + "Drive \"{}\" exist. Nothing to change.".format(drive) + ) + return normalized_path + + path_env_key = "PYPE_STUDIO_PROJECTS_PATH" + mount_env_key = "PYPE_STUDIO_PROJECTS_MOUNT" + missing_envs = [] + if path_env_key not in os.environ: + missing_envs.append(path_env_key) + + if mount_env_key not in os.environ: + missing_envs.append(mount_env_key) + + if missing_envs: + _add_s = "" + if len(missing_envs) > 1: + _add_s = "s" + + self.log.warning(( + "Can't replace MOUNT drive path to UNC path due to missing" + " environment variable{}: `{}`. This may cause issues during" + " publishing process." + ).format(_add_s, ", ".join(missing_envs))) + + return normalized_path + + unc_root = os.environ[path_env_key].replace("\\", "/") + mount_root = os.environ[mount_env_key].replace("\\", "/") + + # --- Remove slashes at the end of mount and unc roots --- + while unc_root.endswith("/"): + unc_root = unc_root[:-1] + + while mount_root.endswith("/"): + mount_root = mount_root[:-1] + # --- + + if forward_slash_path.startswith(unc_root): + self.log.debug(( + "Path already starts with UNC root: \"{}\"" + ).format(unc_root)) + return normalized_path + + if not forward_slash_path.startswith(mount_root): + self.log.warning(( + "Path do not start with MOUNT root \"{}\" " + "set in environment variable \"{}\"" + ).format(unc_root, mount_env_key)) + return normalized_path + + # Replace Mount root with Unc root + path = unc_root + forward_slash_path[len(mount_root):] + + return os.path.normpath(path) + def delete_repre_files(self, repres): if not repres: return From 12e95ef32c883f44f0b1136e3023065a97b469dc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 17:24:40 +0100 Subject: [PATCH 027/118] publihsed_path moved back due to integrity errors connected with removing --- pype/plugins/global/publish/integrate_new.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 779a498451..f8cde10aed 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -373,6 +373,13 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if not dst_start_frame: dst_start_frame = dst_padding + dst = "{0}{1}{2}".format( + dst_head, + dst_start_frame, + dst_tail + ).replace("..", ".") + repre['published_path'] = self.unc_convert(dst) + else: # Single file # _______ @@ -402,7 +409,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): instance.data["transfers"].append([src, dst]) published_files.append(dst) - + repre['published_path'] = self.unc_convert(dst) self.log.debug("__ dst: {}".format(dst)) repre["publishedFiles"] = published_files @@ -473,9 +480,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): self.log.debug("__ representations: {}".format(representations)) for rep in instance.data["representations"]: self.log.debug("__ represNAME: {}".format(rep['name'])) - self.log.debug("__ represPATH:\n{}".format( - ",\n".join(rep['publishedFiles']) - )) + self.log.debug("__ represPATH: {}".format(rep['published_path'])) io.insert_many(representations) instance.data["published_representations"] = ( published_representations From 875ca5cd6fd4f0da67b2dba80897dd27b6505bda Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Mon, 24 Feb 2020 16:26:10 +0000 Subject: [PATCH 028/118] Extraction for blend files is now handled by a single class --- .../blender/publish/collect_current_file.py | 2 + ...{extract_animation.py => extract_blend.py} | 93 +++++++++---------- pype/plugins/blender/publish/extract_model.py | 47 ---------- pype/plugins/blender/publish/extract_rig.py | 47 ---------- 4 files changed, 48 insertions(+), 141 deletions(-) rename pype/plugins/blender/publish/{extract_animation.py => extract_blend.py} (86%) delete mode 100644 pype/plugins/blender/publish/extract_model.py delete mode 100644 pype/plugins/blender/publish/extract_rig.py diff --git a/pype/plugins/blender/publish/collect_current_file.py b/pype/plugins/blender/publish/collect_current_file.py index a097c72047..926d290b31 100644 --- a/pype/plugins/blender/publish/collect_current_file.py +++ b/pype/plugins/blender/publish/collect_current_file.py @@ -14,3 +14,5 @@ class CollectBlenderCurrentFile(pyblish.api.ContextPlugin): """Inject the current working file""" current_file = bpy.data.filepath context.data['currentFile'] = current_file + + assert current_file != '', "Current file is empty. Save the file before continuing." diff --git a/pype/plugins/blender/publish/extract_animation.py b/pype/plugins/blender/publish/extract_blend.py similarity index 86% rename from pype/plugins/blender/publish/extract_animation.py rename to pype/plugins/blender/publish/extract_blend.py index dbfe29af83..7e11e9ef8d 100644 --- a/pype/plugins/blender/publish/extract_animation.py +++ b/pype/plugins/blender/publish/extract_blend.py @@ -1,47 +1,46 @@ -import os -import avalon.blender.workio - -import pype.api - - -class ExtractAnimation(pype.api.Extractor): - """Extract as animation.""" - - label = "Animation" - hosts = ["blender"] - families = ["animation"] - optional = True - - def process(self, instance): - # Define extract output file path - - stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.blend" - filepath = os.path.join(stagingdir, filename) - - # Perform extraction - self.log.info("Performing extraction..") - - # Just save the file to a temporary location. At least for now it's no - # problem to have (possibly) extra stuff in the file. - avalon.blender.workio.save_file(filepath, copy=True) - # - # # Store reference for integration - # if "files" not in instance.data: - # instance.data["files"] = list() - # - # # instance.data["files"].append(filename) - - if "representations" not in instance.data: - instance.data["representations"] = [] - - representation = { - 'name': 'blend', - 'ext': 'blend', - 'files': filename, - "stagingDir": stagingdir, - } - instance.data["representations"].append(representation) - - - self.log.info("Extracted instance '%s' to: %s", instance.name, representation) +import os +import avalon.blender.workio + +import pype.api + + +class ExtractBlend(pype.api.Extractor): + """Extract a blend file.""" + + label = "Extract Blend" + hosts = ["blender"] + families = ["animation", "model", "rig"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.blend" + filepath = os.path.join(stagingdir, filename) + + # Perform extraction + self.log.info("Performing extraction..") + + # Just save the file to a temporary location. At least for now it's no + # problem to have (possibly) extra stuff in the file. + avalon.blender.workio.save_file(filepath, copy=True) + # + # # Store reference for integration + # if "files" not in instance.data: + # instance.data["files"] = list() + # + # # instance.data["files"].append(filename) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'blend', + 'ext': 'blend', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + self.log.info("Extracted instance '%s' to: %s", instance.name, representation) diff --git a/pype/plugins/blender/publish/extract_model.py b/pype/plugins/blender/publish/extract_model.py deleted file mode 100644 index 501c4d9d5c..0000000000 --- a/pype/plugins/blender/publish/extract_model.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import avalon.blender.workio - -import pype.api - - -class ExtractModel(pype.api.Extractor): - """Extract as model.""" - - label = "Model" - hosts = ["blender"] - families = ["model"] - optional = True - - def process(self, instance): - # Define extract output file path - - stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.blend" - filepath = os.path.join(stagingdir, filename) - - # Perform extraction - self.log.info("Performing extraction..") - - # Just save the file to a temporary location. At least for now it's no - # problem to have (possibly) extra stuff in the file. - avalon.blender.workio.save_file(filepath, copy=True) - # - # # Store reference for integration - # if "files" not in instance.data: - # instance.data["files"] = list() - # - # # instance.data["files"].append(filename) - - if "representations" not in instance.data: - instance.data["representations"] = [] - - representation = { - 'name': 'blend', - 'ext': 'blend', - 'files': filename, - "stagingDir": stagingdir, - } - instance.data["representations"].append(representation) - - - self.log.info("Extracted instance '%s' to: %s", instance.name, representation) diff --git a/pype/plugins/blender/publish/extract_rig.py b/pype/plugins/blender/publish/extract_rig.py deleted file mode 100644 index 8a3c83d07c..0000000000 --- a/pype/plugins/blender/publish/extract_rig.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import avalon.blender.workio - -import pype.api - - -class ExtractRig(pype.api.Extractor): - """Extract as rig.""" - - label = "Rig" - hosts = ["blender"] - families = ["rig"] - optional = True - - def process(self, instance): - # Define extract output file path - - stagingdir = self.staging_dir(instance) - filename = f"{instance.name}.blend" - filepath = os.path.join(stagingdir, filename) - - # Perform extraction - self.log.info("Performing extraction..") - - # Just save the file to a temporary location. At least for now it's no - # problem to have (possibly) extra stuff in the file. - avalon.blender.workio.save_file(filepath, copy=True) - # - # # Store reference for integration - # if "files" not in instance.data: - # instance.data["files"] = list() - # - # # instance.data["files"].append(filename) - - if "representations" not in instance.data: - instance.data["representations"] = [] - - representation = { - 'name': 'blend', - 'ext': 'blend', - 'files': filename, - "stagingDir": stagingdir, - } - instance.data["representations"].append(representation) - - - self.log.info("Extracted instance '%s' to: %s", instance.name, representation) From 22e8c301467f3f35e1d39daed855b646602c8633 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 17:26:17 +0100 Subject: [PATCH 029/118] fixed old repres to delete variable --- pype/plugins/global/publish/integrate_master_version.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 1ec0bd00dd..3df74f4e28 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -120,10 +120,9 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): old_repres_to_replace[repre_name_low] = ( old_repres_by_name.pop(repre_name_low) ) - else: - old_repres_to_delete[repre_name_low] = ( - old_repres_by_name.pop(repre_name_low) - ) + + if old_repres_by_name: + old_repres_to_delete = old_repres_by_name archived_repres = list(io.find({ # Check what is type of archived representation From f2733c0a1b05bff6afa7b21a00c84beb436ca31d Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Mon, 24 Feb 2020 16:26:40 +0000 Subject: [PATCH 030/118] Implemented extraction to FBX files --- pype/plugins/blender/publish/extract_fbx.py | 71 +++++++++++ .../blender/publish/extract_fbx_animation.py | 118 ++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 pype/plugins/blender/publish/extract_fbx.py create mode 100644 pype/plugins/blender/publish/extract_fbx_animation.py diff --git a/pype/plugins/blender/publish/extract_fbx.py b/pype/plugins/blender/publish/extract_fbx.py new file mode 100644 index 0000000000..95466c1d2b --- /dev/null +++ b/pype/plugins/blender/publish/extract_fbx.py @@ -0,0 +1,71 @@ +import os +import avalon.blender.workio + +import pype.api + +import bpy + +class ExtractFBX(pype.api.Extractor): + """Extract as FBX.""" + + label = "Extract FBX" + hosts = ["blender"] + families = ["model", "rig"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.fbx" + filepath = os.path.join(stagingdir, filename) + + # Perform extraction + self.log.info("Performing extraction..") + + collections = [obj for obj in instance if type(obj) is bpy.types.Collection] + + assert len(collections) == 1, "There should be one and only one collection collected for this asset" + + old_active_layer_collection = bpy.context.view_layer.active_layer_collection + + # Get the layer collection from the collection we need to export. + # This is needed because in Blender you can only set the active + # collection with the layer collection, and there is no way to get + # the layer collection from the collection (but there is the vice versa). + layer_collections = [layer for layer in bpy.context.view_layer.layer_collection.children if layer.collection == collections[0]] + + assert len(layer_collections) == 1 + + bpy.context.view_layer.active_layer_collection = layer_collections[0] + + old_scale = bpy.context.scene.unit_settings.scale_length + + # We set the scale of the scene for the export + bpy.context.scene.unit_settings.scale_length = 0.01 + + # We export the fbx + bpy.ops.export_scene.fbx( + filepath=filepath, + use_active_collection=True, + mesh_smooth_type='FACE', + add_leaf_bones=False + ) + + bpy.context.view_layer.active_layer_collection = old_active_layer_collection + + bpy.context.scene.unit_settings.scale_length = old_scale + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'fbx', + 'ext': 'fbx', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + self.log.info("Extracted instance '%s' to: %s", + instance.name, representation) diff --git a/pype/plugins/blender/publish/extract_fbx_animation.py b/pype/plugins/blender/publish/extract_fbx_animation.py new file mode 100644 index 0000000000..bc088f8bb7 --- /dev/null +++ b/pype/plugins/blender/publish/extract_fbx_animation.py @@ -0,0 +1,118 @@ +import os +import avalon.blender.workio + +import pype.api + +import bpy +import bpy_extras +import bpy_extras.anim_utils + + +class ExtractAnimationFBX(pype.api.Extractor): + """Extract as animation.""" + + label = "Extract FBX" + hosts = ["blender"] + families = ["animation"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.fbx" + filepath = os.path.join(stagingdir, filename) + + # Perform extraction + self.log.info("Performing extraction..") + + collections = [obj for obj in instance if type(obj) is bpy.types.Collection] + + assert len(collections) == 1, "There should be one and only one collection collected for this asset" + + old_active_layer_collection = bpy.context.view_layer.active_layer_collection + + # Get the layer collection from the collection we need to export. + # This is needed because in Blender you can only set the active + # collection with the layer collection, and there is no way to get + # the layer collection from the collection (but there is the vice versa). + layer_collections = [layer for layer in bpy.context.view_layer.layer_collection.children if layer.collection == collections[0]] + + assert len(layer_collections) == 1 + + bpy.context.view_layer.active_layer_collection = layer_collections[0] + + old_scale = bpy.context.scene.unit_settings.scale_length + + # We set the scale of the scene for the export + bpy.context.scene.unit_settings.scale_length = 0.01 + + # We export all the objects in the collection + objects_to_export = collections[0].objects + + object_action_pairs = [] + original_actions = [] + + starting_frames = [] + ending_frames = [] + + # For each object, we make a copy of the current action + for obj in objects_to_export: + + curr_action = obj.animation_data.action + copy_action = curr_action.copy() + + object_action_pairs.append((obj, copy_action)) + original_actions.append(curr_action) + + curr_frame_range = curr_action.frame_range + + starting_frames.append( curr_frame_range[0] ) + ending_frames.append( curr_frame_range[1] ) + + # We compute the starting and ending frames + max_frame = min( starting_frames ) + min_frame = max( ending_frames ) + + # We bake the copy of the current action for each object + bpy_extras.anim_utils.bake_action_objects( + object_action_pairs, + frames=range(int(min_frame), int(max_frame)), + do_object=False, + do_clean=False + ) + + # We export the fbx + bpy.ops.export_scene.fbx( + filepath=filepath, + use_active_collection=True, + bake_anim_use_nla_strips=False, + bake_anim_use_all_actions=False, + add_leaf_bones=False + ) + + bpy.context.view_layer.active_layer_collection = old_active_layer_collection + + bpy.context.scene.unit_settings.scale_length = old_scale + + # We delete the baked action and set the original one back + for i in range(0, len(object_action_pairs)): + + object_action_pairs[i][0].animation_data.action = original_actions[i] + + object_action_pairs[i][1].user_clear() + bpy.data.actions.remove(object_action_pairs[i][1]) + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'fbx', + 'ext': 'fbx', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + self.log.info("Extracted instance '%s' to: %s", + instance.name, representation) From 50bff7fcc0b968c9113d3980e6b656b57d4c32f1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 17:26:45 +0100 Subject: [PATCH 031/118] bulk is actually written to database --- pype/plugins/global/publish/integrate_master_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 3df74f4e28..ea97f3d779 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -272,7 +272,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ) if bulk_writes: - pass + io._database[io.Session["AVALON_PROJECT"]].bulk_write(bulk_writes) def create_hardlink(self, src_path, dst_path): dst_path = self.path_root_check(dst_path) From 1c098196e69496aa8237b06ebcd36fe34d4db74b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 17:46:05 +0100 Subject: [PATCH 032/118] added few debug logs --- .../publish/integrate_master_version.py | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index ea97f3d779..390c86afce 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -25,6 +25,11 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ] def process(self, instance): + self.log.debug( + "Integrate of Master version for subset `{}` begins.".format( + instance.data.get("subset", str(instance)) + ) + ) published_repres = instance.data.get("published_representations") if not published_repres: self.log.debug( @@ -37,15 +42,21 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # TODO raise error if master not set? anatomy = instance.context.data["anatomy"] if "publish" not in anatomy.templates: - self.warning("Anatomy does not have set publish key!") + self.log.warning("Anatomy does not have set publish key!") return if "master" not in anatomy.templates["publish"]: - self.warning(( + self.log.warning(( "There is not set \"master\" template for project \"{}\"" ).format(project_name)) return + master_template = anatomy.templates["publish"]["master"] + + self.log.debug("`Master` template check was successful. `{}`".format( + master_template + )) + src_version_entity = None filtered_repre_ids = [] @@ -55,6 +66,11 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): src_version_entity = repre_info.get("version_entity") if repre["name"].lower() in self.ignored_representation_names: + self.log.debug( + "Filtering representation with name: `{}`".format( + repre["name"].lower() + ) + ) filtered_repre_ids.append(repre_id) for repre_id in filtered_repre_ids: @@ -67,12 +83,19 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): return if src_version_entity is None: + self.log.debug(( + "Published version entity was not sent in representation data." + " Querying entity from database." + )) src_version_entity = ( self.version_from_representations(published_repres) ) if not src_version_entity: - self.log.warning("Can't find origin version in database.") + self.log.warning(( + "Can't find origin version in database." + " Skipping Master version publish." + )) return old_version, old_repres = ( @@ -99,6 +122,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): bulk_writes = [] if old_version: + self.log.debug("Replacing old master version.") bulk_writes.append( ReplaceOne( {"_id": new_master_version["_id"]}, @@ -106,6 +130,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ) ) else: + self.log.debug("Creating first master version.") bulk_writes.append( InsertOne(new_master_version) ) @@ -282,11 +307,17 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): try: os.makedirs(dirname) + self.log.debug("Folder created: \"{}\"".format(dirname)) except OSError as exc: if exc.errno != errno.EEXIST: self.log.error("An unexpected error occurred.", exc_info=True) raise + self.log.debug("Folder already exists: \"{}\"".format(dirname)) + + self.log.debug("Copying file \"{}\" to \"{}\"".format( + src_path, dst_path + )) filelink.create(src_path, dst_path, filelink.HARDLINK) def path_root_check(self, path): From 03b252556de70a121a273b5acc0474b33c035327 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:19:12 +0100 Subject: [PATCH 033/118] keep only one master_template variable --- pype/plugins/global/publish/integrate_master_version.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 390c86afce..dc15ff2d8d 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -161,8 +161,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): self.delete_repre_files(old_repres) - master_template = anatomy.templates["publish"]["master"] - src_to_dst_file_paths = [] for repre_id, repre_info in published_repres.items(): @@ -193,7 +191,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre_context[key] = anatomy_data[key] - # TODO change repre data and context (new anatomy) # Prepare new repre repre = copy.deepcopy(repre_info["representation"]) repre["parent"] = new_master_version["_id"] @@ -232,7 +229,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): InsertOne(repre) ) - # TODO hardlink files + # Prepare paths of source and destination files if len(published_files) == 1: src_to_dst_file_paths.append( (published_files[0], template_filled) From 236da4f8849042b65d0192e7f0749638891b3870 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:21:54 +0100 Subject: [PATCH 034/118] fixed backup file handling --- .../publish/integrate_master_version.py | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index dc15ff2d8d..d98767cbfd 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -263,7 +263,9 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): (src_file, dst_file) ) + # Copy(hardlink) paths of source and destination files # TODO should we *only* create hardlinks? + # TODO less logs about drives for src_path, dst_path in src_to_dst_file_paths: self.create_hardlink(src_path, dst_path) @@ -443,17 +445,24 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): renamed_files = [] failed = False for file_path in files_to_delete: - # TODO too robust for testing - should be easier in future - _rename_path = file_path + ".BACKUP" - rename_path = None - max_index = 200 - cur_index = 1 - while True: - if max_index >= cur_index: - raise Exception(( + self.log.debug( + "Preparing file for deletion: `{}`".format(file_path) + ) + rename_path = file_path + ".BACKUP" + + max_index = 10 + cur_index = 0 + _rename_path = None + while os.path.exists(rename_path): + if _rename_path is None: + _rename_path = rename_path + + if cur_index >= max_index: + self.log.warning(( "Max while loop index reached! Can't make backup" " for previous master version." )) + failed = True break if not os.path.exists(_rename_path): @@ -462,21 +471,41 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): try: os.remove(_rename_path) + self.log.debug( + "Deleted old backup file: \"{}\"".format(_rename_path) + ) except Exception: + self.log.warning( + "Could not delete old backup file \"{}\".".format( + _rename_path + ), + exc_info=True + ) _rename_path = file_path + ".BACKUP{}".format( str(cur_index) ) cur_index += 1 + # Skip if any already failed + if failed: + break + try: args = (file_path, rename_path) os.rename(*args) renamed_files.append(args) except Exception: + self.log.warning( + "Could not rename file `{}` to `{}`".format( + file_path, rename_path + ), + exc_info=True + ) failed = True break if failed: + # Rename back old renamed files for dst_name, src_name in renamed_files: os.rename(src_name, dst_name) From 390acf4eb394496486b7127b1fa9e75d01fececc Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:24:42 +0100 Subject: [PATCH 035/118] addde important TODO --- pype/plugins/global/publish/integrate_master_version.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index d98767cbfd..be6602ac13 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -317,6 +317,8 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): self.log.debug("Copying file \"{}\" to \"{}\"".format( src_path, dst_path )) + # TODO check if file exists!!! + # - uncomplete publish may cause that file already exists filelink.create(src_path, dst_path, filelink.HARDLINK) def path_root_check(self, path): From ae1b102f4756545ee670972e059c2db96603d712 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:39:52 +0100 Subject: [PATCH 036/118] begin and ending logs have 3 symbol start --- .../publish/integrate_master_version.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index be6602ac13..de2cedc2d7 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -26,14 +26,14 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): def process(self, instance): self.log.debug( - "Integrate of Master version for subset `{}` begins.".format( + "--- Integration of Master version for subset `{}` begins.".format( instance.data.get("subset", str(instance)) ) ) published_repres = instance.data.get("published_representations") if not published_repres: self.log.debug( - "There are not published representations on the instance." + "*** There are not published representations on the instance." ) return @@ -42,12 +42,12 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # TODO raise error if master not set? anatomy = instance.context.data["anatomy"] if "publish" not in anatomy.templates: - self.log.warning("Anatomy does not have set publish key!") + self.log.warning("!!! Anatomy does not have set publish key!") return if "master" not in anatomy.templates["publish"]: self.log.warning(( - "There is not set \"master\" template for project \"{}\"" + "!!! There is not set \"master\" template for project \"{}\"" ).format(project_name)) return @@ -78,7 +78,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): if not published_repres: self.log.debug( - "All published representations were filtered by name." + "*** All published representations were filtered by name." ) return @@ -93,7 +93,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): if not src_version_entity: self.log.warning(( - "Can't find origin version in database." + "!!! Can't find origin version in database." " Skipping Master version publish." )) return @@ -241,7 +241,8 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): raise Exception(( "Integrity error. Files of published representation" " is combination of frame collections and single files." - )) + "Collections: `{}` Single files: `{}`" + ).format(str(collections), str(remainders))) src_col = collections[0] @@ -266,13 +267,12 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # Copy(hardlink) paths of source and destination files # TODO should we *only* create hardlinks? # TODO less logs about drives + # TODO should we keep files for deletion until this is successful? for src_path, dst_path in src_to_dst_file_paths: self.create_hardlink(src_path, dst_path) # Archive not replaced old representations for repre_name_low, repre in old_repres_to_delete.items(): - # TODO delete their files - # Replace archived representation (This is backup) # - should not happen to have both repre and archived repre if repre_name_low in archived_repres_by_name: @@ -298,6 +298,12 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): if bulk_writes: io._database[io.Session["AVALON_PROJECT"]].bulk_write(bulk_writes) + self.log.debug(( + "--- End of Master version integration for subset `{}`." + ).format( + instance.data.get("subset", str(instance)) + )) + def create_hardlink(self, src_path, dst_path): dst_path = self.path_root_check(dst_path) src_path = self.path_root_check(src_path) From e5108a6e37e66fa3ffcf78df7a4872b1a0f517cd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:40:40 +0100 Subject: [PATCH 037/118] reduced logs about drive remapping --- .../publish/integrate_master_version.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index de2cedc2d7..a32c94b43e 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -264,9 +264,10 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): (src_file, dst_file) ) + self.path_checks = [] + # Copy(hardlink) paths of source and destination files # TODO should we *only* create hardlinks? - # TODO less logs about drives # TODO should we keep files for deletion until this is successful? for src_path, dst_path in src_to_dst_file_paths: self.create_hardlink(src_path, dst_path) @@ -333,9 +334,13 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): drive, _path = os.path.splitdrive(normalized_path) if os.path.exists(drive + "/"): - self.log.debug( - "Drive \"{}\" exist. Nothing to change.".format(drive) - ) + key = "drive_check{}".format(drive) + if key not in self.path_checks: + self.log.debug( + "Drive \"{}\" exist. Nothing to change.".format(drive) + ) + self.path_checks.append(key) + return normalized_path path_env_key = "PYPE_STUDIO_PROJECTS_PATH" @@ -348,15 +353,18 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): missing_envs.append(mount_env_key) if missing_envs: - _add_s = "" - if len(missing_envs) > 1: - _add_s = "s" + key = "missing_envs" + if key not in self.path_checks: + self.path_checks.append(key) + _add_s = "" + if len(missing_envs) > 1: + _add_s = "s" - self.log.warning(( - "Can't replace MOUNT drive path to UNC path due to missing" - " environment variable{}: `{}`. This may cause issues during" - " publishing process." - ).format(_add_s, ", ".join(missing_envs))) + self.log.warning(( + "Can't replace MOUNT drive path to UNC path due to missing" + " environment variable{}: `{}`. This may cause issues" + " during publishing process." + ).format(_add_s, ", ".join(missing_envs))) return normalized_path From 58ca4399a1db3264b1af054a80c2d2a96a44c5ce Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 24 Feb 2020 18:45:24 +0100 Subject: [PATCH 038/118] removed unused log --- pype/plugins/global/publish/integrate_master_version.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index a32c94b43e..f767a312d6 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -10,20 +10,21 @@ from avalon import api, io, pipeline from avalon.vendor import filelink -log = logging.getLogger(__name__) - - class IntegrateMasterVersion(pyblish.api.InstancePlugin): label = "Integrate Master Version" # Must happen after IntegrateNew order = pyblish.api.IntegratorOrder + 0.1 + # Can specify representation names that will be ignored (lower case) ignored_representation_names = [] db_representation_context_keys = [ "project", "asset", "task", "subset", "representation", "family", "hierarchy", "task", "username" ] - + # TODO add family filtering + # QUESTION/TODO this process should happen on server if crashed due to + # permissions error on files (files were used or user didn't have perms) + # *but all other plugins must be sucessfully completed def process(self, instance): self.log.debug( "--- Integration of Master version for subset `{}` begins.".format( From 4e833a4f44153988fe90e342c15d72873def3b89 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 25 Feb 2020 14:05:45 +0100 Subject: [PATCH 039/118] master version do not rename each file but whole pusblish folder, also is used master.path anatomy instead of publish.master --- .../publish/integrate_master_version.py | 551 +++++++++--------- 1 file changed, 283 insertions(+), 268 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index f767a312d6..42c93db7e9 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -1,8 +1,8 @@ import os import copy -import logging import clique import errno +import shutil from pymongo import InsertOne, ReplaceOne import pyblish.api @@ -25,6 +25,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # QUESTION/TODO this process should happen on server if crashed due to # permissions error on files (files were used or user didn't have perms) # *but all other plugins must be sucessfully completed + def process(self, instance): self.log.debug( "--- Integration of Master version for subset `{}` begins.".format( @@ -42,24 +43,25 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # TODO raise error if master not set? anatomy = instance.context.data["anatomy"] - if "publish" not in anatomy.templates: - self.log.warning("!!! Anatomy does not have set publish key!") + if "master" not in anatomy.templates: + self.log.warning("!!! Anatomy does not have set `master` key!") return - if "master" not in anatomy.templates["publish"]: + if "path" not in anatomy.templates["master"]: self.log.warning(( - "!!! There is not set \"master\" template for project \"{}\"" + "!!! There is not set `path` template in `master` anatomy" + " for project \"{}\"." ).format(project_name)) return - master_template = anatomy.templates["publish"]["master"] - + master_template = anatomy.templates["master"]["path"] self.log.debug("`Master` template check was successful. `{}`".format( master_template )) - src_version_entity = None + master_publish_dir = self.get_publish_dir(instance) + src_version_entity = None filtered_repre_ids = [] for repre_id, repre_info in published_repres.items(): repre = repre_info["representation"] @@ -99,6 +101,47 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): )) return + all_copied_files = [] + transfers = instance.data.get("transfers", list()) + for src, dst in transfers: + dst = os.path.normpath(dst) + if dst not in all_copied_files: + all_copied_files.append(dst) + + hardlinks = instance.data.get("hardlinks", list()) + for src, dst in hardlinks: + dst = os.path.normpath(dst) + if dst not in all_copied_files: + all_copied_files.append(dst) + + all_repre_file_paths = [] + for repre_info in published_repres: + published_files = repre_info.get("published_files") or [] + for file_path in published_files: + file_path = os.path.normpath(file_path) + if file_path not in all_repre_file_paths: + all_repre_file_paths.append(file_path) + + # TODO this is not best practice of getting resources for publish + # WARNING due to this we must remove all files from master publish dir + instance_publish_dir = os.path.normpath( + instance.data["publishDir"] + ) + other_file_paths_mapping = [] + for file_path in all_copied_files: + # Check if it is from publishDir + if not file_path.startswith(instance_publish_dir): + continue + + if file_path in all_repre_file_paths: + continue + + dst_filepath = file_path.replace( + instance_publish_dir, master_publish_dir + ) + other_file_paths_mapping.append((file_path, dst_filepath)) + + # Current version old_version, old_repres = ( self.current_master_ents(src_version_entity) ) @@ -120,6 +163,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): "schema": "pype:master_version-1.0" } + # Don't make changes in database until everything is O.K. bulk_writes = [] if old_version: @@ -160,145 +204,212 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre_name_low = repre["name"].lower() archived_repres_by_name[repre_name_low] = repre - self.delete_repre_files(old_repres) + if os.path.exists(master_publish_dir): + backup_master_publish_dir = master_publish_dir + ".BACKUP" + max_idx = 10 + idx = 0 + _backup_master_publish_dir = backup_master_publish_dir + while os.path.exists(_backup_master_publish_dir): + self.log.debug(( + "Backup folder already exists." + " Trying to remove \"{}\"" + ).format(_backup_master_publish_dir)) - src_to_dst_file_paths = [] - for repre_id, repre_info in published_repres.items(): + try: + shutil.rmtree(_backup_master_publish_dir) + backup_master_publish_dir = _backup_master_publish_dir + break + except Exception: + self.log.info(( + "Could not remove previous backup folder." + " Trying to add index to folder name" + )) - # Skip if new repre does not have published repre files - published_files = repre_info["published_files"] - if len(published_files) == 0: - continue + _backup_master_publish_dir = ( + backup_master_publish_dir + str(idx) + ) + if not os.path.exists(_backup_master_publish_dir): + backup_master_publish_dir = _backup_master_publish_dir + break - # Prepare anatomy data - anatomy_data = repre_info["anatomy_data"] - anatomy_data.pop("version", None) + if idx > max_idx: + raise AssertionError(( + "Backup folders are fully occupied to max index \"{}\"" + ).format(max_idx)) + break - # Get filled path to repre context - anatomy_filled = anatomy.format(anatomy_data) - template_filled = anatomy_filled["publish"]["master"] + idx += 1 - repre_data = { - "path": str(template_filled), - "template": master_template - } - repre_context = template_filled.used_values - for key in self.db_representation_context_keys: - if ( - key in repre_context or - key not in anatomy_data - ): + self.log.debug("Backup folder path is \"{}\"".format( + backup_master_publish_dir + )) + try: + os.rename(master_publish_dir, backup_master_publish_dir) + except PermissionError: + raise AssertionError(( + "Could not create master version because it is not" + " possible to replace current master files." + )) + try: + src_to_dst_file_paths = [] + for repre_id, repre_info in published_repres.items(): + + # Skip if new repre does not have published repre files + published_files = repre_info["published_files"] + if len(published_files) == 0: continue - repre_context[key] = anatomy_data[key] + # Prepare anatomy data + anatomy_data = repre_info["anatomy_data"] + anatomy_data.pop("version", None) - # Prepare new repre - repre = copy.deepcopy(repre_info["representation"]) - repre["parent"] = new_master_version["_id"] - repre["context"] = repre_context - repre["data"] = repre_data + # Get filled path to repre context + anatomy_filled = anatomy.format(anatomy_data) + template_filled = anatomy_filled["publish"]["master"] - repre_name_low = repre["name"].lower() + repre_data = { + "path": str(template_filled), + "template": master_template + } + repre_context = template_filled.used_values + for key in self.db_representation_context_keys: + if ( + key in repre_context or + key not in anatomy_data + ): + continue - # Replace current representation - if repre_name_low in old_repres_to_replace: - old_repre = old_repres_to_replace.pop(repre_name_low) - repre["_id"] = old_repre["_id"] - bulk_writes.append( - ReplaceOne( - {"_id": old_repre["_id"]}, - repre + repre_context[key] = anatomy_data[key] + + # Prepare new repre + repre = copy.deepcopy(repre_info["representation"]) + repre["parent"] = new_master_version["_id"] + repre["context"] = repre_context + repre["data"] = repre_data + + repre_name_low = repre["name"].lower() + + # Replace current representation + if repre_name_low in old_repres_to_replace: + old_repre = old_repres_to_replace.pop(repre_name_low) + repre["_id"] = old_repre["_id"] + bulk_writes.append( + ReplaceOne( + {"_id": old_repre["_id"]}, + repre + ) ) - ) - # Unarchive representation - elif repre_name_low in archived_repres_by_name: - archived_repre = archived_repres_by_name.pop(repre_name_low) - old_id = archived_repre["old_id"] - repre["_id"] = old_id - bulk_writes.append( - ReplaceOne( - {"old_id": old_id}, - repre + # Unarchive representation + elif repre_name_low in archived_repres_by_name: + archived_repre = archived_repres_by_name.pop( + repre_name_low ) - ) - - # Create representation - else: - repre["_id"] = io.ObjectId() - bulk_writes.append( - InsertOne(repre) - ) - - # Prepare paths of source and destination files - if len(published_files) == 1: - src_to_dst_file_paths.append( - (published_files[0], template_filled) - ) - continue - - collections, remainders = clique.assemble(published_files) - if remainders or not collections or len(collections) > 1: - raise Exception(( - "Integrity error. Files of published representation" - " is combination of frame collections and single files." - "Collections: `{}` Single files: `{}`" - ).format(str(collections), str(remainders))) - - src_col = collections[0] - - # Get head and tail for collection - frame_splitter = "_-_FRAME_SPLIT_-_" - anatomy_data["frame"] = frame_splitter - _anatomy_filled = anatomy.format(anatomy_data) - _template_filled = _anatomy_filled["publish"]["master"] - head, tail = _template_filled.split(frame_splitter) - padding = ( - anatomy.templates["render"]["padding"] - ) - - dst_col = clique.Collection(head=head, padding=padding, tail=tail) - dst_col.indexes.clear() - dst_col.indexes.update(src_col.indexes) - for src_file, dst_file in zip(src_col, dst_col): - src_to_dst_file_paths.append( - (src_file, dst_file) - ) - - self.path_checks = [] - - # Copy(hardlink) paths of source and destination files - # TODO should we *only* create hardlinks? - # TODO should we keep files for deletion until this is successful? - for src_path, dst_path in src_to_dst_file_paths: - self.create_hardlink(src_path, dst_path) - - # Archive not replaced old representations - for repre_name_low, repre in old_repres_to_delete.items(): - # Replace archived representation (This is backup) - # - should not happen to have both repre and archived repre - if repre_name_low in archived_repres_by_name: - archived_repre = archived_repres_by_name.pop(repre_name_low) - repre["old_id"] = repre["_id"] - repre["_id"] = archived_repre["_id"] - repre["type"] = archived_repre["type"] - bulk_writes.append( - ReplaceOne( - {"_id": archived_repre["_id"]}, - repre + old_id = archived_repre["old_id"] + repre["_id"] = old_id + bulk_writes.append( + ReplaceOne( + {"old_id": old_id}, + repre + ) ) + + # Create representation + else: + repre["_id"] = io.ObjectId() + bulk_writes.append( + InsertOne(repre) + ) + + # Prepare paths of source and destination files + if len(published_files) == 1: + src_to_dst_file_paths.append( + (published_files[0], template_filled) + ) + continue + + collections, remainders = clique.assemble(published_files) + if remainders or not collections or len(collections) > 1: + raise Exception(( + "Integrity error. Files of published representation " + "is combination of frame collections and single files." + "Collections: `{}` Single files: `{}`" + ).format(str(collections), str(remainders))) + + src_col = collections[0] + + # Get head and tail for collection + frame_splitter = "_-_FRAME_SPLIT_-_" + anatomy_data["frame"] = frame_splitter + _anatomy_filled = anatomy.format(anatomy_data) + _template_filled = _anatomy_filled["master"]["path"] + head, tail = _template_filled.split(frame_splitter) + padding = ( + anatomy.templates["render"]["padding"] ) - else: - repre["old_id"] = repre["_id"] - repre["_id"] = io.ObjectId() - repre["type"] = "archived_representation" - bulk_writes.append( - InsertOne(repre) + dst_col = clique.Collection( + head=head, padding=padding, tail=tail + ) + dst_col.indexes.clear() + dst_col.indexes.update(src_col.indexes) + for src_file, dst_file in zip(src_col, dst_col): + src_to_dst_file_paths.append( + (src_file, dst_file) + ) + + self.path_checks = [] + + # Copy(hardlink) paths of source and destination files + # TODO should we *only* create hardlinks? + # TODO should we keep files for deletion until this is successful? + for src_path, dst_path in src_to_dst_file_paths: + self.create_hardlink(src_path, dst_path) + + for src_path, dst_path in other_file_paths_mapping: + self.create_hardlink(src_path, dst_path) + + # Archive not replaced old representations + for repre_name_low, repre in old_repres_to_delete.items(): + # Replace archived representation (This is backup) + # - should not happen to have both repre and archived repre + if repre_name_low in archived_repres_by_name: + archived_repre = archived_repres_by_name.pop( + repre_name_low + ) + repre["old_id"] = repre["_id"] + repre["_id"] = archived_repre["_id"] + repre["type"] = archived_repre["type"] + bulk_writes.append( + ReplaceOne( + {"_id": archived_repre["_id"]}, + repre + ) + ) + + else: + repre["old_id"] = repre["_id"] + repre["_id"] = io.ObjectId() + repre["type"] = "archived_representation" + bulk_writes.append( + InsertOne(repre) + ) + + if bulk_writes: + io._database[io.Session["AVALON_PROJECT"]].bulk_write( + bulk_writes ) - if bulk_writes: - io._database[io.Session["AVALON_PROJECT"]].bulk_write(bulk_writes) + # Remove backuped previous master + shutil.rmtree(backup_master_publish_dir) + + except Exception: + os.rename(backup_master_publish_dir, master_publish_dir) + self.log.error(( + "!!! Creating of Master version failed." + " Previous master version maybe lost some data!" + )) + raise self.log.debug(( "--- End of Master version integration for subset `{}`." @@ -306,7 +417,49 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): instance.data.get("subset", str(instance)) )) + def get_all_files_from_path(self, path): + files = [] + for (dir_path, dir_names, file_names) in os.walk(path): + for file_name in file_names: + _path = os.path.join(dir_path, file_name) + files.append(_path) + return files + + def get_publish_dir(self, instance): + anatomy = instance.context.data["anatomy"] + template_data = copy.deepcopy(instance.data["anatomyData"]) + + if "folder" in anatomy.templates["master"]: + anatomy_filled = anatomy.format(template_data) + publish_folder = anatomy_filled["master"]["folder"] + else: + # This is for cases of Deprecated anatomy without `folder` + # TODO remove when all clients have solved this issue + template_data.update({ + "frame": "FRAME_TEMP", + "representation": "TEMP" + }) + anatomy_filled = anatomy.format(template_data) + # solve deprecated situation when `folder` key is not underneath + # `publish` anatomy + project_name = api.Session["AVALON_PROJECT"] + self.log.warning(( + "Deprecation warning: Anatomy does not have set `folder`" + " key underneath `publish` (in global of for project `{}`)." + ).format(project_name)) + + file_path = anatomy_filled["master"]["path"] + # Directory + publish_folder = os.path.dirname(file_path) + + publish_folder = os.path.normpath(publish_folder) + + self.log.debug("Master publish dir: \"{}\"".format(publish_folder)) + + return publish_folder + def create_hardlink(self, src_path, dst_path): + # TODO check drives if are the same to check if cas hardlink dst_path = self.path_root_check(dst_path) src_path = self.path_root_check(src_path) @@ -314,7 +467,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): try: os.makedirs(dirname) - self.log.debug("Folder created: \"{}\"".format(dirname)) + self.log.debug("Folder(s) created: \"{}\"".format(dirname)) except OSError as exc: if exc.errno != errno.EEXIST: self.log.error("An unexpected error occurred.", exc_info=True) @@ -325,8 +478,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): self.log.debug("Copying file \"{}\" to \"{}\"".format( src_path, dst_path )) - # TODO check if file exists!!! - # - uncomplete publish may cause that file already exists filelink.create(src_path, dst_path, filelink.HARDLINK) def path_root_check(self, path): @@ -398,142 +549,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): return os.path.normpath(path) - def delete_repre_files(self, repres): - if not repres: - return - - frame_splitter = "_-_FRAME_-_" - files_to_delete = [] - for repre in repres: - is_sequence = False - if "frame" in repre["context"]: - repre["context"]["frame"] = frame_splitter - is_sequence = True - - template = repre["data"]["template"] - context = repre["context"] - context["root"] = api.registered_root() - path = pipeline.format_template_with_optional_keys( - context, template - ) - path = os.path.normpath(path) - if not is_sequence: - if os.path.exists(path): - files_to_delete.append(path) - continue - - dirpath = os.path.dirname(path) - file_start = None - file_end = None - file_items = path.split(frame_splitter) - if len(file_items) == 0: - continue - elif len(file_items) == 1: - if path.startswith(frame_splitter): - file_end = file_items[0] - else: - file_start = file_items[1] - - elif len(file_items) == 2: - file_start, file_end = file_items - - else: - raise ValueError(( - "Representation template has `frame` key " - "more than once inside." - )) - - for file_name in os.listdir(dirpath): - check_name = str(file_name) - if file_start and not check_name.startswith(file_start): - continue - check_name.replace(file_start, "") - - if file_end and not check_name.endswith(file_end): - continue - check_name.replace(file_end, "") - - # File does not have frame - if not check_name: - continue - - files_to_delete.append(os.path.join(dirpath, file_name)) - - renamed_files = [] - failed = False - for file_path in files_to_delete: - self.log.debug( - "Preparing file for deletion: `{}`".format(file_path) - ) - rename_path = file_path + ".BACKUP" - - max_index = 10 - cur_index = 0 - _rename_path = None - while os.path.exists(rename_path): - if _rename_path is None: - _rename_path = rename_path - - if cur_index >= max_index: - self.log.warning(( - "Max while loop index reached! Can't make backup" - " for previous master version." - )) - failed = True - break - - if not os.path.exists(_rename_path): - rename_path = _rename_path - break - - try: - os.remove(_rename_path) - self.log.debug( - "Deleted old backup file: \"{}\"".format(_rename_path) - ) - except Exception: - self.log.warning( - "Could not delete old backup file \"{}\".".format( - _rename_path - ), - exc_info=True - ) - _rename_path = file_path + ".BACKUP{}".format( - str(cur_index) - ) - cur_index += 1 - - # Skip if any already failed - if failed: - break - - try: - args = (file_path, rename_path) - os.rename(*args) - renamed_files.append(args) - except Exception: - self.log.warning( - "Could not rename file `{}` to `{}`".format( - file_path, rename_path - ), - exc_info=True - ) - failed = True - break - - if failed: - # Rename back old renamed files - for dst_name, src_name in renamed_files: - os.rename(src_name, dst_name) - - raise AssertionError(( - "Could not create master version because it is not possible" - " to replace current master files." - )) - - for _, renamed_path in renamed_files: - os.remove(renamed_path) - def version_from_representations(self, repres): for repre in repres: version = io.find_one({"_id": repre["parent"]}) From 685edf184383dd7e6cc75f1b29568622522aa001 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 25 Feb 2020 14:12:00 +0100 Subject: [PATCH 040/118] few minor fixes --- pype/plugins/global/publish/integrate_master_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 42c93db7e9..f2769a436e 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -115,7 +115,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): all_copied_files.append(dst) all_repre_file_paths = [] - for repre_info in published_repres: + for repre_info in published_repres.values(): published_files = repre_info.get("published_files") or [] for file_path in published_files: file_path = os.path.normpath(file_path) @@ -265,7 +265,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # Get filled path to repre context anatomy_filled = anatomy.format(anatomy_data) - template_filled = anatomy_filled["publish"]["master"] + template_filled = anatomy_filled["master"]["path"] repre_data = { "path": str(template_filled), From 4ced37437b3d875b24fbfe7fb8c613d45ab7d6f5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 25 Feb 2020 14:26:23 +0100 Subject: [PATCH 041/118] create_hardlink changed to copy_file - can handle if paths are cross drives --- .../publish/integrate_master_version.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index f2769a436e..2a23abfbec 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -364,10 +364,10 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # TODO should we *only* create hardlinks? # TODO should we keep files for deletion until this is successful? for src_path, dst_path in src_to_dst_file_paths: - self.create_hardlink(src_path, dst_path) + self.copy_file(src_path, dst_path) for src_path, dst_path in other_file_paths_mapping: - self.create_hardlink(src_path, dst_path) + self.copy_file(src_path, dst_path) # Archive not replaced old representations for repre_name_low, repre in old_repres_to_delete.items(): @@ -412,7 +412,8 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): raise self.log.debug(( - "--- End of Master version integration for subset `{}`." + "--- Master version integration for subset `{}`" + " seems to be successful." ).format( instance.data.get("subset", str(instance)) )) @@ -458,7 +459,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): return publish_folder - def create_hardlink(self, src_path, dst_path): + def copy_file(self, src_path, dst_path): # TODO check drives if are the same to check if cas hardlink dst_path = self.path_root_check(dst_path) src_path = self.path_root_check(src_path) @@ -478,7 +479,19 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): self.log.debug("Copying file \"{}\" to \"{}\"".format( src_path, dst_path )) - filelink.create(src_path, dst_path, filelink.HARDLINK) + + # First try hardlink and copy if paths are cross drive + try: + filelink.create(src_path, dst_path, filelink.HARDLINK) + # Return when successful + return + + except OSError as exc: + # re-raise exception if different than cross drive path + if exc.errno != errno.EXDEV: + raise + + shutil.copy(src_path, dst_path) def path_root_check(self, path): normalized_path = os.path.normpath(path) From 666041c9c94aa94bcb0f43460b5af957799b39a9 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 25 Feb 2020 14:54:59 +0100 Subject: [PATCH 042/118] added schema validation and fixed master version schema --- .../plugins/global/publish/integrate_master_version.py | 7 +++++-- schema/master_version-1.0.json | 10 ++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 2a23abfbec..715d99c1c8 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -6,7 +6,7 @@ import shutil from pymongo import InsertOne, ReplaceOne import pyblish.api -from avalon import api, io, pipeline +from avalon import api, io, schema from avalon.vendor import filelink @@ -162,6 +162,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): "type": "master_version", "schema": "pype:master_version-1.0" } + schema.validate(new_master_version) # Don't make changes in database until everything is O.K. bulk_writes = [] @@ -286,9 +287,11 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre["parent"] = new_master_version["_id"] repre["context"] = repre_context repre["data"] = repre_data + repre.pop("_id", None) + + schema.validate(repre) repre_name_low = repre["name"].lower() - # Replace current representation if repre_name_low in old_repres_to_replace: old_repre = old_repres_to_replace.pop(repre_name_low) diff --git a/schema/master_version-1.0.json b/schema/master_version-1.0.json index 173a076537..991594648b 100644 --- a/schema/master_version-1.0.json +++ b/schema/master_version-1.0.json @@ -17,14 +17,13 @@ "properties": { "_id": { "description": "Document's id (database will create it's if not entered)", - "type": "ObjectId", - "example": "592c33475f8c1b064c4d1696" + "example": "ObjectId(592c33475f8c1b064c4d1696)" }, "schema": { "description": "The schema associated with this document", "type": "string", - "enum": ["avalon-core:master_version-3.0", "pype:master_version-3.0"], - "example": "pype:master_version-3.0" + "enum": ["avalon-core:master_version-1.0", "pype:master_version-1.0"], + "example": "pype:master_version-1.0" }, "type": { "description": "The type of document", @@ -34,8 +33,7 @@ }, "parent": { "description": "Unique identifier to parent document", - "type": "ObjectId", - "example": "592c33475f8c1b064c4d1696" + "example": "ObjectId(592c33475f8c1b064c4d1697)" } } } From ec411a4b5dfd4b1bef1553236f7c0820de9581b2 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Wed, 4 Mar 2020 17:00:00 +0000 Subject: [PATCH 043/118] Added 'action' family and small adjustments --- pype/plugins/blender/create/create_action.py | 38 +++ .../blender/create/create_animation.py | 2 + pype/plugins/blender/load/load_action.py | 295 ++++++++++++++++++ pype/plugins/blender/load/load_animation.py | 9 +- pype/plugins/blender/load/load_model.py | 4 + .../plugins/blender/publish/collect_action.py | 53 ++++ .../blender/publish/collect_animation.py | 2 +- pype/plugins/blender/publish/extract_blend.py | 2 +- pype/plugins/global/publish/integrate_new.py | 3 +- 9 files changed, 401 insertions(+), 7 deletions(-) create mode 100644 pype/plugins/blender/create/create_action.py create mode 100644 pype/plugins/blender/load/load_action.py create mode 100644 pype/plugins/blender/publish/collect_action.py diff --git a/pype/plugins/blender/create/create_action.py b/pype/plugins/blender/create/create_action.py new file mode 100644 index 0000000000..88ecebdfff --- /dev/null +++ b/pype/plugins/blender/create/create_action.py @@ -0,0 +1,38 @@ +"""Create an animation asset.""" + +import bpy + +from avalon import api +from avalon.blender import Creator, lib + + +class CreateAction(Creator): + """Action output for character rigs""" + + name = "actionMain" + label = "Action" + family = "action" + icon = "male" + + def process(self): + import pype.blender + + asset = self.data["asset"] + subset = self.data["subset"] + name = pype.blender.plugin.asset_name(asset, subset) + collection = bpy.data.collections.new(name=name) + bpy.context.scene.collection.children.link(collection) + self.data['task'] = api.Session.get('AVALON_TASK') + lib.imprint(collection, self.data) + + if (self.options or {}).get("useSelection"): + for obj in lib.get_selection(): + if obj.animation_data is not None and obj.animation_data.action is not None: + + empty_obj = bpy.data.objects.new( name = name, object_data = None ) + empty_obj.animation_data_create() + empty_obj.animation_data.action = obj.animation_data.action + empty_obj.animation_data.action.name = name + collection.objects.link(empty_obj) + + return collection diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py index cfe569f918..14a50ba5ea 100644 --- a/pype/plugins/blender/create/create_animation.py +++ b/pype/plugins/blender/create/create_animation.py @@ -1,3 +1,5 @@ +"""Create an animation asset.""" + import bpy from avalon import api diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py new file mode 100644 index 0000000000..6094f712ae --- /dev/null +++ b/pype/plugins/blender/load/load_action.py @@ -0,0 +1,295 @@ +"""Load an action in Blender.""" + +import logging +from pathlib import Path +from pprint import pformat +from typing import Dict, List, Optional + +import avalon.blender.pipeline +import bpy +import pype.blender +from avalon import api + +logger = logging.getLogger("pype").getChild("blender").getChild("load_action") + + +class BlendAnimationLoader(pype.blender.AssetLoader): + """Load action from a .blend file. + + Warning: + Loading the same asset more then once is not properly supported at the + moment. + """ + + families = ["action"] + representations = ["blend"] + + label = "Link Action" + icon = "code-fork" + color = "orange" + + def process_asset( + self, context: dict, name: str, namespace: Optional[str] = None, + options: Optional[Dict] = None + ) -> Optional[List]: + """ + Arguments: + name: Use pre-defined name + namespace: Use pre-defined namespace + context: Full parenthood of representation to load + options: Additional settings dictionary + """ + + libpath = self.fname + asset = context["asset"]["name"] + subset = context["subset"]["name"] + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( + asset, subset, namespace + ) + relative = bpy.context.preferences.filepaths.use_relative_paths + + container = bpy.data.collections.new(lib_container) + container.name = container_name + avalon.blender.pipeline.containerise_existing( + container, + name, + namespace, + context, + self.__class__.__name__, + ) + + container_metadata = container.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container + + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + animation_container = scene.collection.children[lib_container].make_local() + + objects_list = [] + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in animation_container.objects: + + obj = obj.make_local() + + # obj.data.make_local() + + if obj.animation_data is not None and obj.animation_data.action is not None: + + obj.animation_data.action.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": container_name}) + + objects_list.append(obj) + + animation_container.pop(avalon.blender.pipeline.AVALON_PROPERTY) + + # Save the list of objects in the metadata container + container_metadata["objects"] = objects_list + + bpy.ops.object.select_all(action='DESELECT') + + nodes = list(container.objects) + nodes.append(container) + self[:] = nodes + return nodes + + def update(self, container: Dict, representation: Dict): + """Update the loaded asset. + + This will remove all objects of the current collection, load the new + ones and add them to the collection. + If the objects of the collection are used in another collection they + will not be removed, only unlinked. Normally this should not be the + case though. + + Warning: + No nested collections are supported at the moment! + """ + + collection = bpy.data.collections.get( + container["objectName"] + ) + + libpath = Path(api.get_representation_path(representation)) + extension = libpath.suffix.lower() + + logger.info( + "Container: %s\nRepresentation: %s", + pformat(container, indent=2), + pformat(representation, indent=2), + ) + + assert collection, ( + f"The asset is not loaded: {container['objectName']}" + ) + assert not (collection.children), ( + "Nested collections are not supported." + ) + assert libpath, ( + "No existing library file found for {container['objectName']}" + ) + assert libpath.is_file(), ( + f"The file doesn't exist: {libpath}" + ) + assert extension in pype.blender.plugin.VALID_EXTENSIONS, ( + f"Unsupported file: {libpath}" + ) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + collection_libpath = collection_metadata["libpath"] + normalized_collection_libpath = ( + str(Path(bpy.path.abspath(collection_libpath)).resolve()) + ) + normalized_libpath = ( + str(Path(bpy.path.abspath(str(libpath))).resolve()) + ) + logger.debug( + "normalized_collection_libpath:\n %s\nnormalized_libpath:\n %s", + normalized_collection_libpath, + normalized_libpath, + ) + if normalized_collection_libpath == normalized_libpath: + logger.info("Library already loaded, not updating...") + return + + strips = [] + + for obj in collection_metadata["objects"]: + + for armature_obj in [ objj for objj in bpy.data.objects if objj.type == 'ARMATURE' ]: + + if armature_obj.animation_data is not None: + + for track in armature_obj.animation_data.nla_tracks: + + for strip in track.strips: + + if strip.action == obj.animation_data.action: + + strips.append(strip) + + bpy.data.actions.remove(obj.animation_data.action) + bpy.data.objects.remove(obj) + + lib_container = collection_metadata["lib_container"] + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + + relative = bpy.context.preferences.filepaths.use_relative_paths + with bpy.data.libraries.load( + str(libpath), link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + animation_container = scene.collection.children[lib_container].make_local() + + objects_list = [] + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in animation_container.objects: + + obj = obj.make_local() + + if obj.animation_data is not None and obj.animation_data.action is not None: + + obj.animation_data.action.make_local() + + for strip in strips: + + strip.action = obj.animation_data.action + strip.action_frame_end = obj.animation_data.action.frame_range[1] + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": collection.name}) + + objects_list.append(obj) + + animation_container.pop(avalon.blender.pipeline.AVALON_PROPERTY) + + # Save the list of objects in the metadata container + collection_metadata["objects"] = objects_list + collection_metadata["libpath"] = str(libpath) + collection_metadata["representation"] = str(representation["_id"]) + + bpy.ops.object.select_all(action='DESELECT') + + def remove(self, container: Dict) -> bool: + """Remove an existing container from a Blender scene. + + Arguments: + container (avalon-core:container-1.0): Container to remove, + from `host.ls()`. + + Returns: + bool: Whether the container was deleted. + + Warning: + No nested collections are supported at the moment! + """ + + collection = bpy.data.collections.get( + container["objectName"] + ) + if not collection: + return False + assert not (collection.children), ( + "Nested collections are not supported." + ) + + collection_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] + + for obj in objects: + + for armature_obj in [ objj for objj in bpy.data.objects if objj.type == 'ARMATURE' ]: + + if armature_obj.animation_data is not None: + + for track in armature_obj.animation_data.nla_tracks: + + for strip in track.strips: + + if strip.action == obj.animation_data.action: + + track.strips.remove(strip) + + bpy.data.actions.remove(obj.animation_data.action) + bpy.data.objects.remove(obj) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + bpy.data.collections.remove(collection) + + return True diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py index 58a0e94665..c6d18fb1a9 100644 --- a/pype/plugins/blender/load/load_animation.py +++ b/pype/plugins/blender/load/load_animation.py @@ -10,15 +10,12 @@ import bpy import pype.blender from avalon import api -logger = logging.getLogger("pype").getChild("blender").getChild("load_model") +logger = logging.getLogger("pype").getChild("blender").getChild("load_animation") class BlendAnimationLoader(pype.blender.AssetLoader): """Load animations from a .blend file. - Because they come from a .blend file we can simply link the collection that - contains the model. There is no further need to 'containerise' it. - Warning: Loading the same asset more then once is not properly supported at the moment. @@ -94,6 +91,10 @@ class BlendAnimationLoader(pype.blender.AssetLoader): obj.data.make_local() + if obj.animation_data is not None and obj.animation_data.action is not None: + + obj.animation_data.action.make_local() + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index 40d6c3434c..8ba8c5cfc8 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -87,6 +87,10 @@ class BlendModelLoader(pype.blender.AssetLoader): obj.data.make_local() + for material_slot in obj.material_slots: + + material_slot.material.make_local() + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() diff --git a/pype/plugins/blender/publish/collect_action.py b/pype/plugins/blender/publish/collect_action.py new file mode 100644 index 0000000000..0b5e468920 --- /dev/null +++ b/pype/plugins/blender/publish/collect_action.py @@ -0,0 +1,53 @@ +import typing +from typing import Generator + +import bpy + +import avalon.api +import pyblish.api +from avalon.blender.pipeline import AVALON_PROPERTY + + +class CollectAnimation(pyblish.api.ContextPlugin): + """Collect the data of an action.""" + + hosts = ["blender"] + label = "Collect Action" + order = pyblish.api.CollectorOrder + + @staticmethod + def get_action_collections() -> Generator: + """Return all 'animation' collections. + + Check if the family is 'action' and if it doesn't have the + representation set. If the representation is set, it is a loaded action + and we don't want to publish it. + """ + for collection in bpy.data.collections: + avalon_prop = collection.get(AVALON_PROPERTY) or dict() + if (avalon_prop.get('family') == 'action' + and not avalon_prop.get('representation')): + yield collection + + def process(self, context): + """Collect the actions from the current Blender scene.""" + collections = self.get_action_collections() + for collection in collections: + avalon_prop = collection[AVALON_PROPERTY] + asset = avalon_prop['asset'] + family = avalon_prop['family'] + subset = avalon_prop['subset'] + task = avalon_prop['task'] + name = f"{asset}_{subset}" + instance = context.create_instance( + name=name, + family=family, + families=[family], + subset=subset, + asset=asset, + task=task, + ) + members = list(collection.objects) + members.append(collection) + instance[:] = members + self.log.debug(instance.data) diff --git a/pype/plugins/blender/publish/collect_animation.py b/pype/plugins/blender/publish/collect_animation.py index 9bc0b02227..109ae98e6f 100644 --- a/pype/plugins/blender/publish/collect_animation.py +++ b/pype/plugins/blender/publish/collect_animation.py @@ -20,7 +20,7 @@ class CollectAnimation(pyblish.api.ContextPlugin): """Return all 'animation' collections. Check if the family is 'animation' and if it doesn't have the - representation set. If the representation is set, it is a loaded rig + representation set. If the representation is set, it is a loaded animation and we don't want to publish it. """ for collection in bpy.data.collections: diff --git a/pype/plugins/blender/publish/extract_blend.py b/pype/plugins/blender/publish/extract_blend.py index 7e11e9ef8d..032f85897d 100644 --- a/pype/plugins/blender/publish/extract_blend.py +++ b/pype/plugins/blender/publish/extract_blend.py @@ -9,7 +9,7 @@ class ExtractBlend(pype.api.Extractor): label = "Extract Blend" hosts = ["blender"] - families = ["animation", "model", "rig"] + families = ["animation", "model", "rig", "action"] optional = True def process(self, instance): diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 813417bdfc..86ada2f111 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -78,7 +78,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "matchmove", "image" "source", - "assembly" + "assembly", + "action" ] exclude_families = ["clip"] db_representation_context_keys = [ From de972a2fa45c22cc6b5337e342028372261028ac Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Thu, 5 Mar 2020 10:27:57 +0000 Subject: [PATCH 044/118] Fixed a naming issue --- pype/plugins/blender/publish/collect_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/blender/publish/collect_action.py b/pype/plugins/blender/publish/collect_action.py index 0b5e468920..9a54045cea 100644 --- a/pype/plugins/blender/publish/collect_action.py +++ b/pype/plugins/blender/publish/collect_action.py @@ -8,7 +8,7 @@ import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY -class CollectAnimation(pyblish.api.ContextPlugin): +class CollectAction(pyblish.api.ContextPlugin): """Collect the data of an action.""" hosts = ["blender"] From 5bf25ffd3ee322e16c508bda88322faafa198e04 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Wed, 11 Mar 2020 12:39:25 +0000 Subject: [PATCH 045/118] Bug fixing and code optimization --- pype/plugins/blender/create/create_rig.py | 63 ++++++- pype/plugins/blender/load/load_action.py | 6 +- pype/plugins/blender/load/load_animation.py | 161 ++++++++---------- pype/plugins/blender/load/load_model.py | 146 +++++++---------- pype/plugins/blender/load/load_rig.py | 172 ++++++++------------ 5 files changed, 262 insertions(+), 286 deletions(-) diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py index 5d83fafdd3..f630c63966 100644 --- a/pype/plugins/blender/create/create_rig.py +++ b/pype/plugins/blender/create/create_rig.py @@ -14,6 +14,23 @@ class CreateRig(Creator): family = "rig" icon = "wheelchair" + # @staticmethod + # def _find_layer_collection(self, layer_collection, collection): + + # found = None + + # if (layer_collection.collection == collection): + + # return layer_collection + + # for layer in layer_collection.children: + + # found = self._find_layer_collection(layer, collection) + + # if found: + + # return found + def process(self): import pype.blender @@ -25,8 +42,52 @@ class CreateRig(Creator): self.data['task'] = api.Session.get('AVALON_TASK') lib.imprint(collection, self.data) + # Add the rig object and all the children meshes to + # a set and link them all at the end to avoid duplicates. + # Blender crashes if trying to link an object that is already linked. + # This links automatically the children meshes if they were not + # selected, and doesn't link them twice if they, insted, + # were manually selected by the user. + objects_to_link = set() + if (self.options or {}).get("useSelection"): + for obj in lib.get_selection(): - collection.objects.link(obj) + + objects_to_link.add( obj ) + + if obj.type == 'ARMATURE': + + for subobj in obj.children: + + objects_to_link.add( subobj ) + + # Create a new collection and link the widgets that + # the rig uses. + # custom_shapes = set() + + # for posebone in obj.pose.bones: + + # if posebone.custom_shape is not None: + + # custom_shapes.add( posebone.custom_shape ) + + # if len( custom_shapes ) > 0: + + # widgets_collection = bpy.data.collections.new(name="Widgets") + + # collection.children.link(widgets_collection) + + # for custom_shape in custom_shapes: + + # widgets_collection.objects.link( custom_shape ) + + # layer_collection = self._find_layer_collection(bpy.context.view_layer.layer_collection, widgets_collection) + + # layer_collection.exclude = True + + for obj in objects_to_link: + + collection.objects.link(obj) return collection diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py index 6094f712ae..747bcd47f5 100644 --- a/pype/plugins/blender/load/load_action.py +++ b/pype/plugins/blender/load/load_action.py @@ -13,7 +13,7 @@ from avalon import api logger = logging.getLogger("pype").getChild("blender").getChild("load_action") -class BlendAnimationLoader(pype.blender.AssetLoader): +class BlendActionLoader(pype.blender.AssetLoader): """Load action from a .blend file. Warning: @@ -47,7 +47,6 @@ class BlendAnimationLoader(pype.blender.AssetLoader): container_name = pype.blender.plugin.asset_name( asset, subset, namespace ) - relative = bpy.context.preferences.filepaths.use_relative_paths container = bpy.data.collections.new(lib_container) container.name = container_name @@ -65,6 +64,7 @@ class BlendAnimationLoader(pype.blender.AssetLoader): container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container + relative = bpy.context.preferences.filepaths.use_relative_paths with bpy.data.libraries.load( libpath, link=True, relative=relative ) as (_, data_to): @@ -85,8 +85,6 @@ class BlendAnimationLoader(pype.blender.AssetLoader): obj = obj.make_local() - # obj.data.make_local() - if obj.animation_data is not None and obj.animation_data.action is not None: obj.animation_data.action.make_local() diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py index c6d18fb1a9..0610517b67 100644 --- a/pype/plugins/blender/load/load_animation.py +++ b/pype/plugins/blender/load/load_animation.py @@ -28,43 +28,22 @@ class BlendAnimationLoader(pype.blender.AssetLoader): icon = "code-fork" color = "orange" - def process_asset( - self, context: dict, name: str, namespace: Optional[str] = None, - options: Optional[Dict] = None - ) -> Optional[List]: - """ - Arguments: - name: Use pre-defined name - namespace: Use pre-defined namespace - context: Full parenthood of representation to load - options: Additional settings dictionary - """ + @staticmethod + def _remove(self, objects, lib_container): + + for obj in objects: + + if obj.type == 'ARMATURE': + bpy.data.armatures.remove(obj.data) + elif obj.type == 'MESH': + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + + @staticmethod + def _process(self, libpath, lib_container, container_name): - libpath = self.fname - asset = context["asset"]["name"] - subset = context["subset"]["name"] - lib_container = pype.blender.plugin.asset_name(asset, subset) - container_name = pype.blender.plugin.asset_name( - asset, subset, namespace - ) relative = bpy.context.preferences.filepaths.use_relative_paths - - container = bpy.data.collections.new(lib_container) - container.name = container_name - avalon.blender.pipeline.containerise_existing( - container, - name, - namespace, - context, - self.__class__.__name__, - ) - - container_metadata = container.get( - avalon.blender.pipeline.AVALON_PROPERTY) - - container_metadata["libpath"] = libpath - container_metadata["lib_container"] = lib_container - with bpy.data.libraries.load( libpath, link=True, relative=relative ) as (_, data_to): @@ -77,8 +56,9 @@ class BlendAnimationLoader(pype.blender.AssetLoader): animation_container = scene.collection.children[lib_container].make_local() meshes = [obj for obj in animation_container.objects if obj.type == 'MESH'] - armatures = [ - obj for obj in animation_container.objects if obj.type == 'ARMATURE'] + armatures = [obj for obj in animation_container.objects if obj.type == 'ARMATURE'] + + # Should check if there is only an armature? objects_list = [] @@ -106,11 +86,51 @@ class BlendAnimationLoader(pype.blender.AssetLoader): animation_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + bpy.ops.object.select_all(action='DESELECT') + + return objects_list + + def process_asset( + self, context: dict, name: str, namespace: Optional[str] = None, + options: Optional[Dict] = None + ) -> Optional[List]: + """ + Arguments: + name: Use pre-defined name + namespace: Use pre-defined namespace + context: Full parenthood of representation to load + options: Additional settings dictionary + """ + + libpath = self.fname + asset = context["asset"]["name"] + subset = context["subset"]["name"] + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( + asset, subset, namespace + ) + + container = bpy.data.collections.new(lib_container) + container.name = container_name + avalon.blender.pipeline.containerise_existing( + container, + name, + namespace, + context, + self.__class__.__name__, + ) + + container_metadata = container.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container + + objects_list = self._process(self, libpath, lib_container, container_name) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list - bpy.ops.object.select_all(action='DESELECT') - nodes = list(container.objects) nodes.append(container) self[:] = nodes @@ -177,59 +197,16 @@ class BlendAnimationLoader(pype.blender.AssetLoader): logger.info("Library already loaded, not updating...") return - # Get the armature of the rig - armatures = [obj for obj in collection_metadata["objects"] - if obj.type == 'ARMATURE'] - assert(len(armatures) == 1) - - for obj in collection_metadata["objects"]: - - if obj.type == 'ARMATURE': - bpy.data.armatures.remove(obj.data) - elif obj.type == 'MESH': - bpy.data.meshes.remove(obj.data) - + objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] - bpy.data.collections.remove(bpy.data.collections[lib_container]) - - relative = bpy.context.preferences.filepaths.use_relative_paths - with bpy.data.libraries.load( - str(libpath), link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene - - scene.collection.children.link(bpy.data.collections[lib_container]) - - animation_container = scene.collection.children[lib_container].make_local() - - meshes = [obj for obj in animation_container.objects if obj.type == 'MESH'] - armatures = [ - obj for obj in animation_container.objects if obj.type == 'ARMATURE'] - objects_list = [] - + # Get the armature of the rig + armatures = [obj for obj in objects if obj.type == 'ARMATURE'] assert(len(armatures) == 1) - # Link meshes first, then armatures. - # The armature is unparented for all the non-local meshes, - # when it is made local. - for obj in meshes + armatures: + self._remove(self, objects, lib_container) - obj = obj.make_local() - - obj.data.make_local() - - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): - - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() - - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update({"container_name": collection.name}) - objects_list.append(obj) - - animation_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + objects_list = self._process(self, str(libpath), lib_container, collection.name) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list @@ -266,14 +243,8 @@ class BlendAnimationLoader(pype.blender.AssetLoader): objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] - for obj in objects: - - if obj.type == 'ARMATURE': - bpy.data.armatures.remove(obj.data) - elif obj.type == 'MESH': - bpy.data.meshes.remove(obj.data) - - bpy.data.collections.remove(bpy.data.collections[lib_container]) + self._remove(self, objects, lib_container) + bpy.data.collections.remove(collection) return True diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index 8ba8c5cfc8..10904a1f7b 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -31,43 +31,19 @@ class BlendModelLoader(pype.blender.AssetLoader): icon = "code-fork" color = "orange" - def process_asset( - self, context: dict, name: str, namespace: Optional[str] = None, - options: Optional[Dict] = None - ) -> Optional[List]: - """ - Arguments: - name: Use pre-defined name - namespace: Use pre-defined namespace - context: Full parenthood of representation to load - options: Additional settings dictionary - """ + @staticmethod + def _remove(self, objects, lib_container): + + for obj in objects: + + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + + @staticmethod + def _process(self, libpath, lib_container, container_name): - libpath = self.fname - asset = context["asset"]["name"] - subset = context["subset"]["name"] - lib_container = pype.blender.plugin.asset_name(asset, subset) - container_name = pype.blender.plugin.asset_name( - asset, subset, namespace - ) relative = bpy.context.preferences.filepaths.use_relative_paths - - container = bpy.data.collections.new(lib_container) - container.name = container_name - avalon.blender.pipeline.containerise_existing( - container, - name, - namespace, - context, - self.__class__.__name__, - ) - - container_metadata = container.get( - avalon.blender.pipeline.AVALON_PROPERTY) - - container_metadata["libpath"] = libpath - container_metadata["lib_container"] = lib_container - with bpy.data.libraries.load( libpath, link=True, relative=relative ) as (_, data_to): @@ -102,13 +78,53 @@ class BlendModelLoader(pype.blender.AssetLoader): model_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + bpy.ops.object.select_all(action='DESELECT') + + return objects_list + + def process_asset( + self, context: dict, name: str, namespace: Optional[str] = None, + options: Optional[Dict] = None + ) -> Optional[List]: + """ + Arguments: + name: Use pre-defined name + namespace: Use pre-defined namespace + context: Full parenthood of representation to load + options: Additional settings dictionary + """ + + libpath = self.fname + asset = context["asset"]["name"] + subset = context["subset"]["name"] + lib_container = pype.blender.plugin.asset_name(asset, subset) + container_name = pype.blender.plugin.asset_name( + asset, subset, namespace + ) + + collection = bpy.data.collections.new(lib_container) + collection.name = container_name + avalon.blender.pipeline.containerise_existing( + collection, + name, + namespace, + context, + self.__class__.__name__, + ) + + container_metadata = collection.get( + avalon.blender.pipeline.AVALON_PROPERTY) + + container_metadata["libpath"] = libpath + container_metadata["lib_container"] = lib_container + + objects_list = self._process(self, libpath, lib_container, container_name) + # Save the list of objects in the metadata container container_metadata["objects"] = objects_list - bpy.ops.object.select_all(action='DESELECT') - - nodes = list(container.objects) - nodes.append(container) + nodes = list(collection.objects) + nodes.append(collection) self[:] = nodes return nodes @@ -154,8 +170,10 @@ class BlendModelLoader(pype.blender.AssetLoader): collection_metadata = collection.get( avalon.blender.pipeline.AVALON_PROPERTY) - collection_libpath = collection_metadata["libpath"] + objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] + normalized_collection_libpath = ( str(Path(bpy.path.abspath(collection_libpath)).resolve()) ) @@ -171,54 +189,15 @@ class BlendModelLoader(pype.blender.AssetLoader): logger.info("Library already loaded, not updating...") return - for obj in collection_metadata["objects"]: + self._remove(self, objects, lib_container) - bpy.data.meshes.remove(obj.data) - - lib_container = collection_metadata["lib_container"] - - bpy.data.collections.remove(bpy.data.collections[lib_container]) - - relative = bpy.context.preferences.filepaths.use_relative_paths - with bpy.data.libraries.load( - str(libpath), link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene - - scene.collection.children.link(bpy.data.collections[lib_container]) - - model_container = scene.collection.children[lib_container].make_local() - - objects_list = [] - - # Link meshes first, then armatures. - # The armature is unparented for all the non-local meshes, - # when it is made local. - for obj in model_container.objects: - - obj = obj.make_local() - - obj.data.make_local() - - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): - - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() - - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update({"container_name": collection.name}) - objects_list.append(obj) - - model_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + objects_list = self._process(self, str(libpath), lib_container, collection.name) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list collection_metadata["libpath"] = str(libpath) collection_metadata["representation"] = str(representation["_id"]) - bpy.ops.object.select_all(action='DESELECT') - def remove(self, container: Dict) -> bool: """Remove an existing container from a Blender scene. @@ -246,11 +225,8 @@ class BlendModelLoader(pype.blender.AssetLoader): objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] - for obj in objects: + self._remove(self, objects, lib_container) - bpy.data.meshes.remove(obj.data) - - bpy.data.collections.remove(bpy.data.collections[lib_container]) bpy.data.collections.remove(collection) return True diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index c19717cd82..dcb70da6d8 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -30,6 +30,68 @@ class BlendRigLoader(pype.blender.AssetLoader): label = "Link Rig" icon = "code-fork" color = "orange" + + @staticmethod + def _remove(self, objects, lib_container): + + for obj in objects: + + if obj.type == 'ARMATURE': + bpy.data.armatures.remove(obj.data) + elif obj.type == 'MESH': + bpy.data.meshes.remove(obj.data) + + bpy.data.collections.remove(bpy.data.collections[lib_container]) + + @staticmethod + def _process(self, libpath, lib_container, container_name, action): + + relative = bpy.context.preferences.filepaths.use_relative_paths + with bpy.data.libraries.load( + libpath, link=True, relative=relative + ) as (_, data_to): + data_to.collections = [lib_container] + + scene = bpy.context.scene + + scene.collection.children.link(bpy.data.collections[lib_container]) + + rig_container = scene.collection.children[lib_container].make_local() + + meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] + armatures = [obj for obj in rig_container.objects if obj.type == 'ARMATURE'] + + objects_list = [] + + assert(len(armatures) == 1) + + # Link meshes first, then armatures. + # The armature is unparented for all the non-local meshes, + # when it is made local. + for obj in meshes + armatures: + + obj = obj.make_local() + + obj.data.make_local() + + if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + + obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + + avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info.update({"container_name": container_name}) + + if obj.type == 'ARMATURE' and action is not None: + + obj.animation_data.action = action + + objects_list.append(obj) + + rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + + bpy.ops.object.select_all(action='DESELECT') + + return objects_list def process_asset( self, context: dict, name: str, namespace: Optional[str] = None, @@ -50,7 +112,6 @@ class BlendRigLoader(pype.blender.AssetLoader): container_name = pype.blender.plugin.asset_name( asset, subset, namespace ) - relative = bpy.context.preferences.filepaths.use_relative_paths container = bpy.data.collections.new(lib_container) container.name = container_name @@ -68,48 +129,11 @@ class BlendRigLoader(pype.blender.AssetLoader): container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container - with bpy.data.libraries.load( - libpath, link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene - - scene.collection.children.link(bpy.data.collections[lib_container]) - - rig_container = scene.collection.children[lib_container].make_local() - - meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] - armatures = [ - obj for obj in rig_container.objects if obj.type == 'ARMATURE'] - - objects_list = [] - - # Link meshes first, then armatures. - # The armature is unparented for all the non-local meshes, - # when it is made local. - for obj in meshes + armatures: - - obj = obj.make_local() - - obj.data.make_local() - - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): - - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() - - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update({"container_name": container_name}) - - objects_list.append(obj) - - rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + objects_list = self._process(self, libpath, lib_container, container_name, None) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list - bpy.ops.object.select_all(action='DESELECT') - nodes = list(container.objects) nodes.append(container) self[:] = nodes @@ -159,8 +183,10 @@ class BlendRigLoader(pype.blender.AssetLoader): collection_metadata = collection.get( avalon.blender.pipeline.AVALON_PROPERTY) - collection_libpath = collection_metadata["libpath"] + objects = collection_metadata["objects"] + lib_container = collection_metadata["lib_container"] + normalized_collection_libpath = ( str(Path(bpy.path.abspath(collection_libpath)).resolve()) ) @@ -177,64 +203,14 @@ class BlendRigLoader(pype.blender.AssetLoader): return # Get the armature of the rig - armatures = [obj for obj in collection_metadata["objects"] - if obj.type == 'ARMATURE'] + armatures = [obj for obj in objects if obj.type == 'ARMATURE'] assert(len(armatures) == 1) action = armatures[0].animation_data.action - for obj in collection_metadata["objects"]: + self._remove(self, objects, lib_container) - if obj.type == 'ARMATURE': - bpy.data.armatures.remove(obj.data) - elif obj.type == 'MESH': - bpy.data.meshes.remove(obj.data) - - lib_container = collection_metadata["lib_container"] - - bpy.data.collections.remove(bpy.data.collections[lib_container]) - - relative = bpy.context.preferences.filepaths.use_relative_paths - with bpy.data.libraries.load( - str(libpath), link=True, relative=relative - ) as (_, data_to): - data_to.collections = [lib_container] - - scene = bpy.context.scene - - scene.collection.children.link(bpy.data.collections[lib_container]) - - rig_container = scene.collection.children[lib_container].make_local() - - meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] - armatures = [ - obj for obj in rig_container.objects if obj.type == 'ARMATURE'] - objects_list = [] - - assert(len(armatures) == 1) - - # Link meshes first, then armatures. - # The armature is unparented for all the non-local meshes, - # when it is made local. - for obj in meshes + armatures: - - obj = obj.make_local() - - obj.data.make_local() - - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): - - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() - - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] - avalon_info.update({"container_name": collection.name}) - objects_list.append(obj) - - if obj.type == 'ARMATURE' and action is not None: - - obj.animation_data.action = action - - rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + objects_list = self._process(self, str(libpath), lib_container, collection.name, action) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list @@ -271,14 +247,8 @@ class BlendRigLoader(pype.blender.AssetLoader): objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] - for obj in objects: - - if obj.type == 'ARMATURE': - bpy.data.armatures.remove(obj.data) - elif obj.type == 'MESH': - bpy.data.meshes.remove(obj.data) - - bpy.data.collections.remove(bpy.data.collections[lib_container]) + self._remove(self, objects, lib_container) + bpy.data.collections.remove(collection) return True From 877b9e8885dc431775e716f4d96c4766b966335d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 17 Mar 2020 17:27:09 +0100 Subject: [PATCH 046/118] feat(nuke): publish baked mov with preset colorspace --- pype/nuke/lib.py | 22 ++++++++++++++----- .../nuke/publish/extract_review_data_mov.py | 7 +++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index 8e241dad16..f8284d18dd 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -1564,10 +1564,9 @@ class ExporterReviewMov(ExporterReview): self.nodes = {} # deal with now lut defined in viewer lut - if hasattr(klass, "viewer_lut_raw"): - self.viewer_lut_raw = klass.viewer_lut_raw - else: - self.viewer_lut_raw = False + self.viewer_lut_raw = klass.viewer_lut_raw + self.bake_colorspace_fallback = klass.bake_colorspace_fallback + self.bake_colorspace_main = klass.bake_colorspace_main self.name = name or "baked" self.ext = ext or "mov" @@ -1628,8 +1627,19 @@ class ExporterReviewMov(ExporterReview): self.log.debug("ViewProcess... `{}`".format(self._temp_nodes)) if not self.viewer_lut_raw: - # OCIODisplay node - dag_node = nuke.createNode("OCIODisplay") + colorspace = self.bake_colorspace_main \ + or self.bake_colorspace_fallback + + self.log.debug("_ colorspace... `{}`".format(colorspace)) + + if colorspace: + # OCIOColorSpace with controled output + dag_node = nuke.createNode("OCIOColorSpace") + dag_node["out_colorspace"].setValue(str(colorspace)) + else: + # OCIODisplay + dag_node = nuke.createNode("OCIODisplay") + # connect dag_node.setInput(0, self.previous_node) self._temp_nodes.append(dag_node) diff --git a/pype/plugins/nuke/publish/extract_review_data_mov.py b/pype/plugins/nuke/publish/extract_review_data_mov.py index 8b204680a7..1c6efafcfe 100644 --- a/pype/plugins/nuke/publish/extract_review_data_mov.py +++ b/pype/plugins/nuke/publish/extract_review_data_mov.py @@ -3,7 +3,7 @@ import pyblish.api from avalon.nuke import lib as anlib from pype.nuke import lib as pnlib import pype - +reload(pnlib) class ExtractReviewDataMov(pype.api.Extractor): """Extracts movie and thumbnail with baked in luts @@ -18,6 +18,11 @@ class ExtractReviewDataMov(pype.api.Extractor): families = ["review", "render", "render.local"] hosts = ["nuke"] + # presets + viewer_lut_raw = None + bake_colorspace_fallback = None + bake_colorspace_main = None + def process(self, instance): families = instance.data["families"] self.log.info("Creating staging dir...") From 94ce045fec85e82d45e8887cd3bbb7f6d717ee12 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Tue, 17 Mar 2020 18:32:59 +0100 Subject: [PATCH 047/118] fix(nuke): fallback approach of defining colorspace --- pype/nuke/lib.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index f8284d18dd..446f9af6a3 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -1627,15 +1627,22 @@ class ExporterReviewMov(ExporterReview): self.log.debug("ViewProcess... `{}`".format(self._temp_nodes)) if not self.viewer_lut_raw: - colorspace = self.bake_colorspace_main \ - or self.bake_colorspace_fallback + colorspaces = [ + self.bake_colorspace_main, self.bake_colorspace_fallback + ] - self.log.debug("_ colorspace... `{}`".format(colorspace)) - - if colorspace: + if any(colorspaces): # OCIOColorSpace with controled output dag_node = nuke.createNode("OCIOColorSpace") - dag_node["out_colorspace"].setValue(str(colorspace)) + for c in colorspaces: + test = dag_node["out_colorspace"].setValue(str(c)) + if test: + self.log.info( + "Baking in colorspace... `{}`".format(c)) + break + + if not test: + dag_node = nuke.createNode("OCIODisplay") else: # OCIODisplay dag_node = nuke.createNode("OCIODisplay") From 47468403515e33d8b845042a474caad12ac81864 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Mar 2020 12:13:48 +0100 Subject: [PATCH 048/118] add houndci configuration and change flake8 config --- .flake8 | 2 ++ .hound.yml | 0 2 files changed, 2 insertions(+) create mode 100644 .hound.yml diff --git a/.flake8 b/.flake8 index 9de8d23bb2..67ed2d77a3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,7 @@ [flake8] # ignore = D203 +ignore = BLK100 +max-line-length = 79 exclude = .git, __pycache__, diff --git a/.hound.yml b/.hound.yml new file mode 100644 index 0000000000..e69de29bb2 From 6ae5e2d8a54c76240169d1cb3d5513738448a48d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 15:13:57 +0100 Subject: [PATCH 049/118] fix(nuke): separate write render and prerender create plugins --- ...ate_write.py => create_write_prerender.py} | 107 ++---------------- .../nuke/create/create_write_render.py | 101 +++++++++++++++++ 2 files changed, 108 insertions(+), 100 deletions(-) rename pype/plugins/nuke/create/{create_write.py => create_write_prerender.py} (54%) create mode 100644 pype/plugins/nuke/create/create_write_render.py diff --git a/pype/plugins/nuke/create/create_write.py b/pype/plugins/nuke/create/create_write_prerender.py similarity index 54% rename from pype/plugins/nuke/create/create_write.py rename to pype/plugins/nuke/create/create_write_prerender.py index 74e450f267..f8210db9db 100644 --- a/pype/plugins/nuke/create/create_write.py +++ b/pype/plugins/nuke/create/create_write_prerender.py @@ -1,103 +1,11 @@ from collections import OrderedDict -from pype.nuke import plugin +from pype.nuke import ( + plugin, + lib as pnlib + ) import nuke -class CreateWriteRender(plugin.PypeCreator): - # change this to template preset - name = "WriteRender" - label = "Create Write Render" - hosts = ["nuke"] - n_class = "write" - family = "render" - icon = "sign-out" - defaults = ["Main", "Mask"] - - def __init__(self, *args, **kwargs): - super(CreateWriteRender, self).__init__(*args, **kwargs) - - data = OrderedDict() - - data["family"] = self.family - data["families"] = self.n_class - - for k, v in self.data.items(): - if k not in data.keys(): - data.update({k: v}) - - self.data = data - self.nodes = nuke.selectedNodes() - self.log.debug("_ self.data: '{}'".format(self.data)) - - def process(self): - from pype.nuke import lib as pnlib - - inputs = [] - outputs = [] - instance = nuke.toNode(self.data["subset"]) - selected_node = None - - # use selection - if (self.options or {}).get("useSelection"): - nodes = self.nodes - - if not (len(nodes) < 2): - msg = ("Select only one node. The node you want to connect to, " - "or tick off `Use selection`") - log.error(msg) - nuke.message(msg) - - selected_node = nodes[0] - inputs = [selected_node] - outputs = selected_node.dependent() - - if instance: - if (instance.name() in selected_node.name()): - selected_node = instance.dependencies()[0] - - # if node already exist - if instance: - # collect input / outputs - inputs = instance.dependencies() - outputs = instance.dependent() - selected_node = inputs[0] - # remove old one - nuke.delete(instance) - - # recreate new - write_data = { - "class": self.n_class, - "families": [self.family], - "avalon": self.data - } - - if self.presets.get('fpath_template'): - self.log.info("Adding template path from preset") - write_data.update( - {"fpath_template": self.presets["fpath_template"]} - ) - else: - self.log.info("Adding template path from plugin") - write_data.update({ - "fpath_template": "{work}/renders/nuke/{subset}/{subset}.{frame}.{ext}"}) - - write_node = pnlib.create_write_node( - self.data["subset"], - write_data, - input=selected_node) - - # relinking to collected connections - for i, input in enumerate(inputs): - write_node.setInput(i, input) - - write_node.autoplace() - - for output in outputs: - output.setInput(0, write_node) - - return write_node - - class CreateWritePrerender(plugin.PypeCreator): # change this to template preset name = "WritePrerender" @@ -125,8 +33,6 @@ class CreateWritePrerender(plugin.PypeCreator): self.log.debug("_ self.data: '{}'".format(self.data)) def process(self): - from pype.nuke import lib as pnlib - inputs = [] outputs = [] instance = nuke.toNode(self.data["subset"]) @@ -137,8 +43,9 @@ class CreateWritePrerender(plugin.PypeCreator): nodes = self.nodes if not (len(nodes) < 2): - msg = ("Select only one node. The node you want to connect to, " - "or tick off `Use selection`") + msg = ("Select only one node. The node " + "you want to connect to, " + "or tick off `Use selection`") self.log.error(msg) nuke.message(msg) diff --git a/pype/plugins/nuke/create/create_write_render.py b/pype/plugins/nuke/create/create_write_render.py new file mode 100644 index 0000000000..c3b60ba2b0 --- /dev/null +++ b/pype/plugins/nuke/create/create_write_render.py @@ -0,0 +1,101 @@ +from collections import OrderedDict +from pype.nuke import ( + plugin, + lib as pnlib + ) +import nuke + + +class CreateWriteRender(plugin.PypeCreator): + # change this to template preset + name = "WriteRender" + label = "Create Write Render" + hosts = ["nuke"] + n_class = "write" + family = "render" + icon = "sign-out" + defaults = ["Main", "Mask"] + + def __init__(self, *args, **kwargs): + super(CreateWriteRender, self).__init__(*args, **kwargs) + + data = OrderedDict() + + data["family"] = self.family + data["families"] = self.n_class + + for k, v in self.data.items(): + if k not in data.keys(): + data.update({k: v}) + + self.data = data + self.nodes = nuke.selectedNodes() + self.log.debug("_ self.data: '{}'".format(self.data)) + + def process(self): + + inputs = [] + outputs = [] + instance = nuke.toNode(self.data["subset"]) + selected_node = None + + # use selection + if (self.options or {}).get("useSelection"): + nodes = self.nodes + + if not (len(nodes) < 2): + msg = ("Select only one node. " + "The node you want to connect to, " + "or tick off `Use selection`") + self.log.error(msg) + nuke.message(msg) + + selected_node = nodes[0] + inputs = [selected_node] + outputs = selected_node.dependent() + + if instance: + if (instance.name() in selected_node.name()): + selected_node = instance.dependencies()[0] + + # if node already exist + if instance: + # collect input / outputs + inputs = instance.dependencies() + outputs = instance.dependent() + selected_node = inputs[0] + # remove old one + nuke.delete(instance) + + # recreate new + write_data = { + "class": self.n_class, + "families": [self.family], + "avalon": self.data + } + + if self.presets.get('fpath_template'): + self.log.info("Adding template path from preset") + write_data.update( + {"fpath_template": self.presets["fpath_template"]} + ) + else: + self.log.info("Adding template path from plugin") + write_data.update({ + "fpath_template": "{work}/renders/nuke/{subset}/{subset}.{frame}.{ext}"}) + + write_node = pnlib.create_write_node( + self.data["subset"], + write_data, + input=selected_node) + + # relinking to collected connections + for i, input in enumerate(inputs): + write_node.setInput(i, input) + + write_node.autoplace() + + for output in outputs: + output.setInput(0, write_node) + + return write_node From fb9cd34cb5b5e153906728a2b89fd48a1863e10f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Mar 2020 18:16:44 +0100 Subject: [PATCH 050/118] allow exports of non-baked cameras --- .../maya/publish/extract_camera_mayaAscii.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pype/plugins/maya/publish/extract_camera_mayaAscii.py b/pype/plugins/maya/publish/extract_camera_mayaAscii.py index 30f686f6f5..ef80ed4ad4 100644 --- a/pype/plugins/maya/publish/extract_camera_mayaAscii.py +++ b/pype/plugins/maya/publish/extract_camera_mayaAscii.py @@ -94,11 +94,6 @@ class ExtractCameraMayaAscii(pype.api.Extractor): step = instance.data.get("step", 1.0) bake_to_worldspace = instance.data("bakeToWorldSpace", True) - # TODO: Implement a bake to non-world space - # Currently it will always bake the resulting camera to world-space - # and it does not allow to include the parent hierarchy, even though - # with `bakeToWorldSpace` set to False it should include its - # hierarchy to be correct with the family implementation. if not bake_to_worldspace: self.log.warning("Camera (Maya Ascii) export only supports world" "space baked camera extractions. The disabled " @@ -113,7 +108,7 @@ class ExtractCameraMayaAscii(pype.api.Extractor): framerange[1] + handles] # validate required settings - assert len(cameras) == 1, "Not a single camera found in extraction" + assert len(cameras) == 1, "Single camera must be found in extraction" assert isinstance(step, float), "Step must be a float value" camera = cameras[0] transform = cmds.listRelatives(camera, parent=True, fullPath=True) @@ -124,21 +119,24 @@ class ExtractCameraMayaAscii(pype.api.Extractor): path = os.path.join(dir_path, filename) # Perform extraction - self.log.info("Performing camera bakes for: {0}".format(transform)) with avalon.maya.maintained_selection(): with lib.evaluation("off"): with avalon.maya.suspended_refresh(): - baked = lib.bake_to_world_space( - transform, - frame_range=range_with_handles, - step=step - ) - baked_shapes = cmds.ls(baked, - type="camera", - dag=True, - shapes=True, - long=True) - + if bake_to_worldspace: + self.log.info( + "Performing camera bakes: {}".format(transform)) + baked = lib.bake_to_world_space( + transform, + frame_range=range_with_handles, + step=step + ) + baked_shapes = cmds.ls(baked, + type="camera", + dag=True, + shapes=True, + long=True) + else: + baked_shapes = cameras # Fix PLN-178: Don't allow background color to be non-black for cam in baked_shapes: attrs = {"backgroundColorR": 0.0, @@ -164,7 +162,8 @@ class ExtractCameraMayaAscii(pype.api.Extractor): expressions=False) # Delete the baked hierarchy - cmds.delete(baked) + if bake_to_worldspace: + cmds.delete(baked) massage_ma_file(path) From 5dfbe54df1b33b6391dc1b7bf718495982da234d Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:01:09 +0100 Subject: [PATCH 051/118] clean(nuke): old code --- pype/plugins/nuke/publish/collect_instances.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_instances.py b/pype/plugins/nuke/publish/collect_instances.py index cbbef70e4a..57b4208ce4 100644 --- a/pype/plugins/nuke/publish/collect_instances.py +++ b/pype/plugins/nuke/publish/collect_instances.py @@ -89,8 +89,6 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): instance.append(i) node.end() - family = avalon_knob_data["family"] - families = list() families_ak = avalon_knob_data.get("families") if families_ak: From a4938110c2c7025f74fe4686a1c4de51706359e1 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:01:47 +0100 Subject: [PATCH 052/118] feat(nuke): accepting `prerender` family on instance --- pype/plugins/nuke/publish/collect_instances.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/plugins/nuke/publish/collect_instances.py b/pype/plugins/nuke/publish/collect_instances.py index 57b4208ce4..893c6db7e5 100644 --- a/pype/plugins/nuke/publish/collect_instances.py +++ b/pype/plugins/nuke/publish/collect_instances.py @@ -80,6 +80,8 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): add_family = "render.farm" instance.data["transfer"] = False families.append(add_family) + if "prerender" in family: + families.append("prerender") else: # add family into families families.insert(0, family) From 7bbc0e5677169091c34611cf9b657ce871de402e Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:02:24 +0100 Subject: [PATCH 053/118] clean(nuke): write family is more direct and lest anatomical --- pype/plugins/nuke/publish/collect_review.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/nuke/publish/collect_review.py b/pype/plugins/nuke/publish/collect_review.py index c95c94541d..f02f22e053 100644 --- a/pype/plugins/nuke/publish/collect_review.py +++ b/pype/plugins/nuke/publish/collect_review.py @@ -9,7 +9,7 @@ class CollectReview(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.3 label = "Collect Review" hosts = ["nuke"] - families = ["render", "render.local", "render.farm"] + families = ["write", "prerender"] def process(self, instance): From d9a2c7dad5dac122fe1194dc3bf0e02cd9ebac0c Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:02:46 +0100 Subject: [PATCH 054/118] clean(nuke): not used import --- pype/plugins/nuke/publish/collect_writes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index 0dc7c81fae..a6fbdbab8b 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -1,7 +1,6 @@ import os import nuke import pyblish.api -import pype.api as pype @pyblish.api.log From 3d32068c10b8e51f1e05d90fdb75485cde312ed2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:06:13 +0100 Subject: [PATCH 055/118] feat(nuke): accepting prerender in img collection --- pype/plugins/nuke/publish/collect_writes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index a6fbdbab8b..b1213199f5 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -67,6 +67,8 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): ) if 'render' in instance.data['families']: + if [fm for fm in instance.data['families'] + if fm in ["render", "prerender"]]: if "representations" not in instance.data: instance.data["representations"] = list() From e2f51960452f29bb8e717324415c45aa638ba5bf Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:06:41 +0100 Subject: [PATCH 056/118] clean(nuke): removing wrong family definition --- pype/plugins/nuke/publish/validate_write_bounding_box.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/nuke/publish/validate_write_bounding_box.py b/pype/plugins/nuke/publish/validate_write_bounding_box.py index e4b7c77a25..cedeea6d9f 100644 --- a/pype/plugins/nuke/publish/validate_write_bounding_box.py +++ b/pype/plugins/nuke/publish/validate_write_bounding_box.py @@ -57,7 +57,7 @@ class ValidateNukeWriteBoundingBox(pyblish.api.InstancePlugin): order = pyblish.api.ValidatorOrder optional = True - families = ["render", "render.local", "render.farm"] + families = ["write"] label = "Write Bounding Box" hosts = ["nuke"] actions = [RepairNukeBoundingBoxAction] From 8c8a4e11ab9a17c2319569d64673f20cc15c9773 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:09:05 +0100 Subject: [PATCH 057/118] feat(nuke): prerender family clarifying --- pype/plugins/nuke/publish/collect_writes.py | 22 ++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index b1213199f5..aa5f825a98 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -66,7 +66,6 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): int(last_frame) ) - if 'render' in instance.data['families']: if [fm for fm in instance.data['families'] if fm in ["render", "prerender"]]: if "representations" not in instance.data: @@ -96,7 +95,8 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): # this will only run if slate frame is not already # rendered from previews publishes if "slate" in instance.data["families"] \ - and (frame_length == collected_frames_len): + and (frame_length == collected_frames_len) \ + and ("prerender" not in instance.data["families"]): frame_slate_str = "%0{}d".format( len(str(last_frame))) % (first_frame - 1) slate_frame = collected_frames[0].replace( @@ -105,6 +105,8 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): representation['files'] = collected_frames instance.data["representations"].append(representation) + if "render" not in instance.data["families"]: + instance.data["families"].append("render") except Exception: instance.data["representations"].append(representation) self.log.debug("couldn't collect frames: {}".format(label)) @@ -144,5 +146,19 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): "deadlineChunkSize": deadlineChunkSize, "deadlinePriority": deadlinePriority }) - + self.log.debug("families: {}".format(families)) + if "prerender" in families: + _families = list() + for fm in families: + if fm in _families: + continue + if "render" in fm: + if "prerender" in fm: + continue + _families.append(fm) + instance.data.update({ + "family": "prerender", + "families": _families + }) + self.log.debug("_families: {}".format(_families)) self.log.debug("instance.data: {}".format(instance.data)) From c7aba1564aab06258841581cd423047c838e4a53 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Wed, 18 Mar 2020 19:14:08 +0100 Subject: [PATCH 058/118] group AOVs from maya render --- .../global/publish/submit_publish_job.py | 5 +++-- pype/plugins/maya/publish/collect_render.py | 20 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index dcf19ae32c..e517198ba2 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -170,7 +170,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "review": ["lutPath"], "render.farm": ["bakeScriptPath", "bakeRenderPath", "bakeWriteNodeName", "version"] - } + } # list of family names to transfer to new family if present families_transfer = ["render3d", "render2d", "ftrack", "slate"] @@ -276,7 +276,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # if override remove all frames we are expecting to be rendered # so we'll copy only those missing from current render if instance.data.get("overrideExistingFrame"): - for frame in range(start, end+1): + for frame in range(start, end + 1): if frame not in r_col.indexes: continue r_col.indexes.remove(frame) @@ -366,6 +366,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): new_instance = copy(instance_data) new_instance["subset"] = subset_name + new_instance["group"] = aov ext = cols[0].tail.lstrip(".") diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index be3878e6bd..8d74d242b3 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -211,19 +211,23 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "attachTo": attachTo, "setMembers": layer_name, "publish": True, - "frameStart": int(context.data["assetEntity"]['data']['frameStart']), - "frameEnd": int(context.data["assetEntity"]['data']['frameEnd']), - "frameStartHandle": int(self.get_render_attribute("startFrame", - layer=layer_name)), - "frameEndHandle": int(self.get_render_attribute("endFrame", - layer=layer_name)), + "frameStart": int( + context.data["assetEntity"]['data']['frameStart']), + "frameEnd": int( + context.data["assetEntity"]['data']['frameEnd']), + "frameStartHandle": int( + self.get_render_attribute("startFrame", layer=layer_name)), + "frameEndHandle": int( + self.get_render_attribute("endFrame", layer=layer_name)), "byFrameStep": int( self.get_render_attribute("byFrameStep", layer=layer_name)), "renderer": self.get_render_attribute("currentRenderer", layer=layer_name), - "handleStart": int(context.data["assetEntity"]['data']['handleStart']), - "handleEnd": int(context.data["assetEntity"]['data']['handleEnd']), + "handleStart": int( + context.data["assetEntity"]['data']['handleStart']), + "handleEnd": int( + context.data["assetEntity"]['data']['handleEnd']), # instance subset "family": "renderlayer", From 9248e4b8cf95c467560259f02126ef0d531ab3ac Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 18 Mar 2020 19:15:04 +0100 Subject: [PATCH 059/118] clear(nuke): cleaning wrong families --- pype/plugins/nuke/publish/extract_review_data_mov.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/nuke/publish/extract_review_data_mov.py b/pype/plugins/nuke/publish/extract_review_data_mov.py index 8b204680a7..683da24fc8 100644 --- a/pype/plugins/nuke/publish/extract_review_data_mov.py +++ b/pype/plugins/nuke/publish/extract_review_data_mov.py @@ -15,7 +15,7 @@ class ExtractReviewDataMov(pype.api.Extractor): order = pyblish.api.ExtractorOrder + 0.01 label = "Extract Review Data Mov" - families = ["review", "render", "render.local"] + families = ["review"] hosts = ["nuke"] def process(self, instance): From 3c4e427b135bde7f2d8203462e74c287bedd7664 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 18 Mar 2020 20:28:43 +0100 Subject: [PATCH 060/118] store popen to variable --- pype/ftrack/lib/ftrack_app_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/ftrack/lib/ftrack_app_handler.py b/pype/ftrack/lib/ftrack_app_handler.py index 2b46dd43d8..eebffda280 100644 --- a/pype/ftrack/lib/ftrack_app_handler.py +++ b/pype/ftrack/lib/ftrack_app_handler.py @@ -286,7 +286,9 @@ class AppAction(BaseHandler): # Run SW if was found executable if execfile is not None: - avalonlib.launch(executable=execfile, args=[], environment=env) + popen = avalonlib.launch( + executable=execfile, args=[], environment=env + ) else: return { 'success': False, From 7f035d146f580b7591238d7d5491f09692eb61a1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 18 Mar 2020 20:28:57 +0100 Subject: [PATCH 061/118] blender init cleanup --- pype/blender/__init__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pype/blender/__init__.py b/pype/blender/__init__.py index 8a29917e40..4b6074a820 100644 --- a/pype/blender/__init__.py +++ b/pype/blender/__init__.py @@ -1,16 +1,8 @@ -import logging -from pathlib import Path import os -import bpy - from avalon import api as avalon from pyblish import api as pyblish -from .plugin import AssetLoader - -logger = logging.getLogger("pype.blender") - PARENT_DIR = os.path.dirname(__file__) PACKAGE_DIR = os.path.dirname(PARENT_DIR) PLUGINS_DIR = os.path.join(PACKAGE_DIR, "plugins") From fcc26cd9f7d36d823e42656fa6f2a9952c8efdf3 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 18 Mar 2020 21:47:05 +0100 Subject: [PATCH 062/118] add all frame data to context and fix order --- .../global/publish/collect_avalon_entities.py | 11 ++++++++++- .../global/publish/collect_rendered_files.py | 2 +- pype/plugins/maya/publish/collect_scene.py | 14 ++++++++------ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/collect_avalon_entities.py b/pype/plugins/global/publish/collect_avalon_entities.py index 103f5abd1a..53f11aa693 100644 --- a/pype/plugins/global/publish/collect_avalon_entities.py +++ b/pype/plugins/global/publish/collect_avalon_entities.py @@ -15,7 +15,7 @@ import pyblish.api class CollectAvalonEntities(pyblish.api.ContextPlugin): """Collect Anatomy into Context""" - order = pyblish.api.CollectorOrder + order = pyblish.api.CollectorOrder - 0.02 label = "Collect Avalon Entities" def process(self, context): @@ -47,7 +47,16 @@ class CollectAvalonEntities(pyblish.api.ContextPlugin): context.data["assetEntity"] = asset_entity data = asset_entity['data'] + + context.data["frameStart"] = data.get("frameStart") + context.data["frameEnd"] = data.get("frameEnd") + handles = int(data.get("handles") or 0) context.data["handles"] = handles context.data["handleStart"] = int(data.get("handleStart", handles)) context.data["handleEnd"] = int(data.get("handleEnd", handles)) + + frame_start_h = data.get("frameStart") - context.data["handleStart"] + frame_end_h = data.get("frameEnd") + context.data["handleEnd"] + context.data["frameStartHandle"] = frame_start_h + context.data["frameEndHandle"] = frame_end_h diff --git a/pype/plugins/global/publish/collect_rendered_files.py b/pype/plugins/global/publish/collect_rendered_files.py index 552fd49f6d..8ecf7ba156 100644 --- a/pype/plugins/global/publish/collect_rendered_files.py +++ b/pype/plugins/global/publish/collect_rendered_files.py @@ -13,7 +13,7 @@ class CollectRenderedFiles(pyblish.api.ContextPlugin): `PYPE_PUBLISH_DATA`. Those files _MUST_ share same context. """ - order = pyblish.api.CollectorOrder - 0.0001 + order = pyblish.api.CollectorOrder - 0.1 targets = ["filesequence"] label = "Collect rendered frames" diff --git a/pype/plugins/maya/publish/collect_scene.py b/pype/plugins/maya/publish/collect_scene.py index 089019f2d3..e6976356e8 100644 --- a/pype/plugins/maya/publish/collect_scene.py +++ b/pype/plugins/maya/publish/collect_scene.py @@ -9,13 +9,14 @@ from pype.maya import lib class CollectMayaScene(pyblish.api.ContextPlugin): """Inject the current working file into context""" - order = pyblish.api.CollectorOrder - 0.1 + order = pyblish.api.CollectorOrder - 0.01 label = "Maya Workfile" hosts = ['maya'] def process(self, context): """Inject the current working file""" - current_file = context.data['currentFile'] + current_file = cmds.file(query=True, sceneName=True) + context.data['currentFile'] = current_file folder, file = os.path.split(current_file) filename, ext = os.path.splitext(file) @@ -24,9 +25,6 @@ class CollectMayaScene(pyblish.api.ContextPlugin): data = {} - for key, value in lib.collect_animation_data().items(): - data[key] = value - # create instance instance = context.create_instance(name=filename) subset = 'workfile' + task.capitalize() @@ -38,7 +36,11 @@ class CollectMayaScene(pyblish.api.ContextPlugin): "publish": True, "family": 'workfile', "families": ['workfile'], - "setMembers": [current_file] + "setMembers": [current_file], + "frameStart": context.data['frameStart'], + "frameEnd": context.data['frameEnd'], + "handleStart": context.data['handleStart'], + "handleEnd": context.data['handleEnd'] }) data['representations'] = [{ From 4778dbd3e6aa48d11c653f429e87018bdd55d1e4 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 18 Mar 2020 21:47:37 +0100 Subject: [PATCH 063/118] remove handles from custom pointcache range --- pype/plugins/maya/publish/extract_pointcache.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pype/plugins/maya/publish/extract_pointcache.py b/pype/plugins/maya/publish/extract_pointcache.py index cec4886712..e40ab6e7da 100644 --- a/pype/plugins/maya/publish/extract_pointcache.py +++ b/pype/plugins/maya/publish/extract_pointcache.py @@ -25,12 +25,8 @@ class ExtractAlembic(pype.api.Extractor): nodes = instance[:] # Collect the start and end including handles - start = instance.data.get("frameStart", 1) - end = instance.data.get("frameEnd", 1) - handles = instance.data.get("handles", 0) - if handles: - start -= handles - end += handles + start = float(instance.data.get("frameStartHandle", 1)) + end = float(instance.data.get("frameEndHandle", 1)) attrs = instance.data.get("attr", "").split(";") attrs = [value for value in attrs if value.strip()] From b329fde9835df94a56def669c2bb60ceb913f473 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 18 Mar 2020 21:48:14 +0100 Subject: [PATCH 064/118] use custom frame range if other than asset --- .../plugins/maya/publish/collect_instances.py | 44 ++++++++++++++++--- pype/plugins/maya/publish/collect_render.py | 41 ++++++++++++----- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/pype/plugins/maya/publish/collect_instances.py b/pype/plugins/maya/publish/collect_instances.py index 5af717ba4d..9ea3ebe7fa 100644 --- a/pype/plugins/maya/publish/collect_instances.py +++ b/pype/plugins/maya/publish/collect_instances.py @@ -1,6 +1,7 @@ from maya import cmds import pyblish.api +import json class CollectInstances(pyblish.api.ContextPlugin): @@ -32,6 +33,13 @@ class CollectInstances(pyblish.api.ContextPlugin): objectset = cmds.ls("*.id", long=True, type="objectSet", recursive=True, objectsOnly=True) + ctx_frame_start = context.data['frameStart'] + ctx_frame_end = context.data['frameEnd'] + ctx_handle_start = context.data['handleStart'] + ctx_handle_end = context.data['handleEnd'] + ctx_frame_start_handle = context.data['frameStartHandle'] + ctx_frame_end_handle = context.data['frameEndHandle'] + context.data['objectsets'] = objectset for objset in objectset: @@ -108,14 +116,36 @@ class CollectInstances(pyblish.api.ContextPlugin): label = "{0} ({1})".format(name, data["asset"]) - if "handles" in data: - data["handleStart"] = data["handles"] - data["handleEnd"] = data["handles"] - # Append start frame and end frame to label if present if "frameStart" and "frameEnd" in data: - data["frameStartHandle"] = data["frameStart"] - data["handleStart"] - data["frameEndHandle"] = data["frameEnd"] + data["handleEnd"] + + # if frame range on maya set is the same as full shot range + # adjust the values to match the asset data + if (ctx_frame_start_handle == data["frameStart"] + and ctx_frame_end_handle == data["frameEnd"]): + data["frameStartHandle"] = ctx_frame_start_handle + data["frameEndHandle"] = ctx_frame_end_handle + data["frameStart"] = ctx_frame_start + data["frameEnd"] = ctx_frame_end + data["handleStart"] = ctx_handle_start + data["handleEnd"] = ctx_handle_end + + # if there are user values on start and end frame not matching + # the asset, use them + + else: + if "handles" in data: + data["handleStart"] = data["handles"] + data["handleEnd"] = data["handles"] + else: + data["handleStart"] = 0 + data["handleEnd"] = 0 + + data["frameStartHandle"] = data["frameStart"] - data["handleStart"] + data["frameEndHandle"] = data["frameEnd"] + data["handleEnd"] + + if "handles" in data: + data.pop('handles') label += " [{0}-{1}]".format(int(data["frameStartHandle"]), int(data["frameEndHandle"])) @@ -127,7 +157,7 @@ class CollectInstances(pyblish.api.ContextPlugin): # Produce diagnostic message for any graphical # user interface interested in visualising it. self.log.info("Found: \"%s\" " % instance.data["name"]) - self.log.debug("DATA: \"%s\" " % instance.data) + self.log.debug("DATA: {} ".format(json.dumps(instance.data, indent=4))) def sort_by_family(instance): """Sort by family""" diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index be3878e6bd..88c1be477d 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -41,6 +41,7 @@ import re import os import types import six +import json from abc import ABCMeta, abstractmethod from maya import cmds @@ -202,6 +203,28 @@ class CollectMayaRender(pyblish.api.ContextPlugin): full_paths.append(full_path) aov_dict["beauty"] = full_paths + frame_start_render = int(self.get_render_attribute("startFrame", + layer=layer_name)) + frame_end_render = int(self.get_render_attribute("endFrame", + layer=layer_name)) + + if (int(context.data['frameStartHandle']) == frame_start_render and + int(context.data['frameEndHandle']) == frame_end_render): + + handle_start = context.data['handleStart'] + handle_end = context.data['handleEnd'] + frame_start = context.data['frameStart'] + frame_end = context.data['frameEnd'] + frame_start_handle = context.data['frameStartHandle'] + frame_end_handle = context.data['frameEndHandle'] + else: + handle_start = 0 + handle_end = 0 + frame_start = frame_start_render + frame_end = frame_end_render + frame_start_handle = frame_start_render + frame_end_handle = frame_end_render + full_exp_files.append(aov_dict) self.log.info(full_exp_files) self.log.info("collecting layer: {}".format(layer_name)) @@ -211,20 +234,18 @@ class CollectMayaRender(pyblish.api.ContextPlugin): "attachTo": attachTo, "setMembers": layer_name, "publish": True, - "frameStart": int(context.data["assetEntity"]['data']['frameStart']), - "frameEnd": int(context.data["assetEntity"]['data']['frameEnd']), - "frameStartHandle": int(self.get_render_attribute("startFrame", - layer=layer_name)), - "frameEndHandle": int(self.get_render_attribute("endFrame", - layer=layer_name)), + + "handleStart": handle_start, + "handleEnd": handle_end, + "frameStart": frame_start, + "frameEnd": frame_end, + "frameStartHandle": frame_start_handle, + "frameEndHandle": frame_end_handle, "byFrameStep": int( self.get_render_attribute("byFrameStep", layer=layer_name)), "renderer": self.get_render_attribute("currentRenderer", layer=layer_name), - "handleStart": int(context.data["assetEntity"]['data']['handleStart']), - "handleEnd": int(context.data["assetEntity"]['data']['handleEnd']), - # instance subset "family": "renderlayer", "families": ["renderlayer"], @@ -267,7 +288,7 @@ class CollectMayaRender(pyblish.api.ContextPlugin): instance = context.create_instance(expected_layer_name) instance.data["label"] = label instance.data.update(data) - pass + self.log.debug("data: {}".format(json.dumps(data, indent=4))) def parse_options(self, render_globals): """Get all overrides with a value, skip those without From 5649b112f1772e6acf8544829b380e48aca29268 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Wed, 18 Mar 2020 21:48:34 +0100 Subject: [PATCH 065/118] remove unnecessary current scene collector --- .../plugins/maya/publish/collect_current_file.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 pype/plugins/maya/publish/collect_current_file.py diff --git a/pype/plugins/maya/publish/collect_current_file.py b/pype/plugins/maya/publish/collect_current_file.py deleted file mode 100644 index 0b38ebcf3d..0000000000 --- a/pype/plugins/maya/publish/collect_current_file.py +++ /dev/null @@ -1,16 +0,0 @@ -from maya import cmds - -import pyblish.api - - -class CollectMayaCurrentFile(pyblish.api.ContextPlugin): - """Inject the current working file into context""" - - order = pyblish.api.CollectorOrder - 0.5 - label = "Maya Current File" - hosts = ['maya'] - - def process(self, context): - """Inject the current working file""" - current_file = cmds.file(query=True, sceneName=True) - context.data['currentFile'] = current_file From c73059869ec088f93dc2082e3a896f397e196bf5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 10:34:37 +0100 Subject: [PATCH 066/118] grammar fixes --- pype/plugins/global/publish/integrate_thumbnail.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/integrate_thumbnail.py b/pype/plugins/global/publish/integrate_thumbnail.py index 0bb34eab58..97122d2c39 100644 --- a/pype/plugins/global/publish/integrate_thumbnail.py +++ b/pype/plugins/global/publish/integrate_thumbnail.py @@ -34,7 +34,7 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): published_repres = instance.data.get("published_representations") if not published_repres: self.log.debug( - "There are not published representations on the instance." + "There are no published representations on the instance." ) return @@ -42,12 +42,12 @@ class IntegrateThumbnails(pyblish.api.InstancePlugin): anatomy = instance.context.data["anatomy"] if "publish" not in anatomy.templates: - self.warning("Anatomy does not have set publish key!") + self.log.warning("Anatomy is missing the \"publish\" key!") return if "thumbnail" not in anatomy.templates["publish"]: - self.warning(( - "There is not set \"thumbnail\" template for project \"{}\"" + self.log.warning(( + "There is no \"thumbnail\" template set for the project \"{}\"" ).format(project_name)) return From 1ac0961f3abc10d4aa963b253b9eb177dbb9b3be Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 10:36:37 +0100 Subject: [PATCH 067/118] added missing version_id to master version schema --- schema/master_version-1.0.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/schema/master_version-1.0.json b/schema/master_version-1.0.json index 991594648b..9dff570b3a 100644 --- a/schema/master_version-1.0.json +++ b/schema/master_version-1.0.json @@ -9,6 +9,7 @@ "additionalProperties": true, "required": [ + "version_id", "schema", "type", "parent" @@ -19,6 +20,10 @@ "description": "Document's id (database will create it's if not entered)", "example": "ObjectId(592c33475f8c1b064c4d1696)" }, + "version_id": { + "description": "The version ID from which it was created", + "example": "ObjectId(592c33475f8c1b064c4d1695)" + }, "schema": { "description": "The schema associated with this document", "type": "string", From c06d4f6ecda1cc2fae0355765f13b98170debeb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Thu, 19 Mar 2020 10:46:52 +0100 Subject: [PATCH 068/118] hound config should point to flake8 config --- .hound.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.hound.yml b/.hound.yml index e69de29bb2..409cc4416a 100644 --- a/.hound.yml +++ b/.hound.yml @@ -0,0 +1,4 @@ +flake8: + enabled: true + config_file: .flake8 + From a364b90ae330d2a3691cb219e8f5df83497c5373 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 10:50:48 +0100 Subject: [PATCH 069/118] integrate_new store subset and version entity to instance.data --- pype/plugins/global/publish/integrate_master_version.py | 5 +---- pype/plugins/global/publish/integrate_new.py | 7 +++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 715d99c1c8..1cee7d1f24 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -61,13 +61,10 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): master_publish_dir = self.get_publish_dir(instance) - src_version_entity = None + src_version_entity = instance.data.get("versionEntity") filtered_repre_ids = [] for repre_id, repre_info in published_repres.items(): repre = repre_info["representation"] - if src_version_entity is None: - src_version_entity = repre_info.get("version_entity") - if repre["name"].lower() in self.ignored_representation_names: self.log.debug( "Filtering representation with name: `{}`".format( diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 8c27ccfa84..71a045a004 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -162,6 +162,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): ) subset = self.get_subset(asset_entity, instance) + instance.data["subsetEntity"] = subset version_number = instance.data["version"] self.log.debug("Next version: v{}".format(version_number)) @@ -237,6 +238,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): ) version = io.find_one({"_id": version_id}) + instance.data["versionEntity"] = version existing_repres = list(io.find({ "parent": version_id, @@ -463,10 +465,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): published_representations[repre_id] = { "representation": representation, "anatomy_data": template_data, - "published_files": published_files, - # TODO prabably should store subset and version to instance - "subset_entity": subset, - "version_entity": version + "published_files": published_files } self.log.debug("__ representations: {}".format(representations)) From 7d614c616daf65f5a384e549fa2202d3b3f5079e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 11:10:43 +0100 Subject: [PATCH 070/118] fixed bugs in itegrate master version when publishing first version --- pype/plugins/global/publish/integrate_master_version.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 1cee7d1f24..4600a95aa4 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -202,6 +202,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre_name_low = repre["name"].lower() archived_repres_by_name[repre_name_low] = repre + backup_master_publish_dir = str(master_publish_dir) if os.path.exists(master_publish_dir): backup_master_publish_dir = master_publish_dir + ".BACKUP" max_idx = 10 @@ -401,10 +402,12 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ) # Remove backuped previous master - shutil.rmtree(backup_master_publish_dir) + if os.path.exists(backup_master_publish_dir): + shutil.rmtree(backup_master_publish_dir) except Exception: - os.rename(backup_master_publish_dir, master_publish_dir) + if os.path.exists(backup_master_publish_dir): + os.rename(backup_master_publish_dir, master_publish_dir) self.log.error(( "!!! Creating of Master version failed." " Previous master version maybe lost some data!" From 2b384bcfca2a9e6d80b9a564445b010407f1a9a2 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 11:12:31 +0100 Subject: [PATCH 071/118] more specific validations of previous fix --- .../global/publish/integrate_master_version.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 4600a95aa4..0eba275407 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -202,7 +202,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre_name_low = repre["name"].lower() archived_repres_by_name[repre_name_low] = repre - backup_master_publish_dir = str(master_publish_dir) + backup_master_publish_dir = None if os.path.exists(master_publish_dir): backup_master_publish_dir = master_publish_dir + ".BACKUP" max_idx = 10 @@ -402,11 +402,17 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): ) # Remove backuped previous master - if os.path.exists(backup_master_publish_dir): + if ( + backup_master_publish_dir is not None and + os.path.exists(backup_master_publish_dir) + ): shutil.rmtree(backup_master_publish_dir) except Exception: - if os.path.exists(backup_master_publish_dir): + if ( + backup_master_publish_dir is not None and + os.path.exists(backup_master_publish_dir) + ): os.rename(backup_master_publish_dir, master_publish_dir) self.log.error(( "!!! Creating of Master version failed." From 61df87ff4bb3e9cd53391a1be2524f3bf372a0ad Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 12:51:58 +0100 Subject: [PATCH 072/118] hopefully intent is backwards compatible --- .../publish/integrate_ftrack_instances.py | 5 ++++- .../ftrack/publish/integrate_ftrack_note.py | 9 +++++++-- pype/plugins/global/publish/extract_burnin.py | 9 ++++++--- pype/plugins/global/publish/integrate_new.py | 18 ++++++++++++------ .../nuke/publish/extract_slate_frame.py | 6 ++++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py index 591dcf0dc2..db257e901a 100644 --- a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py +++ b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py @@ -127,7 +127,10 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): # Add custom attributes for AssetVersion assetversion_cust_attrs = {} - intent_val = instance.context.data.get("intent", {}).get("value") + intent_val = instance.context.data.get("intent") + if intent_val and isinstance(intent_val, dict): + intent_val = intent_val.get("value") + if intent_val: assetversion_cust_attrs["intent"] = intent_val diff --git a/pype/plugins/ftrack/publish/integrate_ftrack_note.py b/pype/plugins/ftrack/publish/integrate_ftrack_note.py index 679010ca58..9566207145 100644 --- a/pype/plugins/ftrack/publish/integrate_ftrack_note.py +++ b/pype/plugins/ftrack/publish/integrate_ftrack_note.py @@ -71,8 +71,13 @@ class IntegrateFtrackNote(pyblish.api.InstancePlugin): session = instance.context.data["ftrackSession"] - intent_val = instance.context.data.get("intent", {}).get("value") - intent_label = instance.context.data.get("intent", {}).get("label") + intent = instance.context.data.get("intent") + if intent and isinstance(intent, dict): + intent_val = intent.get("value") + intent_label = intent.get("label") + else: + intent_val = intent_label = intent + final_label = None if intent_val: final_label = self.get_intent_label(session, intent_val) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 086a1fdfb2..71463e296e 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -54,9 +54,12 @@ class ExtractBurnin(pype.api.Extractor): "comment": instance.context.data.get("comment", "") }) - intent = instance.context.data.get("intent", {}).get("label") - if intent: - prep_data["intent"] = intent + intent_label = instance.context.data.get("intent") + if intent_label and isinstance(intent_label, dict): + intent_label = intent_label.get("label") + + if intent_label: + prep_data["intent"] = intent_label # get anatomy project anatomy = instance.context.data['anatomy'] diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index aa214f36cb..ccfb3689e2 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -243,9 +243,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): instance.data['version'] = version['name'] - intent = context.data.get("intent") - if intent is not None: - anatomy_data["intent"] = intent + intent_value = instance.context.data.get("intent") + if intent_value and isinstance(intent_value, dict): + intent_value = intent_value.get("value") + + if intent_value: + anatomy_data["intent"] = intent_value anatomy = instance.context.data['anatomy'] @@ -653,9 +656,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "fps": context.data.get( "fps", instance.data.get("fps"))} - intent = context.data.get("intent") - if intent is not None: - version_data["intent"] = intent + intent_value = instance.context.data.get("intent") + if intent_value and isinstance(intent_value, dict): + intent_value = intent_value.get("value") + + if intent_value: + version_data["intent"] = intent_value # Include optional data if present in optionals = [ diff --git a/pype/plugins/nuke/publish/extract_slate_frame.py b/pype/plugins/nuke/publish/extract_slate_frame.py index 369cbe0496..e1c05c3d1a 100644 --- a/pype/plugins/nuke/publish/extract_slate_frame.py +++ b/pype/plugins/nuke/publish/extract_slate_frame.py @@ -157,11 +157,13 @@ class ExtractSlateFrame(pype.api.Extractor): return comment = instance.context.data.get("comment") - intent = instance.context.data.get("intent", {}).get("value", "") + intent_value = instance.context.data.get("intent") + if intent_value and isinstance(intent_value, dict): + intent_value = intent_value.get("value") try: node["f_submission_note"].setValue(comment) - node["f_submitting_for"].setValue(intent) + node["f_submitting_for"].setValue(intent_value or "") except NameError: return instance.data.pop("slateNode") From f579e8467027c30da160be55365357c18c5993b2 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 19 Mar 2020 12:53:45 +0100 Subject: [PATCH 073/118] fix to subsetGroup --- pype/plugins/global/publish/submit_publish_job.py | 2 +- pype/scripts/otio_burnin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index e517198ba2..2914203578 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -366,7 +366,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): new_instance = copy(instance_data) new_instance["subset"] = subset_name - new_instance["group"] = aov + new_instance["subsetGroup"] = aov ext = cols[0].tail.lstrip(".") diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 8d0b925089..8b52216968 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -296,7 +296,7 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): args=args, overwrite=overwrite ) - print(command) + # print(command) proc = subprocess.Popen(command, shell=True) proc.communicate() From f5426c78a61697e2381fb3c0c6438cefa69f75cf Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 19 Mar 2020 17:56:54 +0100 Subject: [PATCH 074/118] install custom excepthook to not crash blender --- pype/blender/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pype/blender/__init__.py b/pype/blender/__init__.py index 4b6074a820..4f52b4168a 100644 --- a/pype/blender/__init__.py +++ b/pype/blender/__init__.py @@ -1,4 +1,6 @@ import os +import sys +import traceback from avalon import api as avalon from pyblish import api as pyblish @@ -11,9 +13,16 @@ PUBLISH_PATH = os.path.join(PLUGINS_DIR, "blender", "publish") LOAD_PATH = os.path.join(PLUGINS_DIR, "blender", "load") CREATE_PATH = os.path.join(PLUGINS_DIR, "blender", "create") +ORIGINAL_EXCEPTHOOK = sys.excepthook + + +def pype_excepthook_handler(*args): + traceback.print_exception(*args) + def install(): """Install Blender configuration for Avalon.""" + sys.excepthook = pype_excepthook_handler pyblish.register_plugin_path(str(PUBLISH_PATH)) avalon.register_plugin_path(avalon.Loader, str(LOAD_PATH)) avalon.register_plugin_path(avalon.Creator, str(CREATE_PATH)) @@ -21,6 +30,7 @@ def install(): def uninstall(): """Uninstall Blender configuration for Avalon.""" + sys.excepthook = ORIGINAL_EXCEPTHOOK pyblish.deregister_plugin_path(str(PUBLISH_PATH)) avalon.deregister_plugin_path(avalon.Loader, str(LOAD_PATH)) avalon.deregister_plugin_path(avalon.Creator, str(CREATE_PATH)) From 05441eb2fe3df8b44e465118d6ef319ba774d535 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 19 Mar 2020 18:07:36 +0100 Subject: [PATCH 075/118] fix imports --- pype/plugins/blender/create/create_action.py | 2 ++ .../blender/create/create_animation.py | 2 ++ pype/plugins/blender/create/create_model.py | 3 +- pype/plugins/blender/create/create_rig.py | 11 ++++--- pype/plugins/blender/load/load_action.py | 29 +++++++++-------- pype/plugins/blender/load/load_animation.py | 26 ++++++++-------- pype/plugins/blender/load/load_model.py | 25 +++++++-------- pype/plugins/blender/load/load_rig.py | 31 +++++++++---------- 8 files changed, 65 insertions(+), 64 deletions(-) diff --git a/pype/plugins/blender/create/create_action.py b/pype/plugins/blender/create/create_action.py index 88ecebdfff..64dfe9ff90 100644 --- a/pype/plugins/blender/create/create_action.py +++ b/pype/plugins/blender/create/create_action.py @@ -4,6 +4,8 @@ import bpy from avalon import api from avalon.blender import Creator, lib +import pype.blender.plugin + class CreateAction(Creator): diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py index 14a50ba5ea..0758db280f 100644 --- a/pype/plugins/blender/create/create_animation.py +++ b/pype/plugins/blender/create/create_animation.py @@ -4,6 +4,8 @@ import bpy from avalon import api from avalon.blender import Creator, lib +import pype.blender.plugin + class CreateAnimation(Creator): diff --git a/pype/plugins/blender/create/create_model.py b/pype/plugins/blender/create/create_model.py index a3b2ffc55b..7a53f215f2 100644 --- a/pype/plugins/blender/create/create_model.py +++ b/pype/plugins/blender/create/create_model.py @@ -4,7 +4,7 @@ import bpy from avalon import api from avalon.blender import Creator, lib - +import pype.blender.plugin class CreateModel(Creator): """Polygonal static geometry""" @@ -15,7 +15,6 @@ class CreateModel(Creator): icon = "cube" def process(self): - import pype.blender asset = self.data["asset"] subset = self.data["subset"] diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py index f630c63966..b5860787ea 100644 --- a/pype/plugins/blender/create/create_rig.py +++ b/pype/plugins/blender/create/create_rig.py @@ -4,6 +4,7 @@ import bpy from avalon import api from avalon.blender import Creator, lib +import pype.blender.plugin class CreateRig(Creator): @@ -42,16 +43,16 @@ class CreateRig(Creator): self.data['task'] = api.Session.get('AVALON_TASK') lib.imprint(collection, self.data) - # Add the rig object and all the children meshes to - # a set and link them all at the end to avoid duplicates. + # Add the rig object and all the children meshes to + # a set and link them all at the end to avoid duplicates. # Blender crashes if trying to link an object that is already linked. - # This links automatically the children meshes if they were not + # This links automatically the children meshes if they were not # selected, and doesn't link them twice if they, insted, # were manually selected by the user. objects_to_link = set() if (self.options or {}).get("useSelection"): - + for obj in lib.get_selection(): objects_to_link.add( obj ) @@ -75,7 +76,7 @@ class CreateRig(Creator): # if len( custom_shapes ) > 0: # widgets_collection = bpy.data.collections.new(name="Widgets") - + # collection.children.link(widgets_collection) # for custom_shape in custom_shapes: diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py index 747bcd47f5..afde8b90a1 100644 --- a/pype/plugins/blender/load/load_action.py +++ b/pype/plugins/blender/load/load_action.py @@ -5,10 +5,9 @@ from pathlib import Path from pprint import pformat from typing import Dict, List, Optional -import avalon.blender.pipeline +from avalon import api, blender import bpy -import pype.blender -from avalon import api +import pype.blender.plugin logger = logging.getLogger("pype").getChild("blender").getChild("load_action") @@ -50,7 +49,7 @@ class BlendActionLoader(pype.blender.AssetLoader): container = bpy.data.collections.new(lib_container) container.name = container_name - avalon.blender.pipeline.containerise_existing( + blender.pipeline.containerise_existing( container, name, namespace, @@ -59,7 +58,7 @@ class BlendActionLoader(pype.blender.AssetLoader): ) container_metadata = container.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container @@ -89,16 +88,16 @@ class BlendActionLoader(pype.blender.AssetLoader): obj.animation_data.action.make_local() - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + if not obj.get(blender.pipeline.AVALON_PROPERTY): - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + obj[blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info = obj[blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": container_name}) objects_list.append(obj) - animation_container.pop(avalon.blender.pipeline.AVALON_PROPERTY) + animation_container.pop(blender.pipeline.AVALON_PROPERTY) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -153,7 +152,7 @@ class BlendActionLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) collection_libpath = collection_metadata["libpath"] normalized_collection_libpath = ( @@ -224,16 +223,16 @@ class BlendActionLoader(pype.blender.AssetLoader): strip.action = obj.animation_data.action strip.action_frame_end = obj.animation_data.action.frame_range[1] - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + if not obj.get(blender.pipeline.AVALON_PROPERTY): - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + obj[blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info = obj[blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": collection.name}) objects_list.append(obj) - animation_container.pop(avalon.blender.pipeline.AVALON_PROPERTY) + animation_container.pop(blender.pipeline.AVALON_PROPERTY) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list @@ -266,7 +265,7 @@ class BlendActionLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py index 0610517b67..ec3e24443f 100644 --- a/pype/plugins/blender/load/load_animation.py +++ b/pype/plugins/blender/load/load_animation.py @@ -5,15 +5,15 @@ from pathlib import Path from pprint import pformat from typing import Dict, List, Optional -import avalon.blender.pipeline +from avalon import api, blender import bpy -import pype.blender -from avalon import api +import pype.blender.plugin + logger = logging.getLogger("pype").getChild("blender").getChild("load_animation") -class BlendAnimationLoader(pype.blender.AssetLoader): +class BlendAnimationLoader(pype.blender.plugin.AssetLoader): """Load animations from a .blend file. Warning: @@ -75,16 +75,16 @@ class BlendAnimationLoader(pype.blender.AssetLoader): obj.animation_data.action.make_local() - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + if not obj.get(blender.pipeline.AVALON_PROPERTY): - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + obj[blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info = obj[blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": container_name}) objects_list.append(obj) - animation_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + animation_container.pop( blender.pipeline.AVALON_PROPERTY ) bpy.ops.object.select_all(action='DESELECT') @@ -112,7 +112,7 @@ class BlendAnimationLoader(pype.blender.AssetLoader): container = bpy.data.collections.new(lib_container) container.name = container_name - avalon.blender.pipeline.containerise_existing( + blender.pipeline.containerise_existing( container, name, namespace, @@ -121,7 +121,7 @@ class BlendAnimationLoader(pype.blender.AssetLoader): ) container_metadata = container.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container @@ -179,7 +179,7 @@ class BlendAnimationLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) collection_libpath = collection_metadata["libpath"] normalized_collection_libpath = ( @@ -239,12 +239,12 @@ class BlendAnimationLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] self._remove(self, objects, lib_container) - + bpy.data.collections.remove(collection) return True diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index 10904a1f7b..b8b6b9b956 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -5,15 +5,14 @@ from pathlib import Path from pprint import pformat from typing import Dict, List, Optional -import avalon.blender.pipeline +from avalon import api, blender import bpy -import pype.blender -from avalon import api +import pype.blender.plugin logger = logging.getLogger("pype").getChild("blender").getChild("load_model") -class BlendModelLoader(pype.blender.AssetLoader): +class BlendModelLoader(pype.blender.plugin.AssetLoader): """Load models from a .blend file. Because they come from a .blend file we can simply link the collection that @@ -67,16 +66,16 @@ class BlendModelLoader(pype.blender.AssetLoader): material_slot.material.make_local() - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + if not obj.get(blender.pipeline.AVALON_PROPERTY): - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + obj[blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info = obj[blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": container_name}) objects_list.append(obj) - model_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + model_container.pop( blender.pipeline.AVALON_PROPERTY ) bpy.ops.object.select_all(action='DESELECT') @@ -104,7 +103,7 @@ class BlendModelLoader(pype.blender.AssetLoader): collection = bpy.data.collections.new(lib_container) collection.name = container_name - avalon.blender.pipeline.containerise_existing( + blender.pipeline.containerise_existing( collection, name, namespace, @@ -113,7 +112,7 @@ class BlendModelLoader(pype.blender.AssetLoader): ) container_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container @@ -169,7 +168,7 @@ class BlendModelLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) collection_libpath = collection_metadata["libpath"] objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] @@ -221,7 +220,7 @@ class BlendModelLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] @@ -232,7 +231,7 @@ class BlendModelLoader(pype.blender.AssetLoader): return True -class CacheModelLoader(pype.blender.AssetLoader): +class CacheModelLoader(pype.blender.plugin.AssetLoader): """Load cache models. Stores the imported asset in a collection named after the asset. diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index dcb70da6d8..44d47b41a1 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -5,15 +5,14 @@ from pathlib import Path from pprint import pformat from typing import Dict, List, Optional -import avalon.blender.pipeline +from avalon import api, blender import bpy -import pype.blender -from avalon import api +import pype.blender.plugin logger = logging.getLogger("pype").getChild("blender").getChild("load_model") -class BlendRigLoader(pype.blender.AssetLoader): +class BlendRigLoader(pype.blender.plugin.AssetLoader): """Load rigs from a .blend file. Because they come from a .blend file we can simply link the collection that @@ -30,7 +29,7 @@ class BlendRigLoader(pype.blender.AssetLoader): label = "Link Rig" icon = "code-fork" color = "orange" - + @staticmethod def _remove(self, objects, lib_container): @@ -60,7 +59,7 @@ class BlendRigLoader(pype.blender.AssetLoader): meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] armatures = [obj for obj in rig_container.objects if obj.type == 'ARMATURE'] - + objects_list = [] assert(len(armatures) == 1) @@ -74,11 +73,11 @@ class BlendRigLoader(pype.blender.AssetLoader): obj.data.make_local() - if not obj.get(avalon.blender.pipeline.AVALON_PROPERTY): + if not obj.get(blender.pipeline.AVALON_PROPERTY): - obj[avalon.blender.pipeline.AVALON_PROPERTY] = dict() + obj[blender.pipeline.AVALON_PROPERTY] = dict() - avalon_info = obj[avalon.blender.pipeline.AVALON_PROPERTY] + avalon_info = obj[blender.pipeline.AVALON_PROPERTY] avalon_info.update({"container_name": container_name}) if obj.type == 'ARMATURE' and action is not None: @@ -86,8 +85,8 @@ class BlendRigLoader(pype.blender.AssetLoader): obj.animation_data.action = action objects_list.append(obj) - - rig_container.pop( avalon.blender.pipeline.AVALON_PROPERTY ) + + rig_container.pop( blender.pipeline.AVALON_PROPERTY ) bpy.ops.object.select_all(action='DESELECT') @@ -115,7 +114,7 @@ class BlendRigLoader(pype.blender.AssetLoader): container = bpy.data.collections.new(lib_container) container.name = container_name - avalon.blender.pipeline.containerise_existing( + blender.pipeline.containerise_existing( container, name, namespace, @@ -124,7 +123,7 @@ class BlendRigLoader(pype.blender.AssetLoader): ) container_metadata = container.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container @@ -182,7 +181,7 @@ class BlendRigLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) collection_libpath = collection_metadata["libpath"] objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] @@ -243,12 +242,12 @@ class BlendRigLoader(pype.blender.AssetLoader): ) collection_metadata = collection.get( - avalon.blender.pipeline.AVALON_PROPERTY) + blender.pipeline.AVALON_PROPERTY) objects = collection_metadata["objects"] lib_container = collection_metadata["lib_container"] self._remove(self, objects, lib_container) - + bpy.data.collections.remove(collection) return True From a6ec2060ccff6117cf39ab5768094a4109e4cb1d Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 19 Mar 2020 19:29:54 +0100 Subject: [PATCH 076/118] filter master version to only some families --- pype/plugins/global/publish/integrate_master_version.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 0eba275407..16aa0dd23d 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -15,6 +15,15 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # Must happen after IntegrateNew order = pyblish.api.IntegratorOrder + 0.1 + optional = True + + families = ["model", + "rig", + "setdress", + "look", + "pointcache", + "animation"] + # Can specify representation names that will be ignored (lower case) ignored_representation_names = [] db_representation_context_keys = [ From 5ff73c064ecae24e0470529503bda8ab76875ddd Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Thu, 19 Mar 2020 19:40:40 +0100 Subject: [PATCH 077/118] remove extra imports --- pype/plugins/blender/create/create_action.py | 8 ++++---- pype/plugins/blender/create/create_animation.py | 2 -- pype/plugins/blender/create/create_model.py | 1 + pype/plugins/blender/create/create_rig.py | 3 +-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pype/plugins/blender/create/create_action.py b/pype/plugins/blender/create/create_action.py index 64dfe9ff90..68e2a50b61 100644 --- a/pype/plugins/blender/create/create_action.py +++ b/pype/plugins/blender/create/create_action.py @@ -7,7 +7,6 @@ from avalon.blender import Creator, lib import pype.blender.plugin - class CreateAction(Creator): """Action output for character rigs""" @@ -17,7 +16,6 @@ class CreateAction(Creator): icon = "male" def process(self): - import pype.blender asset = self.data["asset"] subset = self.data["subset"] @@ -29,9 +27,11 @@ class CreateAction(Creator): if (self.options or {}).get("useSelection"): for obj in lib.get_selection(): - if obj.animation_data is not None and obj.animation_data.action is not None: + if (obj.animation_data is not None + and obj.animation_data.action is not None): - empty_obj = bpy.data.objects.new( name = name, object_data = None ) + empty_obj = bpy.data.objects.new(name=name, + object_data=None) empty_obj.animation_data_create() empty_obj.animation_data.action = obj.animation_data.action empty_obj.animation_data.action.name = name diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py index 0758db280f..b40a456c8f 100644 --- a/pype/plugins/blender/create/create_animation.py +++ b/pype/plugins/blender/create/create_animation.py @@ -7,7 +7,6 @@ from avalon.blender import Creator, lib import pype.blender.plugin - class CreateAnimation(Creator): """Animation output for character rigs""" @@ -17,7 +16,6 @@ class CreateAnimation(Creator): icon = "male" def process(self): - import pype.blender asset = self.data["asset"] subset = self.data["subset"] diff --git a/pype/plugins/blender/create/create_model.py b/pype/plugins/blender/create/create_model.py index 7a53f215f2..303a7a63a1 100644 --- a/pype/plugins/blender/create/create_model.py +++ b/pype/plugins/blender/create/create_model.py @@ -6,6 +6,7 @@ from avalon import api from avalon.blender import Creator, lib import pype.blender.plugin + class CreateModel(Creator): """Polygonal static geometry""" diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py index b5860787ea..d28e854232 100644 --- a/pype/plugins/blender/create/create_rig.py +++ b/pype/plugins/blender/create/create_rig.py @@ -33,7 +33,6 @@ class CreateRig(Creator): # return found def process(self): - import pype.blender asset = self.data["asset"] subset = self.data["subset"] @@ -61,7 +60,7 @@ class CreateRig(Creator): for subobj in obj.children: - objects_to_link.add( subobj ) + objects_to_link.add(subobj) # Create a new collection and link the widgets that # the rig uses. From fe8a71f97a271a6a07079c1947130f3fee207b18 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 19 Mar 2020 21:16:40 +0100 Subject: [PATCH 078/118] grouping by layer name --- pype/plugins/global/publish/submit_publish_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 2914203578..9c556f3512 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -366,7 +366,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): new_instance = copy(instance_data) new_instance["subset"] = subset_name - new_instance["subsetGroup"] = aov + new_instance["subsetGroup"] = subset ext = cols[0].tail.lstrip(".") From 81ccb8d767f9eefeb5dc00eea9874330f0113976 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 20 Mar 2020 08:59:09 +0100 Subject: [PATCH 079/118] stop hound from reporting line break before operator --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index 67ed2d77a3..f28d8cbfc3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,7 @@ [flake8] # ignore = D203 ignore = BLK100 +ignore = W504 max-line-length = 79 exclude = .git, From e8ac6ddbf9b89558601eee7fbc17533518c20b82 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Fri, 20 Mar 2020 11:32:21 +0100 Subject: [PATCH 080/118] fixed group name to include full subset name but '_AOV' --- pype/plugins/global/publish/submit_publish_job.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 9c556f3512..556132cd77 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -348,10 +348,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): assert len(cols) == 1, "only one image sequence type is expected" # create subset name `familyTaskSubset_AOV` - subset_name = 'render{}{}{}{}_{}'.format( + group_name = 'render{}{}{}{}'.format( task[0].upper(), task[1:], - subset[0].upper(), subset[1:], - aov) + subset[0].upper(), subset[1:]) + + subset_name = '{}_{}'.format(group_name, aov) staging = os.path.dirname(list(cols[0])[0]) @@ -366,7 +367,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): new_instance = copy(instance_data) new_instance["subset"] = subset_name - new_instance["subsetGroup"] = subset + new_instance["subsetGroup"] = group_name ext = cols[0].tail.lstrip(".") From ac178e9d5b5336d607fa37c0755329e4977c224e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 20 Mar 2020 12:05:10 +0100 Subject: [PATCH 081/118] use right intent variable in integrate new --- pype/plugins/global/publish/integrate_new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index ccfb3689e2..5052ae3aff 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -263,8 +263,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): for idx, repre in enumerate(instance.data["representations"]): # create template data for Anatomy template_data = copy.deepcopy(anatomy_data) - if intent is not None: - template_data["intent"] = intent + if intent_value is not None: + template_data["intent"] = intent_value resolution_width = repre.get("resolutionWidth") resolution_height = repre.get("resolutionHeight") From 0f8b35c831420dc55ff07699ab54f1064b9a3004 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 20 Mar 2020 11:31:59 +0000 Subject: [PATCH 082/118] Fixed Loader's parent class --- pype/plugins/blender/load/load_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py index afde8b90a1..e185bff7a8 100644 --- a/pype/plugins/blender/load/load_action.py +++ b/pype/plugins/blender/load/load_action.py @@ -12,7 +12,7 @@ import pype.blender.plugin logger = logging.getLogger("pype").getChild("blender").getChild("load_action") -class BlendActionLoader(pype.blender.AssetLoader): +class BlendActionLoader(pype.blender.plugin.AssetLoader): """Load action from a .blend file. Warning: From 7e8bb47ed727a5798427e393dac2361ba555b065 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 20 Mar 2020 12:29:34 +0000 Subject: [PATCH 083/118] Fixed creation of the animation and fbx publishing --- .../blender/create/create_animation.py | 22 +++++++++++- .../blender/publish/extract_fbx_animation.py | 36 +++++++++++-------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py index b40a456c8f..6b7616bbfd 100644 --- a/pype/plugins/blender/create/create_animation.py +++ b/pype/plugins/blender/create/create_animation.py @@ -25,8 +25,28 @@ class CreateAnimation(Creator): self.data['task'] = api.Session.get('AVALON_TASK') lib.imprint(collection, self.data) + # Add the rig object and all the children meshes to + # a set and link them all at the end to avoid duplicates. + # Blender crashes if trying to link an object that is already linked. + # This links automatically the children meshes if they were not + # selected, and doesn't link them twice if they, insted, + # were manually selected by the user. + objects_to_link = set() + if (self.options or {}).get("useSelection"): + for obj in lib.get_selection(): - collection.objects.link(obj) + + objects_to_link.add( obj ) + + if obj.type == 'ARMATURE': + + for subobj in obj.children: + + objects_to_link.add(subobj) + + for obj in objects_to_link: + + collection.objects.link(obj) return collection diff --git a/pype/plugins/blender/publish/extract_fbx_animation.py b/pype/plugins/blender/publish/extract_fbx_animation.py index bc088f8bb7..4b1fe98c2f 100644 --- a/pype/plugins/blender/publish/extract_fbx_animation.py +++ b/pype/plugins/blender/publish/extract_fbx_animation.py @@ -47,8 +47,7 @@ class ExtractAnimationFBX(pype.api.Extractor): # We set the scale of the scene for the export bpy.context.scene.unit_settings.scale_length = 0.01 - # We export all the objects in the collection - objects_to_export = collections[0].objects + armatures = [obj for obj in collections[0].objects if obj.type == 'ARMATURE'] object_action_pairs = [] original_actions = [] @@ -56,20 +55,25 @@ class ExtractAnimationFBX(pype.api.Extractor): starting_frames = [] ending_frames = [] - # For each object, we make a copy of the current action - for obj in objects_to_export: + # For each armature, we make a copy of the current action + for obj in armatures: - curr_action = obj.animation_data.action - copy_action = curr_action.copy() + curr_action = None + copy_action = None + + if obj.animation_data and obj.animation_data.action: + + curr_action = obj.animation_data.action + copy_action = curr_action.copy() + + curr_frame_range = curr_action.frame_range + + starting_frames.append( curr_frame_range[0] ) + ending_frames.append( curr_frame_range[1] ) object_action_pairs.append((obj, copy_action)) original_actions.append(curr_action) - curr_frame_range = curr_action.frame_range - - starting_frames.append( curr_frame_range[0] ) - ending_frames.append( curr_frame_range[1] ) - # We compute the starting and ending frames max_frame = min( starting_frames ) min_frame = max( ending_frames ) @@ -98,10 +102,14 @@ class ExtractAnimationFBX(pype.api.Extractor): # We delete the baked action and set the original one back for i in range(0, len(object_action_pairs)): - object_action_pairs[i][0].animation_data.action = original_actions[i] + if original_actions[i]: - object_action_pairs[i][1].user_clear() - bpy.data.actions.remove(object_action_pairs[i][1]) + object_action_pairs[i][0].animation_data.action = original_actions[i] + + if object_action_pairs[i][1]: + + object_action_pairs[i][1].user_clear() + bpy.data.actions.remove(object_action_pairs[i][1]) if "representations" not in instance.data: instance.data["representations"] = [] From a3d025f7845ad3ef30f04fe5b7fcb7f6b0c8ab20 Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 20 Mar 2020 16:13:55 +0000 Subject: [PATCH 084/118] PEP8 compliance --- pype/blender/plugin.py | 16 +++--- pype/plugins/blender/create/create_action.py | 4 +- .../blender/create/create_animation.py | 2 +- pype/plugins/blender/create/create_rig.py | 43 +------------- pype/plugins/blender/load/load_action.py | 40 ++++++++----- pype/plugins/blender/load/load_animation.py | 24 +++++--- pype/plugins/blender/load/load_model.py | 8 ++- pype/plugins/blender/load/load_rig.py | 11 ++-- .../plugins/blender/publish/collect_action.py | 6 +- .../blender/publish/collect_animation.py | 10 ++-- .../blender/publish/collect_current_file.py | 3 +- pype/plugins/blender/publish/collect_model.py | 6 +- pype/plugins/blender/publish/collect_rig.py | 6 +- pype/plugins/blender/publish/extract_blend.py | 3 +- pype/plugins/blender/publish/extract_fbx.py | 34 +++++++---- .../blender/publish/extract_fbx_animation.py | 57 ++++++++++++------- 16 files changed, 137 insertions(+), 136 deletions(-) diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index b441714c0d..5e98d8314b 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -10,14 +10,16 @@ from avalon import api VALID_EXTENSIONS = [".blend"] -def asset_name(asset: str, subset: str, namespace: Optional[str] = None) -> str: +def asset_name( + asset: str, subset: str, namespace: Optional[str] = None +) -> str: """Return a consistent name for an asset.""" name = f"{asset}_{subset}" if namespace: name = f"{namespace}:{name}" return name -def create_blender_context( obj: Optional[bpy.types.Object] = None ): +def create_blender_context(obj: Optional[bpy.types.Object] = None): """Create a new Blender context. If an object is passed as parameter, it is set as selected and active. """ @@ -27,16 +29,16 @@ def create_blender_context( obj: Optional[bpy.types.Object] = None ): for region in area.regions: if region.type == 'WINDOW': override_context = { - 'window': win, - 'screen': win.screen, - 'area': area, - 'region': region, + 'window': win, + 'screen': win.screen, + 'area': area, + 'region': region, 'scene': bpy.context.scene, 'active_object': obj, 'selected_objects': [obj] } return override_context - raise Exception( "Could not create a custom Blender context." ) + raise Exception("Could not create a custom Blender context.") class AssetLoader(api.Loader): """A basic AssetLoader for Blender diff --git a/pype/plugins/blender/create/create_action.py b/pype/plugins/blender/create/create_action.py index 68e2a50b61..6c24065f81 100644 --- a/pype/plugins/blender/create/create_action.py +++ b/pype/plugins/blender/create/create_action.py @@ -27,8 +27,8 @@ class CreateAction(Creator): if (self.options or {}).get("useSelection"): for obj in lib.get_selection(): - if (obj.animation_data is not None - and obj.animation_data.action is not None): + if (obj.animation_data is not None and + obj.animation_data.action is not None): empty_obj = bpy.data.objects.new(name=name, object_data=None) diff --git a/pype/plugins/blender/create/create_animation.py b/pype/plugins/blender/create/create_animation.py index 6b7616bbfd..3a5985d7a2 100644 --- a/pype/plugins/blender/create/create_animation.py +++ b/pype/plugins/blender/create/create_animation.py @@ -37,7 +37,7 @@ class CreateAnimation(Creator): for obj in lib.get_selection(): - objects_to_link.add( obj ) + objects_to_link.add(obj) if obj.type == 'ARMATURE': diff --git a/pype/plugins/blender/create/create_rig.py b/pype/plugins/blender/create/create_rig.py index d28e854232..dc97d8b4ce 100644 --- a/pype/plugins/blender/create/create_rig.py +++ b/pype/plugins/blender/create/create_rig.py @@ -15,23 +15,6 @@ class CreateRig(Creator): family = "rig" icon = "wheelchair" - # @staticmethod - # def _find_layer_collection(self, layer_collection, collection): - - # found = None - - # if (layer_collection.collection == collection): - - # return layer_collection - - # for layer in layer_collection.children: - - # found = self._find_layer_collection(layer, collection) - - # if found: - - # return found - def process(self): asset = self.data["asset"] @@ -54,7 +37,7 @@ class CreateRig(Creator): for obj in lib.get_selection(): - objects_to_link.add( obj ) + objects_to_link.add(obj) if obj.type == 'ARMATURE': @@ -62,30 +45,6 @@ class CreateRig(Creator): objects_to_link.add(subobj) - # Create a new collection and link the widgets that - # the rig uses. - # custom_shapes = set() - - # for posebone in obj.pose.bones: - - # if posebone.custom_shape is not None: - - # custom_shapes.add( posebone.custom_shape ) - - # if len( custom_shapes ) > 0: - - # widgets_collection = bpy.data.collections.new(name="Widgets") - - # collection.children.link(widgets_collection) - - # for custom_shape in custom_shapes: - - # widgets_collection.objects.link( custom_shape ) - - # layer_collection = self._find_layer_collection(bpy.context.view_layer.layer_collection, widgets_collection) - - # layer_collection.exclude = True - for obj in objects_to_link: collection.objects.link(obj) diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py index e185bff7a8..303d1ead4d 100644 --- a/pype/plugins/blender/load/load_action.py +++ b/pype/plugins/blender/load/load_action.py @@ -69,11 +69,11 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): ) as (_, data_to): data_to.collections = [lib_container] - scene = bpy.context.scene + collection = bpy.context.scene.collection - scene.collection.children.link(bpy.data.collections[lib_container]) + collection.children.link(bpy.data.collections[lib_container]) - animation_container = scene.collection.children[lib_container].make_local() + animation_container = collection.children[lib_container].make_local() objects_list = [] @@ -84,9 +84,11 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): obj = obj.make_local() - if obj.animation_data is not None and obj.animation_data.action is not None: + anim_data = obj.animation_data - obj.animation_data.action.make_local() + if anim_data is not None and anim_data.action is not None: + + anim_data.action.make_local() if not obj.get(blender.pipeline.AVALON_PROPERTY): @@ -173,8 +175,12 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): strips = [] for obj in collection_metadata["objects"]: + + # Get all the strips that use the action + arm_objs = [ + arm for arm in bpy.data.objects if arm.type == 'ARMATURE'] - for armature_obj in [ objj for objj in bpy.data.objects if objj.type == 'ARMATURE' ]: + for armature_obj in arm_objs: if armature_obj.animation_data is not None: @@ -203,25 +209,27 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): scene.collection.children.link(bpy.data.collections[lib_container]) - animation_container = scene.collection.children[lib_container].make_local() + anim_container = scene.collection.children[lib_container].make_local() objects_list = [] # Link meshes first, then armatures. # The armature is unparented for all the non-local meshes, # when it is made local. - for obj in animation_container.objects: + for obj in anim_container.objects: obj = obj.make_local() - if obj.animation_data is not None and obj.animation_data.action is not None: + anim_data = obj.animation_data - obj.animation_data.action.make_local() + if anim_data is not None and anim_data.action is not None: + + anim_data.action.make_local() for strip in strips: - strip.action = obj.animation_data.action - strip.action_frame_end = obj.animation_data.action.frame_range[1] + strip.action = anim_data.action + strip.action_frame_end = anim_data.action.frame_range[1] if not obj.get(blender.pipeline.AVALON_PROPERTY): @@ -232,7 +240,7 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): objects_list.append(obj) - animation_container.pop(blender.pipeline.AVALON_PROPERTY) + anim_container.pop(blender.pipeline.AVALON_PROPERTY) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list @@ -271,7 +279,11 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): for obj in objects: - for armature_obj in [ objj for objj in bpy.data.objects if objj.type == 'ARMATURE' ]: + # Get all the strips that use the action + arm_objs = [ + arm for arm in bpy.data.objects if arm.type == 'ARMATURE'] + + for armature_obj in arm_objs: if armature_obj.animation_data is not None: diff --git a/pype/plugins/blender/load/load_animation.py b/pype/plugins/blender/load/load_animation.py index ec3e24443f..395684a3ba 100644 --- a/pype/plugins/blender/load/load_animation.py +++ b/pype/plugins/blender/load/load_animation.py @@ -10,7 +10,8 @@ import bpy import pype.blender.plugin -logger = logging.getLogger("pype").getChild("blender").getChild("load_animation") +logger = logging.getLogger("pype").getChild( + "blender").getChild("load_animation") class BlendAnimationLoader(pype.blender.plugin.AssetLoader): @@ -53,10 +54,11 @@ class BlendAnimationLoader(pype.blender.plugin.AssetLoader): scene.collection.children.link(bpy.data.collections[lib_container]) - animation_container = scene.collection.children[lib_container].make_local() + anim_container = scene.collection.children[lib_container].make_local() - meshes = [obj for obj in animation_container.objects if obj.type == 'MESH'] - armatures = [obj for obj in animation_container.objects if obj.type == 'ARMATURE'] + meshes = [obj for obj in anim_container.objects if obj.type == 'MESH'] + armatures = [ + obj for obj in anim_container.objects if obj.type == 'ARMATURE'] # Should check if there is only an armature? @@ -71,9 +73,11 @@ class BlendAnimationLoader(pype.blender.plugin.AssetLoader): obj.data.make_local() - if obj.animation_data is not None and obj.animation_data.action is not None: + anim_data = obj.animation_data - obj.animation_data.action.make_local() + if anim_data is not None and anim_data.action is not None: + + anim_data.action.make_local() if not obj.get(blender.pipeline.AVALON_PROPERTY): @@ -84,7 +88,7 @@ class BlendAnimationLoader(pype.blender.plugin.AssetLoader): objects_list.append(obj) - animation_container.pop( blender.pipeline.AVALON_PROPERTY ) + anim_container.pop(blender.pipeline.AVALON_PROPERTY) bpy.ops.object.select_all(action='DESELECT') @@ -126,7 +130,8 @@ class BlendAnimationLoader(pype.blender.plugin.AssetLoader): container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container - objects_list = self._process(self, libpath, lib_container, container_name) + objects_list = self._process( + self, libpath, lib_container, container_name) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -206,7 +211,8 @@ class BlendAnimationLoader(pype.blender.plugin.AssetLoader): self._remove(self, objects, lib_container) - objects_list = self._process(self, str(libpath), lib_container, collection.name) + objects_list = self._process( + self, str(libpath), lib_container, collection.name) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list diff --git a/pype/plugins/blender/load/load_model.py b/pype/plugins/blender/load/load_model.py index b8b6b9b956..ff7c6c49c2 100644 --- a/pype/plugins/blender/load/load_model.py +++ b/pype/plugins/blender/load/load_model.py @@ -75,7 +75,7 @@ class BlendModelLoader(pype.blender.plugin.AssetLoader): objects_list.append(obj) - model_container.pop( blender.pipeline.AVALON_PROPERTY ) + model_container.pop(blender.pipeline.AVALON_PROPERTY) bpy.ops.object.select_all(action='DESELECT') @@ -117,7 +117,8 @@ class BlendModelLoader(pype.blender.plugin.AssetLoader): container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container - objects_list = self._process(self, libpath, lib_container, container_name) + objects_list = self._process( + self, libpath, lib_container, container_name) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -190,7 +191,8 @@ class BlendModelLoader(pype.blender.plugin.AssetLoader): self._remove(self, objects, lib_container) - objects_list = self._process(self, str(libpath), lib_container, collection.name) + objects_list = self._process( + self, str(libpath), lib_container, collection.name) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list diff --git a/pype/plugins/blender/load/load_rig.py b/pype/plugins/blender/load/load_rig.py index 44d47b41a1..d14a868722 100644 --- a/pype/plugins/blender/load/load_rig.py +++ b/pype/plugins/blender/load/load_rig.py @@ -58,7 +58,8 @@ class BlendRigLoader(pype.blender.plugin.AssetLoader): rig_container = scene.collection.children[lib_container].make_local() meshes = [obj for obj in rig_container.objects if obj.type == 'MESH'] - armatures = [obj for obj in rig_container.objects if obj.type == 'ARMATURE'] + armatures = [ + obj for obj in rig_container.objects if obj.type == 'ARMATURE'] objects_list = [] @@ -86,7 +87,7 @@ class BlendRigLoader(pype.blender.plugin.AssetLoader): objects_list.append(obj) - rig_container.pop( blender.pipeline.AVALON_PROPERTY ) + rig_container.pop(blender.pipeline.AVALON_PROPERTY) bpy.ops.object.select_all(action='DESELECT') @@ -128,7 +129,8 @@ class BlendRigLoader(pype.blender.plugin.AssetLoader): container_metadata["libpath"] = libpath container_metadata["lib_container"] = lib_container - objects_list = self._process(self, libpath, lib_container, container_name, None) + objects_list = self._process( + self, libpath, lib_container, container_name, None) # Save the list of objects in the metadata container container_metadata["objects"] = objects_list @@ -209,7 +211,8 @@ class BlendRigLoader(pype.blender.plugin.AssetLoader): self._remove(self, objects, lib_container) - objects_list = self._process(self, str(libpath), lib_container, collection.name, action) + objects_list = self._process( + self, str(libpath), lib_container, collection.name, action) # Save the list of objects in the metadata container collection_metadata["objects"] = objects_list diff --git a/pype/plugins/blender/publish/collect_action.py b/pype/plugins/blender/publish/collect_action.py index 9a54045cea..a8ceed9c82 100644 --- a/pype/plugins/blender/publish/collect_action.py +++ b/pype/plugins/blender/publish/collect_action.py @@ -1,9 +1,7 @@ -import typing from typing import Generator import bpy -import avalon.api import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY @@ -25,8 +23,8 @@ class CollectAction(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'action' - and not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'action' and + not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_animation.py b/pype/plugins/blender/publish/collect_animation.py index 109ae98e6f..50d49692b8 100644 --- a/pype/plugins/blender/publish/collect_animation.py +++ b/pype/plugins/blender/publish/collect_animation.py @@ -1,9 +1,7 @@ -import typing from typing import Generator import bpy -import avalon.api import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY @@ -20,13 +18,13 @@ class CollectAnimation(pyblish.api.ContextPlugin): """Return all 'animation' collections. Check if the family is 'animation' and if it doesn't have the - representation set. If the representation is set, it is a loaded animation - and we don't want to publish it. + representation set. If the representation is set, it is a loaded + animation and we don't want to publish it. """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'animation' - and not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'animation' and + not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_current_file.py b/pype/plugins/blender/publish/collect_current_file.py index 926d290b31..72976c490b 100644 --- a/pype/plugins/blender/publish/collect_current_file.py +++ b/pype/plugins/blender/publish/collect_current_file.py @@ -15,4 +15,5 @@ class CollectBlenderCurrentFile(pyblish.api.ContextPlugin): current_file = bpy.data.filepath context.data['currentFile'] = current_file - assert current_file != '', "Current file is empty. Save the file before continuing." + assert current_file != '', "Current file is empty. " \ + "Save the file before continuing." diff --git a/pype/plugins/blender/publish/collect_model.py b/pype/plugins/blender/publish/collect_model.py index ee10eaf7f2..df5c1e709a 100644 --- a/pype/plugins/blender/publish/collect_model.py +++ b/pype/plugins/blender/publish/collect_model.py @@ -1,9 +1,7 @@ -import typing from typing import Generator import bpy -import avalon.api import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY @@ -25,8 +23,8 @@ class CollectModel(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'model' - and not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'model' and + not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_rig.py b/pype/plugins/blender/publish/collect_rig.py index a4b30541f6..01958da37a 100644 --- a/pype/plugins/blender/publish/collect_rig.py +++ b/pype/plugins/blender/publish/collect_rig.py @@ -1,9 +1,7 @@ -import typing from typing import Generator import bpy -import avalon.api import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY @@ -25,8 +23,8 @@ class CollectRig(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'rig' - and not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'rig' and + not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/extract_blend.py b/pype/plugins/blender/publish/extract_blend.py index 032f85897d..5f3fdac293 100644 --- a/pype/plugins/blender/publish/extract_blend.py +++ b/pype/plugins/blender/publish/extract_blend.py @@ -43,4 +43,5 @@ class ExtractBlend(pype.api.Extractor): } instance.data["representations"].append(representation) - self.log.info("Extracted instance '%s' to: %s", instance.name, representation) + self.log.info("Extracted instance '%s' to: %s", + instance.name, representation) diff --git a/pype/plugins/blender/publish/extract_fbx.py b/pype/plugins/blender/publish/extract_fbx.py index 95466c1d2b..231bfdde24 100644 --- a/pype/plugins/blender/publish/extract_fbx.py +++ b/pype/plugins/blender/publish/extract_fbx.py @@ -1,10 +1,10 @@ import os -import avalon.blender.workio import pype.api import bpy + class ExtractFBX(pype.api.Extractor): """Extract as FBX.""" @@ -20,29 +20,39 @@ class ExtractFBX(pype.api.Extractor): filename = f"{instance.name}.fbx" filepath = os.path.join(stagingdir, filename) + context = bpy.context + scene = context.scene + view_layer = context.view_layer + # Perform extraction self.log.info("Performing extraction..") - collections = [obj for obj in instance if type(obj) is bpy.types.Collection] + collections = [ + obj for obj in instance if type(obj) is bpy.types.Collection] - assert len(collections) == 1, "There should be one and only one collection collected for this asset" + assert len(collections) == 1, "There should be one and only one " \ + "collection collected for this asset" - old_active_layer_collection = bpy.context.view_layer.active_layer_collection + old_active_layer_collection = view_layer.active_layer_collection + + layers = view_layer.layer_collection.children # Get the layer collection from the collection we need to export. - # This is needed because in Blender you can only set the active + # This is needed because in Blender you can only set the active # collection with the layer collection, and there is no way to get - # the layer collection from the collection (but there is the vice versa). - layer_collections = [layer for layer in bpy.context.view_layer.layer_collection.children if layer.collection == collections[0]] + # the layer collection from the collection + # (but there is the vice versa). + layer_collections = [ + layer for layer in layers if layer.collection == collections[0]] assert len(layer_collections) == 1 - bpy.context.view_layer.active_layer_collection = layer_collections[0] + view_layer.active_layer_collection = layer_collections[0] - old_scale = bpy.context.scene.unit_settings.scale_length + old_scale = scene.unit_settings.scale_length # We set the scale of the scene for the export - bpy.context.scene.unit_settings.scale_length = 0.01 + scene.unit_settings.scale_length = 0.01 # We export the fbx bpy.ops.export_scene.fbx( @@ -52,9 +62,9 @@ class ExtractFBX(pype.api.Extractor): add_leaf_bones=False ) - bpy.context.view_layer.active_layer_collection = old_active_layer_collection + view_layer.active_layer_collection = old_active_layer_collection - bpy.context.scene.unit_settings.scale_length = old_scale + scene.unit_settings.scale_length = old_scale if "representations" not in instance.data: instance.data["representations"] = [] diff --git a/pype/plugins/blender/publish/extract_fbx_animation.py b/pype/plugins/blender/publish/extract_fbx_animation.py index 4b1fe98c2f..d51c641e9c 100644 --- a/pype/plugins/blender/publish/extract_fbx_animation.py +++ b/pype/plugins/blender/publish/extract_fbx_animation.py @@ -1,5 +1,4 @@ import os -import avalon.blender.workio import pype.api @@ -23,31 +22,42 @@ class ExtractAnimationFBX(pype.api.Extractor): filename = f"{instance.name}.fbx" filepath = os.path.join(stagingdir, filename) + context = bpy.context + scene = context.scene + view_layer = context.view_layer + # Perform extraction self.log.info("Performing extraction..") - collections = [obj for obj in instance if type(obj) is bpy.types.Collection] + collections = [ + obj for obj in instance if type(obj) is bpy.types.Collection] - assert len(collections) == 1, "There should be one and only one collection collected for this asset" + assert len(collections) == 1, "There should be one and only one " \ + "collection collected for this asset" - old_active_layer_collection = bpy.context.view_layer.active_layer_collection + old_active_layer_collection = view_layer.active_layer_collection + + layers = view_layer.layer_collection.children # Get the layer collection from the collection we need to export. - # This is needed because in Blender you can only set the active + # This is needed because in Blender you can only set the active # collection with the layer collection, and there is no way to get - # the layer collection from the collection (but there is the vice versa). - layer_collections = [layer for layer in bpy.context.view_layer.layer_collection.children if layer.collection == collections[0]] + # the layer collection from the collection + # (but there is the vice versa). + layer_collections = [ + layer for layer in layers if layer.collection == collections[0]] assert len(layer_collections) == 1 - bpy.context.view_layer.active_layer_collection = layer_collections[0] + view_layer.active_layer_collection = layer_collections[0] - old_scale = bpy.context.scene.unit_settings.scale_length + old_scale = scene.unit_settings.scale_length # We set the scale of the scene for the export - bpy.context.scene.unit_settings.scale_length = 0.01 + scene.unit_settings.scale_length = 0.01 - armatures = [obj for obj in collections[0].objects if obj.type == 'ARMATURE'] + armatures = [ + obj for obj in collections[0].objects if obj.type == 'ARMATURE'] object_action_pairs = [] original_actions = [] @@ -68,15 +78,15 @@ class ExtractAnimationFBX(pype.api.Extractor): curr_frame_range = curr_action.frame_range - starting_frames.append( curr_frame_range[0] ) - ending_frames.append( curr_frame_range[1] ) + starting_frames.append(curr_frame_range[0]) + ending_frames.append(curr_frame_range[1]) object_action_pairs.append((obj, copy_action)) original_actions.append(curr_action) # We compute the starting and ending frames - max_frame = min( starting_frames ) - min_frame = max( ending_frames ) + max_frame = min(starting_frames) + min_frame = max(ending_frames) # We bake the copy of the current action for each object bpy_extras.anim_utils.bake_action_objects( @@ -95,21 +105,24 @@ class ExtractAnimationFBX(pype.api.Extractor): add_leaf_bones=False ) - bpy.context.view_layer.active_layer_collection = old_active_layer_collection + view_layer.active_layer_collection = old_active_layer_collection - bpy.context.scene.unit_settings.scale_length = old_scale + scene.unit_settings.scale_length = old_scale # We delete the baked action and set the original one back for i in range(0, len(object_action_pairs)): - if original_actions[i]: + pair = object_action_pairs[i] + action = original_actions[i] - object_action_pairs[i][0].animation_data.action = original_actions[i] + if action: - if object_action_pairs[i][1]: + pair[0].animation_data.action = action - object_action_pairs[i][1].user_clear() - bpy.data.actions.remove(object_action_pairs[i][1]) + if pair[1]: + + pair[1].user_clear() + bpy.data.actions.remove(pair[1]) if "representations" not in instance.data: instance.data["representations"] = [] From 40cf778f106c32deec2d850d0c72c906537ebfcf Mon Sep 17 00:00:00 2001 From: Simone Barbieri Date: Fri, 20 Mar 2020 16:46:45 +0000 Subject: [PATCH 085/118] More PEP8 compliance --- pype/blender/plugin.py | 5 ++++- pype/plugins/blender/create/create_action.py | 4 ++-- pype/plugins/blender/load/load_action.py | 2 +- pype/plugins/blender/publish/collect_action.py | 4 ++-- pype/plugins/blender/publish/collect_animation.py | 6 +++--- pype/plugins/blender/publish/collect_model.py | 4 ++-- pype/plugins/blender/publish/collect_rig.py | 4 ++-- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index 5e98d8314b..8f72d04a1d 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -19,6 +19,7 @@ def asset_name( name = f"{namespace}:{name}" return name + def create_blender_context(obj: Optional[bpy.types.Object] = None): """Create a new Blender context. If an object is passed as parameter, it is set as selected and active. @@ -40,6 +41,7 @@ def create_blender_context(obj: Optional[bpy.types.Object] = None): return override_context raise Exception("Could not create a custom Blender context.") + class AssetLoader(api.Loader): """A basic AssetLoader for Blender @@ -89,7 +91,8 @@ class AssetLoader(api.Loader): assert obj.library, f"'{obj.name}' is not linked." libraries.add(obj.library) - assert len(libraries) == 1, "'{container.name}' contains objects from more then 1 library." + assert len( + libraries) == 1, "'{container.name}' contains objects from more then 1 library." return list(libraries)[0] diff --git a/pype/plugins/blender/create/create_action.py b/pype/plugins/blender/create/create_action.py index 6c24065f81..68e2a50b61 100644 --- a/pype/plugins/blender/create/create_action.py +++ b/pype/plugins/blender/create/create_action.py @@ -27,8 +27,8 @@ class CreateAction(Creator): if (self.options or {}).get("useSelection"): for obj in lib.get_selection(): - if (obj.animation_data is not None and - obj.animation_data.action is not None): + if (obj.animation_data is not None + and obj.animation_data.action is not None): empty_obj = bpy.data.objects.new(name=name, object_data=None) diff --git a/pype/plugins/blender/load/load_action.py b/pype/plugins/blender/load/load_action.py index 303d1ead4d..a1b1ad3cea 100644 --- a/pype/plugins/blender/load/load_action.py +++ b/pype/plugins/blender/load/load_action.py @@ -175,7 +175,7 @@ class BlendActionLoader(pype.blender.plugin.AssetLoader): strips = [] for obj in collection_metadata["objects"]: - + # Get all the strips that use the action arm_objs = [ arm for arm in bpy.data.objects if arm.type == 'ARMATURE'] diff --git a/pype/plugins/blender/publish/collect_action.py b/pype/plugins/blender/publish/collect_action.py index a8ceed9c82..c359198490 100644 --- a/pype/plugins/blender/publish/collect_action.py +++ b/pype/plugins/blender/publish/collect_action.py @@ -23,8 +23,8 @@ class CollectAction(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'action' and - not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'action' + and not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_animation.py b/pype/plugins/blender/publish/collect_animation.py index 50d49692b8..681f945f25 100644 --- a/pype/plugins/blender/publish/collect_animation.py +++ b/pype/plugins/blender/publish/collect_animation.py @@ -18,13 +18,13 @@ class CollectAnimation(pyblish.api.ContextPlugin): """Return all 'animation' collections. Check if the family is 'animation' and if it doesn't have the - representation set. If the representation is set, it is a loaded + representation set. If the representation is set, it is a loaded animation and we don't want to publish it. """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'animation' and - not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'animation' + and not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_model.py b/pype/plugins/blender/publish/collect_model.py index df5c1e709a..5cbd097a4e 100644 --- a/pype/plugins/blender/publish/collect_model.py +++ b/pype/plugins/blender/publish/collect_model.py @@ -23,8 +23,8 @@ class CollectModel(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'model' and - not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'model' + and not avalon_prop.get('representation')): yield collection def process(self, context): diff --git a/pype/plugins/blender/publish/collect_rig.py b/pype/plugins/blender/publish/collect_rig.py index 01958da37a..730f209e89 100644 --- a/pype/plugins/blender/publish/collect_rig.py +++ b/pype/plugins/blender/publish/collect_rig.py @@ -23,8 +23,8 @@ class CollectRig(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'rig' and - not avalon_prop.get('representation')): + if (avalon_prop.get('family') == 'rig' + and not avalon_prop.get('representation')): yield collection def process(self, context): From 31f0ab63788037a11c1fecb7817bb621651159ad Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:35:23 +0100 Subject: [PATCH 086/118] feat(nuke): prerender node dont need `review` knob --- pype/nuke/lib.py | 38 +++++++++++++------ .../nuke/create/create_write_prerender.py | 4 +- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index 8e241dad16..989cbf569f 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -215,14 +215,14 @@ def script_name(): def add_button_write_to_read(node): name = "createReadNode" - label = "Create Read" + label = "[ Create Read ]" value = "import write_to_read;write_to_read.write_to_read(nuke.thisNode())" k = nuke.PyScript_Knob(name, label, value) k.setFlag(0x1000) node.addKnob(k) -def create_write_node(name, data, input=None, prenodes=None): +def create_write_node(name, data, input=None, prenodes=None, review=True): ''' Creating write node which is group node Arguments: @@ -231,6 +231,7 @@ def create_write_node(name, data, input=None, prenodes=None): input (node): selected node to connect to prenodes (list, optional): list of lists, definitions for nodes to be created before write + review (bool): adding review knob Example: prenodes = [( @@ -380,15 +381,8 @@ def create_write_node(name, data, input=None, prenodes=None): add_rendering_knobs(GN) - # adding write to read button - add_button_write_to_read(GN) - - divider = nuke.Text_Knob('') - GN.addKnob(divider) - - # set tile color - tile_color = _data.get("tile_color", "0xff0000ff") - GN["tile_color"].setValue(tile_color) + if review: + add_review_knob(GN) # add render button lnk = nuke.Link_Knob("Render") @@ -396,9 +390,20 @@ def create_write_node(name, data, input=None, prenodes=None): lnk.setName("Render") GN.addKnob(lnk) + divider = nuke.Text_Knob('') + GN.addKnob(divider) + + # adding write to read button + add_button_write_to_read(GN) + # Deadline tab. add_deadline_tab(GN) + + # set tile color + tile_color = _data.get("tile_color", "0xff0000ff") + GN["tile_color"].setValue(tile_color) + return GN @@ -420,6 +425,17 @@ def add_rendering_knobs(node): knob = nuke.Boolean_Knob("render_farm", "Render on Farm") knob.setValue(False) node.addKnob(knob) + return node + +def add_review_knob(node): + ''' Adds additional review knob to given node + + Arguments: + node (obj): nuke node object to be fixed + + Return: + node (obj): with added knob + ''' if "review" not in node.knobs(): knob = nuke.Boolean_Knob("review", "Review") knob.setValue(True) diff --git a/pype/plugins/nuke/create/create_write_prerender.py b/pype/plugins/nuke/create/create_write_prerender.py index f8210db9db..6e242f886c 100644 --- a/pype/plugins/nuke/create/create_write_prerender.py +++ b/pype/plugins/nuke/create/create_write_prerender.py @@ -87,7 +87,9 @@ class CreateWritePrerender(plugin.PypeCreator): self.data["subset"], write_data, input=selected_node, - prenodes=[]) + prenodes=[], + review=False + ) # relinking to collected connections for i, input in enumerate(inputs): From 5c7bba0c0676a95a35b1de2e03c523989d7197b2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:36:13 +0100 Subject: [PATCH 087/118] feat(nuke): setting up properly families --- pype/plugins/nuke/publish/collect_review.py | 2 +- pype/plugins/nuke/publish/collect_slate_node.py | 2 +- pype/plugins/nuke/publish/extract_render_local.py | 2 +- pype/plugins/nuke/publish/submit_nuke_deadline.py | 2 +- pype/plugins/nuke/publish/validate_write_bounding_box.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_review.py b/pype/plugins/nuke/publish/collect_review.py index f02f22e053..c95c94541d 100644 --- a/pype/plugins/nuke/publish/collect_review.py +++ b/pype/plugins/nuke/publish/collect_review.py @@ -9,7 +9,7 @@ class CollectReview(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.3 label = "Collect Review" hosts = ["nuke"] - families = ["write", "prerender"] + families = ["render", "render.local", "render.farm"] def process(self, instance): diff --git a/pype/plugins/nuke/publish/collect_slate_node.py b/pype/plugins/nuke/publish/collect_slate_node.py index d8d6b50f05..9c7f1b5e95 100644 --- a/pype/plugins/nuke/publish/collect_slate_node.py +++ b/pype/plugins/nuke/publish/collect_slate_node.py @@ -8,7 +8,7 @@ class CollectSlate(pyblish.api.InstancePlugin): order = pyblish.api.CollectorOrder + 0.09 label = "Collect Slate Node" hosts = ["nuke"] - families = ["write"] + families = ["render", "render.local", "render.farm"] def process(self, instance): node = instance[0] diff --git a/pype/plugins/nuke/publish/extract_render_local.py b/pype/plugins/nuke/publish/extract_render_local.py index 5467d239c2..1dad413ee5 100644 --- a/pype/plugins/nuke/publish/extract_render_local.py +++ b/pype/plugins/nuke/publish/extract_render_local.py @@ -17,7 +17,7 @@ class NukeRenderLocal(pype.api.Extractor): order = pyblish.api.ExtractorOrder label = "Render Local" hosts = ["nuke"] - families = ["render.local"] + families = ["render.local", "prerender.local"] def process(self, instance): node = None diff --git a/pype/plugins/nuke/publish/submit_nuke_deadline.py b/pype/plugins/nuke/publish/submit_nuke_deadline.py index 0a9ef33398..3da2e58e4d 100644 --- a/pype/plugins/nuke/publish/submit_nuke_deadline.py +++ b/pype/plugins/nuke/publish/submit_nuke_deadline.py @@ -19,7 +19,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): label = "Submit to Deadline" order = pyblish.api.IntegratorOrder + 0.1 hosts = ["nuke", "nukestudio"] - families = ["render.farm"] + families = ["render.farm", "prerender.farm"] optional = True deadline_priority = 50 diff --git a/pype/plugins/nuke/publish/validate_write_bounding_box.py b/pype/plugins/nuke/publish/validate_write_bounding_box.py index cedeea6d9f..e4b7c77a25 100644 --- a/pype/plugins/nuke/publish/validate_write_bounding_box.py +++ b/pype/plugins/nuke/publish/validate_write_bounding_box.py @@ -57,7 +57,7 @@ class ValidateNukeWriteBoundingBox(pyblish.api.InstancePlugin): order = pyblish.api.ValidatorOrder optional = True - families = ["write"] + families = ["render", "render.local", "render.farm"] label = "Write Bounding Box" hosts = ["nuke"] actions = [RepairNukeBoundingBoxAction] From 63eac17649a7c42b4ba4f8d93c06f0dfad1af346 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:36:46 +0100 Subject: [PATCH 088/118] clean(nuke): old code and better definition of families --- pype/plugins/nuke/publish/collect_writes.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index aa5f825a98..f3f33b7a6d 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -105,8 +105,6 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): representation['files'] = collected_frames instance.data["representations"].append(representation) - if "render" not in instance.data["families"]: - instance.data["families"].append("render") except Exception: instance.data["representations"].append(representation) self.log.debug("couldn't collect frames: {}".format(label)) @@ -127,6 +125,7 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): deadlinePriority = group_node["deadlinePriority"].value() families = [f for f in instance.data["families"] if "write" not in f] + instance.data.update({ "versionData": version_data, "path": path, @@ -147,18 +146,5 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): "deadlinePriority": deadlinePriority }) self.log.debug("families: {}".format(families)) - if "prerender" in families: - _families = list() - for fm in families: - if fm in _families: - continue - if "render" in fm: - if "prerender" in fm: - continue - _families.append(fm) - instance.data.update({ - "family": "prerender", - "families": _families - }) - self.log.debug("_families: {}".format(_families)) + self.log.debug("instance.data: {}".format(instance.data)) From 7fbb72b8a6eecdaf0d3799b59a248aa6cded0aa3 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:37:10 +0100 Subject: [PATCH 089/118] clean(nuke): improving code --- .../plugins/nuke/publish/collect_instances.py | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_instances.py b/pype/plugins/nuke/publish/collect_instances.py index 893c6db7e5..54891d189c 100644 --- a/pype/plugins/nuke/publish/collect_instances.py +++ b/pype/plugins/nuke/publish/collect_instances.py @@ -52,6 +52,7 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): # establish families family = avalon_knob_data["family"] + families_ak = avalon_knob_data.get("families") families = list() # except disabled nodes but exclude backdrops in test @@ -68,20 +69,16 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): # Add all nodes in group instances. if node.Class() == "Group": # only alter families for render family - if ("render" in family): - # check if node is not disabled - families.append(avalon_knob_data["families"]) + if "write" in families_ak: if node["render"].value(): self.log.info("flagged for render") - add_family = "render.local" + add_family = "{}.local".format(family) # dealing with local/farm rendering if node["render_farm"].value(): self.log.info("adding render farm family") - add_family = "render.farm" + add_family = "{}.farm".format(family) instance.data["transfer"] = False families.append(add_family) - if "prerender" in family: - families.append("prerender") else: # add family into families families.insert(0, family) @@ -91,7 +88,7 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): instance.append(i) node.end() - families_ak = avalon_knob_data.get("families") + self.log.debug("__ families: `{}`".format(families)) if families_ak: families.append(families_ak) @@ -104,22 +101,6 @@ class CollectNukeInstances(pyblish.api.ContextPlugin): resolution_height = format.height() pixel_aspect = format.pixelAspect() - if node.Class() not in "Read": - if "render" not in node.knobs().keys(): - pass - elif node["render"].value(): - self.log.info("flagged for render") - add_family = "render.local" - # dealing with local/farm rendering - if node["render_farm"].value(): - self.log.info("adding render farm family") - add_family = "render.farm" - instance.data["transfer"] = False - families.append(add_family) - else: - # add family into families - families.insert(0, family) - instance.data.update({ "subset": subset, "asset": os.environ["AVALON_ASSET"], From b3702c49df14b2351cfc9c99e8987fb4e5685896 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:37:37 +0100 Subject: [PATCH 090/118] feat(global): explicit families for integrate new --- pype/plugins/global/publish/integrate_new.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index aa214f36cb..ddb40e321a 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -64,6 +64,9 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "scene", "vrayproxy", "render", + "render.local", + "prerender", + "prerender.local", "imagesequence", "review", "rendersetup", From 9a1167c26d784a43bb6643b4fcd6514d2aebe7b0 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 20 Mar 2020 21:38:11 +0100 Subject: [PATCH 091/118] feat(nuke): dealing with `prerender` family when submitting to deadline --- pype/plugins/global/publish/submit_publish_job.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index dcf19ae32c..0b7a8473d4 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -141,7 +141,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): hosts = ["fusion", "maya", "nuke"] - families = ["render.farm", "renderlayer", "imagesequence"] + families = ["render.farm", "prerender.farm", "renderlayer", "imagesequence"] aov_filter = {"maya": ["beauty"]} @@ -583,6 +583,12 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "resolutionHeight": data.get("resolutionHeight", 1080), } + if "prerender.farm" in instance.data["families"]: + instance_skeleton_data.update({ + "family": "prerender", + "families": ["prerender"] + }) + # transfer specific families from original instance to new render for item in self.families_transfer: if item in instance.data.get("families", []): From 043d8225a368510acb3427a92ae2672e23501367 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Sat, 21 Mar 2020 01:07:02 +0100 Subject: [PATCH 092/118] unify collectors --- .../plugins/blender/publish/collect_action.py | 51 ------------------- .../blender/publish/collect_animation.py | 51 ------------------- ...{collect_model.py => collect_instances.py} | 17 ++++--- pype/plugins/blender/publish/collect_rig.py | 51 ------------------- 4 files changed, 10 insertions(+), 160 deletions(-) delete mode 100644 pype/plugins/blender/publish/collect_action.py delete mode 100644 pype/plugins/blender/publish/collect_animation.py rename pype/plugins/blender/publish/{collect_model.py => collect_instances.py} (78%) delete mode 100644 pype/plugins/blender/publish/collect_rig.py diff --git a/pype/plugins/blender/publish/collect_action.py b/pype/plugins/blender/publish/collect_action.py deleted file mode 100644 index c359198490..0000000000 --- a/pype/plugins/blender/publish/collect_action.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Generator - -import bpy - -import pyblish.api -from avalon.blender.pipeline import AVALON_PROPERTY - - -class CollectAction(pyblish.api.ContextPlugin): - """Collect the data of an action.""" - - hosts = ["blender"] - label = "Collect Action" - order = pyblish.api.CollectorOrder - - @staticmethod - def get_action_collections() -> Generator: - """Return all 'animation' collections. - - Check if the family is 'action' and if it doesn't have the - representation set. If the representation is set, it is a loaded action - and we don't want to publish it. - """ - for collection in bpy.data.collections: - avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'action' - and not avalon_prop.get('representation')): - yield collection - - def process(self, context): - """Collect the actions from the current Blender scene.""" - collections = self.get_action_collections() - for collection in collections: - avalon_prop = collection[AVALON_PROPERTY] - asset = avalon_prop['asset'] - family = avalon_prop['family'] - subset = avalon_prop['subset'] - task = avalon_prop['task'] - name = f"{asset}_{subset}" - instance = context.create_instance( - name=name, - family=family, - families=[family], - subset=subset, - asset=asset, - task=task, - ) - members = list(collection.objects) - members.append(collection) - instance[:] = members - self.log.debug(instance.data) diff --git a/pype/plugins/blender/publish/collect_animation.py b/pype/plugins/blender/publish/collect_animation.py deleted file mode 100644 index 681f945f25..0000000000 --- a/pype/plugins/blender/publish/collect_animation.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Generator - -import bpy - -import pyblish.api -from avalon.blender.pipeline import AVALON_PROPERTY - - -class CollectAnimation(pyblish.api.ContextPlugin): - """Collect the data of an animation.""" - - hosts = ["blender"] - label = "Collect Animation" - order = pyblish.api.CollectorOrder - - @staticmethod - def get_animation_collections() -> Generator: - """Return all 'animation' collections. - - Check if the family is 'animation' and if it doesn't have the - representation set. If the representation is set, it is a loaded - animation and we don't want to publish it. - """ - for collection in bpy.data.collections: - avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'animation' - and not avalon_prop.get('representation')): - yield collection - - def process(self, context): - """Collect the animations from the current Blender scene.""" - collections = self.get_animation_collections() - for collection in collections: - avalon_prop = collection[AVALON_PROPERTY] - asset = avalon_prop['asset'] - family = avalon_prop['family'] - subset = avalon_prop['subset'] - task = avalon_prop['task'] - name = f"{asset}_{subset}" - instance = context.create_instance( - name=name, - family=family, - families=[family], - subset=subset, - asset=asset, - task=task, - ) - members = list(collection.objects) - members.append(collection) - instance[:] = members - self.log.debug(instance.data) diff --git a/pype/plugins/blender/publish/collect_model.py b/pype/plugins/blender/publish/collect_instances.py similarity index 78% rename from pype/plugins/blender/publish/collect_model.py rename to pype/plugins/blender/publish/collect_instances.py index 5cbd097a4e..1d3693216d 100644 --- a/pype/plugins/blender/publish/collect_model.py +++ b/pype/plugins/blender/publish/collect_instances.py @@ -1,20 +1,21 @@ from typing import Generator import bpy +import json import pyblish.api from avalon.blender.pipeline import AVALON_PROPERTY -class CollectModel(pyblish.api.ContextPlugin): +class CollectInstances(pyblish.api.ContextPlugin): """Collect the data of a model.""" hosts = ["blender"] - label = "Collect Model" + label = "Collect Instances" order = pyblish.api.CollectorOrder @staticmethod - def get_model_collections() -> Generator: + def get_collections() -> Generator: """Return all 'model' collections. Check if the family is 'model' and if it doesn't have the @@ -23,13 +24,13 @@ class CollectModel(pyblish.api.ContextPlugin): """ for collection in bpy.data.collections: avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'model' - and not avalon_prop.get('representation')): + if avalon_prop.get('id') == 'pyblish.avalon.instance': yield collection def process(self, context): """Collect the models from the current Blender scene.""" - collections = self.get_model_collections() + collections = self.get_collections() + for collection in collections: avalon_prop = collection[AVALON_PROPERTY] asset = avalon_prop['asset'] @@ -48,4 +49,6 @@ class CollectModel(pyblish.api.ContextPlugin): members = list(collection.objects) members.append(collection) instance[:] = members - self.log.debug(instance.data) + self.log.debug(json.dumps(instance.data, indent=4)) + for obj in instance: + self.log.debug(obj) diff --git a/pype/plugins/blender/publish/collect_rig.py b/pype/plugins/blender/publish/collect_rig.py deleted file mode 100644 index 730f209e89..0000000000 --- a/pype/plugins/blender/publish/collect_rig.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Generator - -import bpy - -import pyblish.api -from avalon.blender.pipeline import AVALON_PROPERTY - - -class CollectRig(pyblish.api.ContextPlugin): - """Collect the data of a rig.""" - - hosts = ["blender"] - label = "Collect Rig" - order = pyblish.api.CollectorOrder - - @staticmethod - def get_rig_collections() -> Generator: - """Return all 'rig' collections. - - Check if the family is 'rig' and if it doesn't have the - representation set. If the representation is set, it is a loaded rig - and we don't want to publish it. - """ - for collection in bpy.data.collections: - avalon_prop = collection.get(AVALON_PROPERTY) or dict() - if (avalon_prop.get('family') == 'rig' - and not avalon_prop.get('representation')): - yield collection - - def process(self, context): - """Collect the rigs from the current Blender scene.""" - collections = self.get_rig_collections() - for collection in collections: - avalon_prop = collection[AVALON_PROPERTY] - asset = avalon_prop['asset'] - family = avalon_prop['family'] - subset = avalon_prop['subset'] - task = avalon_prop['task'] - name = f"{asset}_{subset}" - instance = context.create_instance( - name=name, - family=family, - families=[family], - subset=subset, - asset=asset, - task=task, - ) - members = list(collection.objects) - members.append(collection) - instance[:] = members - self.log.debug(instance.data) From 00bec457481636d48be8ec516786b83594061a82 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Sat, 21 Mar 2020 01:07:34 +0100 Subject: [PATCH 093/118] naive fix to crashing uv validator --- .../blender/publish/validate_mesh_has_uv.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pype/plugins/blender/publish/validate_mesh_has_uv.py b/pype/plugins/blender/publish/validate_mesh_has_uv.py index b71a40ad8f..d0cd33645b 100644 --- a/pype/plugins/blender/publish/validate_mesh_has_uv.py +++ b/pype/plugins/blender/publish/validate_mesh_has_uv.py @@ -35,12 +35,15 @@ class ValidateMeshHasUvs(pyblish.api.InstancePlugin): invalid = [] # TODO (jasper): only check objects in the collection that will be published? for obj in [ - obj for obj in bpy.data.objects if obj.type == 'MESH' - ]: - # Make sure we are in object mode. - bpy.ops.object.mode_set(mode='OBJECT') - if not cls.has_uvs(obj): - invalid.append(obj) + obj for obj in instance]: + try: + if obj.type == 'MESH': + # Make sure we are in object mode. + bpy.ops.object.mode_set(mode='OBJECT') + if not cls.has_uvs(obj): + invalid.append(obj) + except: + continue return invalid def process(self, instance): From 28c626b69469c1b46e3939022ab7e997c2c7c09c Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Sat, 21 Mar 2020 01:07:59 +0100 Subject: [PATCH 094/118] attempt at alembic extractor --- pype/blender/plugin.py | 11 ++- pype/plugins/blender/publish/extract_abc.py | 91 +++++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 pype/plugins/blender/publish/extract_abc.py diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index 8f72d04a1d..f27bf0daab 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -20,10 +20,15 @@ def asset_name( return name -def create_blender_context(obj: Optional[bpy.types.Object] = None): +def create_blender_context(active: Optional[bpy.types.Object] = None, + selected: Optional[bpy.types.Object] = None,): """Create a new Blender context. If an object is passed as parameter, it is set as selected and active. """ + + if not isinstance(selected, list): + selected = [selected] + for win in bpy.context.window_manager.windows: for area in win.screen.areas: if area.type == 'VIEW_3D': @@ -35,8 +40,8 @@ def create_blender_context(obj: Optional[bpy.types.Object] = None): 'area': area, 'region': region, 'scene': bpy.context.scene, - 'active_object': obj, - 'selected_objects': [obj] + 'active_object': selected[0], + 'selected_objects': selected } return override_context raise Exception("Could not create a custom Blender context.") diff --git a/pype/plugins/blender/publish/extract_abc.py b/pype/plugins/blender/publish/extract_abc.py new file mode 100644 index 0000000000..b953d41ba2 --- /dev/null +++ b/pype/plugins/blender/publish/extract_abc.py @@ -0,0 +1,91 @@ +import os + +import pype.api +import pype.blender.plugin + +import bpy + + +class ExtractABC(pype.api.Extractor): + """Extract as ABC.""" + + label = "Extract ABC" + hosts = ["blender"] + families = ["model"] + optional = True + + def process(self, instance): + # Define extract output file path + + stagingdir = self.staging_dir(instance) + filename = f"{instance.name}.fbx" + filepath = os.path.join(stagingdir, filename) + + context = bpy.context + scene = context.scene + view_layer = context.view_layer + + # Perform extraction + self.log.info("Performing extraction..") + + collections = [ + obj for obj in instance if type(obj) is bpy.types.Collection] + + assert len(collections) == 1, "There should be one and only one " \ + "collection collected for this asset" + + old_active_layer_collection = view_layer.active_layer_collection + + layers = view_layer.layer_collection.children + + # Get the layer collection from the collection we need to export. + # This is needed because in Blender you can only set the active + # collection with the layer collection, and there is no way to get + # the layer collection from the collection + # (but there is the vice versa). + layer_collections = [ + layer for layer in layers if layer.collection == collections[0]] + + assert len(layer_collections) == 1 + + view_layer.active_layer_collection = layer_collections[0] + + old_scale = scene.unit_settings.scale_length + + selected = list() + + for obj in instance: + selected.append(obj) + + new_context = pype.blender.plugin.create_blender_context(active=None, selected=selected) + + # We set the scale of the scene for the export + scene.unit_settings.scale_length = 0.01 + + self.log.info(new_context) + + # We export the abc + bpy.ops.wm.alembic_export( + new_context, + filepath=filepath, + start=1, + end=1 + ) + + view_layer.active_layer_collection = old_active_layer_collection + + scene.unit_settings.scale_length = old_scale + + if "representations" not in instance.data: + instance.data["representations"] = [] + + representation = { + 'name': 'abc', + 'ext': 'abc', + 'files': filename, + "stagingDir": stagingdir, + } + instance.data["representations"].append(representation) + + self.log.info("Extracted instance '%s' to: %s", + instance.name, representation) From 59dd912a7626e3e98238c11a4c38ebaf53162f23 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Sat, 21 Mar 2020 01:26:40 +0100 Subject: [PATCH 095/118] tweak context override --- pype/blender/plugin.py | 2 +- pype/plugins/blender/publish/extract_abc.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pype/blender/plugin.py b/pype/blender/plugin.py index f27bf0daab..77fce90d65 100644 --- a/pype/blender/plugin.py +++ b/pype/blender/plugin.py @@ -40,7 +40,7 @@ def create_blender_context(active: Optional[bpy.types.Object] = None, 'area': area, 'region': region, 'scene': bpy.context.scene, - 'active_object': selected[0], + 'active_object': active, 'selected_objects': selected } return override_context diff --git a/pype/plugins/blender/publish/extract_abc.py b/pype/plugins/blender/publish/extract_abc.py index b953d41ba2..d2c0c769ae 100644 --- a/pype/plugins/blender/publish/extract_abc.py +++ b/pype/plugins/blender/publish/extract_abc.py @@ -55,9 +55,13 @@ class ExtractABC(pype.api.Extractor): selected = list() for obj in instance: - selected.append(obj) + try: + obj.select_set(True) + selected.append(obj) + except: + continue - new_context = pype.blender.plugin.create_blender_context(active=None, selected=selected) + new_context = pype.blender.plugin.create_blender_context(active=selected[0], selected=selected) # We set the scale of the scene for the export scene.unit_settings.scale_length = 0.01 From e5fd3d3b5df0f2717d5a566264940ddabcaf8ad9 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Sat, 21 Mar 2020 19:13:53 +0100 Subject: [PATCH 096/118] frame range validator remake --- .../maya/publish/validate_frame_range.py | 158 +++++++++++++++--- 1 file changed, 135 insertions(+), 23 deletions(-) diff --git a/pype/plugins/maya/publish/validate_frame_range.py b/pype/plugins/maya/publish/validate_frame_range.py index d4aad812d5..26235b37ae 100644 --- a/pype/plugins/maya/publish/validate_frame_range.py +++ b/pype/plugins/maya/publish/validate_frame_range.py @@ -1,18 +1,18 @@ import pyblish.api import pype.api +from maya import cmds + class ValidateFrameRange(pyblish.api.InstancePlugin): """Valides the frame ranges. - Checks the `startFrame`, `endFrame` and `handles` data. - This does NOT ensure there's actual data present. + This is optional validator checking if the frame range matches the one of + asset. - This validates: - - `startFrame` is lower than or equal to the `endFrame`. - - must have both the `startFrame` and `endFrame` data. - - The `handles` value is not lower than zero. + Repair action will change everything to match asset. + This can be turned off by artist to allow custom ranges. """ label = "Validate Frame Range" @@ -20,26 +20,138 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): families = ["animation", "pointcache", "camera", - "renderlayer", - "colorbleed.vrayproxy"] + "render", + "review", + "yeticache"] + optional = True + actions = [pype.api.RepairAction] def process(self, instance): + context = instance.context - start = instance.data.get("frameStart", None) - end = instance.data.get("frameEnd", None) - handles = instance.data.get("handles", None) + frame_start_handle = int(context.data.get("frameStartHandle")) + frame_end_handle = int(context.data.get("frameEndHandle")) + handles = int(context.data.get("handles")) + handle_start = int(context.data.get("handleStart")) + handle_end = int(context.data.get("handleEnd")) + frame_start = int(context.data.get("frameStart")) + frame_end = int(context.data.get("frameEnd")) - # Check if any of the values are present - if any(value is None for value in [start, end]): - raise ValueError("No time values for this instance. " - "(Missing `startFrame` or `endFrame`)") + inst_start = int(instance.data.get("frameStartHandle")) + inst_end = int(instance.data.get("frameEndHandle")) - self.log.info("Comparing start (%s) and end (%s)" % (start, end)) - if start > end: - raise RuntimeError("The start frame is a higher value " - "than the end frame: " - "{0}>{1}".format(start, end)) + # basic sanity checks + assert frame_start_handle <= frame_end_handle, ( + "start frame is lower then end frame") - if handles is not None: - if handles < 0.0: - raise RuntimeError("Handles are set to a negative value") + assert handles >= 0, ("handles cannot have negative values") + + # compare with data on instance + errors = [] + + if(inst_start != frame_start_handle): + errors.append("Instance start frame [ {} ] doesn't " + "match the one set on instance [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + inst_start, + frame_start_handle, + handle_start, frame_start, frame_end, handle_end + )) + + if(inst_end != frame_end_handle): + errors.append("Instance end frame [ {} ] doesn't " + "match the one set on instance [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + inst_end, + frame_end_handle, + handle_start, frame_start, frame_end, handle_end + )) + + minTime = int(cmds.playbackOptions(minTime=True, query=True)) + maxTime = int(cmds.playbackOptions(maxTime=True, query=True)) + animStartTime = int(cmds.playbackOptions(animationStartTime=True, + query=True)) + animEndTime = int(cmds.playbackOptions(animationEndTime=True, + query=True)) + + if int(minTime) != inst_start: + errors.append("Start of Maya timeline is set to frame [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + minTime, + frame_start_handle, + handle_start, frame_start, frame_end, handle_end + )) + + if int(maxTime) != inst_end: + errors.append("End of Maya timeline is set to frame [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + maxTime, + frame_end_handle, + handle_start, frame_start, frame_end, handle_end + )) + + if int(animStartTime) != inst_start: + errors.append("Animation start in Maya is set to frame [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + animStartTime, + frame_start_handle, + handle_start, frame_start, frame_end, handle_end + )) + + if int(animEndTime) != inst_end: + errors.append("Animation start in Maya is set to frame [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + animEndTime, + frame_end_handle, + handle_start, frame_start, frame_end, handle_end + )) + + render_start = int(cmds.getAttr("defaultRenderGlobals.startFrame")) + render_end = int(cmds.getAttr("defaultRenderGlobals.endFrame")) + + if int(render_start) != inst_start: + errors.append("Render settings start frame is set to [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + int(render_start), + frame_start_handle, + handle_start, frame_start, frame_end, handle_end + )) + + if int(render_end) != inst_end: + errors.append("Render settings end frame is set to [ {} ] " + " and doesn't match the one set on asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + int(render_end), + frame_end_handle, + handle_start, frame_start, frame_end, handle_end + )) + + for e in errors: + self.log.error(e) + + assert len(errors) == 0, ("Frame range settings are incorrect") + + @classmethod + def repair(cls, instance): + """ + Repair by calling avalon reset frame range function. This will set + timeline frame range, render settings range and frame information + on instance container to match asset data. + """ + import avalon.maya.interactive + avalon.maya.interactive.reset_frame_range() + cls.log.debug("-" * 80) + cls.log.debug("{}.frameStart".format(instance.data["name"])) + cmds.setAttr( + "{}.frameStart".format(instance.data["name"]), + instance.context.data.get("frameStartHandle")) + + cmds.setAttr( + "{}.frameEnd".format(instance.data["name"]), + instance.context.data.get("frameEndHandle")) + cls.log.debug("-" * 80) From 8d32d77668ebb7b535f5b660d72bfec367788d30 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Sat, 21 Mar 2020 21:00:39 +0100 Subject: [PATCH 097/118] remove debug prints --- pype/plugins/maya/publish/validate_frame_range.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pype/plugins/maya/publish/validate_frame_range.py b/pype/plugins/maya/publish/validate_frame_range.py index 26235b37ae..0be77644a0 100644 --- a/pype/plugins/maya/publish/validate_frame_range.py +++ b/pype/plugins/maya/publish/validate_frame_range.py @@ -145,8 +145,6 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): """ import avalon.maya.interactive avalon.maya.interactive.reset_frame_range() - cls.log.debug("-" * 80) - cls.log.debug("{}.frameStart".format(instance.data["name"])) cmds.setAttr( "{}.frameStart".format(instance.data["name"]), instance.context.data.get("frameStartHandle")) @@ -154,4 +152,3 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): cmds.setAttr( "{}.frameEnd".format(instance.data["name"]), instance.context.data.get("frameEndHandle")) - cls.log.debug("-" * 80) From 0e991c4fb572ebfb3a54c8d9e8b882f8377fdba1 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Sat, 21 Mar 2020 23:21:41 +0100 Subject: [PATCH 098/118] shutting up hound --- .../plugins/maya/publish/collect_instances.py | 9 ++++---- pype/plugins/maya/publish/collect_render.py | 21 ++++++++----------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/pype/plugins/maya/publish/collect_instances.py b/pype/plugins/maya/publish/collect_instances.py index 9ea3ebe7fa..1d59a68bf6 100644 --- a/pype/plugins/maya/publish/collect_instances.py +++ b/pype/plugins/maya/publish/collect_instances.py @@ -122,7 +122,7 @@ class CollectInstances(pyblish.api.ContextPlugin): # if frame range on maya set is the same as full shot range # adjust the values to match the asset data if (ctx_frame_start_handle == data["frameStart"] - and ctx_frame_end_handle == data["frameEnd"]): + and ctx_frame_end_handle == data["frameEnd"]): # noqa: W503, E501 data["frameStartHandle"] = ctx_frame_start_handle data["frameEndHandle"] = ctx_frame_end_handle data["frameStart"] = ctx_frame_start @@ -141,8 +141,8 @@ class CollectInstances(pyblish.api.ContextPlugin): data["handleStart"] = 0 data["handleEnd"] = 0 - data["frameStartHandle"] = data["frameStart"] - data["handleStart"] - data["frameEndHandle"] = data["frameEnd"] + data["handleEnd"] + data["frameStartHandle"] = data["frameStart"] - data["handleStart"] # noqa: E501 + data["frameEndHandle"] = data["frameEnd"] + data["handleEnd"] # noqa: E501 if "handles" in data: data.pop('handles') @@ -157,7 +157,8 @@ class CollectInstances(pyblish.api.ContextPlugin): # Produce diagnostic message for any graphical # user interface interested in visualising it. self.log.info("Found: \"%s\" " % instance.data["name"]) - self.log.debug("DATA: {} ".format(json.dumps(instance.data, indent=4))) + self.log.debug( + "DATA: {} ".format(json.dumps(instance.data, indent=4))) def sort_by_family(instance): """Sort by family""" diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index 88c1be477d..365b0b5a13 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -203,13 +203,13 @@ class CollectMayaRender(pyblish.api.ContextPlugin): full_paths.append(full_path) aov_dict["beauty"] = full_paths - frame_start_render = int(self.get_render_attribute("startFrame", - layer=layer_name)) - frame_end_render = int(self.get_render_attribute("endFrame", - layer=layer_name)) + frame_start_render = int(self.get_render_attribute( + "startFrame", layer=layer_name)) + frame_end_render = int(self.get_render_attribute( + "endFrame", layer=layer_name)) - if (int(context.data['frameStartHandle']) == frame_start_render and - int(context.data['frameEndHandle']) == frame_end_render): + if (int(context.data['frameStartHandle']) == frame_start_render + and int(context.data['frameEndHandle']) == frame_end_render): # noqa: W503, E501 handle_start = context.data['handleStart'] handle_end = context.data['handleEnd'] @@ -506,7 +506,7 @@ class AExpectedFiles: expected_files.append( '{}.{}.{}'.format(file_prefix, str(frame).rjust( - layer_data["padding"], "0"), + layer_data["padding"], "0"), layer_data["defaultExt"])) return expected_files @@ -642,7 +642,7 @@ class ExpectedFilesArnold(AExpectedFiles): enabled_aovs = [] try: if not (cmds.getAttr('defaultArnoldRenderOptions.aovMode') - and not cmds.getAttr('defaultArnoldDriver.mergeAOVs')): + and not cmds.getAttr('defaultArnoldDriver.mergeAOVs')): # noqa: W503, E501 # AOVs are merged in mutli-channel file return enabled_aovs except ValueError: @@ -763,10 +763,7 @@ class ExpectedFilesVray(AExpectedFiles): if enabled: # todo: find how vray set format for AOVs enabled_aovs.append( - ( - self._get_vray_aov_name(aov), - default_ext) - ) + (self._get_vray_aov_name(aov), default_ext)) return enabled_aovs def _get_vray_aov_name(self, node): From 80aa8a52b755b36a11769d2a4e81ee7fb3b189e8 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 23 Mar 2020 11:58:11 +0100 Subject: [PATCH 099/118] validate only important things --- .../maya/publish/validate_frame_range.py | 90 ++++++------------- 1 file changed, 27 insertions(+), 63 deletions(-) diff --git a/pype/plugins/maya/publish/validate_frame_range.py b/pype/plugins/maya/publish/validate_frame_range.py index 0be77644a0..c0a43fe4c7 100644 --- a/pype/plugins/maya/publish/validate_frame_range.py +++ b/pype/plugins/maya/publish/validate_frame_range.py @@ -7,8 +7,9 @@ from maya import cmds class ValidateFrameRange(pyblish.api.InstancePlugin): """Valides the frame ranges. - This is optional validator checking if the frame range matches the one of - asset. + This is optional validator checking if the frame range on instance + matches the one of asset. It also validate render frame range of render + layers Repair action will change everything to match asset. @@ -20,7 +21,7 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): families = ["animation", "pointcache", "camera", - "render", + "renderlayer", "review", "yeticache"] optional = True @@ -67,69 +68,32 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): handle_start, frame_start, frame_end, handle_end )) - minTime = int(cmds.playbackOptions(minTime=True, query=True)) - maxTime = int(cmds.playbackOptions(maxTime=True, query=True)) - animStartTime = int(cmds.playbackOptions(animationStartTime=True, - query=True)) - animEndTime = int(cmds.playbackOptions(animationEndTime=True, - query=True)) + if "renderlayer" in self.families: - if int(minTime) != inst_start: - errors.append("Start of Maya timeline is set to frame [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - minTime, - frame_start_handle, - handle_start, frame_start, frame_end, handle_end - )) + render_start = int(cmds.getAttr("defaultRenderGlobals.startFrame")) + render_end = int(cmds.getAttr("defaultRenderGlobals.endFrame")) - if int(maxTime) != inst_end: - errors.append("End of Maya timeline is set to frame [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - maxTime, - frame_end_handle, - handle_start, frame_start, frame_end, handle_end - )) + if int(render_start) != inst_start: + errors.append("Render settings start frame is set to [ {} ] " + "and doesn't match the one set on " + "asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + int(render_start), + frame_start_handle, + handle_start, frame_start, frame_end, + handle_end + )) - if int(animStartTime) != inst_start: - errors.append("Animation start in Maya is set to frame [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - animStartTime, - frame_start_handle, - handle_start, frame_start, frame_end, handle_end - )) - - if int(animEndTime) != inst_end: - errors.append("Animation start in Maya is set to frame [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - animEndTime, - frame_end_handle, - handle_start, frame_start, frame_end, handle_end - )) - - render_start = int(cmds.getAttr("defaultRenderGlobals.startFrame")) - render_end = int(cmds.getAttr("defaultRenderGlobals.endFrame")) - - if int(render_start) != inst_start: - errors.append("Render settings start frame is set to [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - int(render_start), - frame_start_handle, - handle_start, frame_start, frame_end, handle_end - )) - - if int(render_end) != inst_end: - errors.append("Render settings end frame is set to [ {} ] " - " and doesn't match the one set on asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - int(render_end), - frame_end_handle, - handle_start, frame_start, frame_end, handle_end - )) + if int(render_end) != inst_end: + errors.append("Render settings end frame is set to [ {} ] " + "and doesn't match the one set on " + "asset [ {} ]: " + "{}/{}/{}/{} (handle/start/end/handle)".format( + int(render_end), + frame_end_handle, + handle_start, frame_start, frame_end, + handle_end + )) for e in errors: self.log.error(e) From f2eb13e3ca595525cad4b7a9ec7a2b77cb566a9a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 23 Mar 2020 14:57:25 +0100 Subject: [PATCH 100/118] hound cleanups --- .../publish/integrate_master_version.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 16aa0dd23d..3c7838b708 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -17,12 +17,14 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): optional = True - families = ["model", - "rig", - "setdress", - "look", - "pointcache", - "animation"] + families = [ + "model", + "rig", + "setdress", + "look", + "pointcache", + "animation" + ] # Can specify representation names that will be ignored (lower case) ignored_representation_names = [] @@ -109,13 +111,13 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): all_copied_files = [] transfers = instance.data.get("transfers", list()) - for src, dst in transfers: + for dst in transfers.values(): dst = os.path.normpath(dst) if dst not in all_copied_files: all_copied_files.append(dst) hardlinks = instance.data.get("hardlinks", list()) - for src, dst in hardlinks: + for dst in hardlinks.values(): dst = os.path.normpath(dst) if dst not in all_copied_files: all_copied_files.append(dst) @@ -190,7 +192,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): # Separate old representations into `to replace` and `to delete` old_repres_to_replace = {} old_repres_to_delete = {} - for repre_id, repre_info in published_repres.items(): + for repre_info in published_repres.values(): repre = repre_info["representation"] repre_name_low = repre["name"].lower() if repre_name_low in old_repres_by_name: @@ -260,7 +262,7 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): )) try: src_to_dst_file_paths = [] - for repre_id, repre_info in published_repres.items(): + for repre_info in published_repres.values(): # Skip if new repre does not have published repre files published_files = repre_info["published_files"] From 566d4952486a24cd8e3e867490a9647fac57a582 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 23 Mar 2020 18:06:26 +0100 Subject: [PATCH 101/118] simplified frame range validator --- .../maya/publish/validate_frame_range.py | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/pype/plugins/maya/publish/validate_frame_range.py b/pype/plugins/maya/publish/validate_frame_range.py index c0a43fe4c7..0d51a83cf5 100644 --- a/pype/plugins/maya/publish/validate_frame_range.py +++ b/pype/plugins/maya/publish/validate_frame_range.py @@ -68,33 +68,6 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): handle_start, frame_start, frame_end, handle_end )) - if "renderlayer" in self.families: - - render_start = int(cmds.getAttr("defaultRenderGlobals.startFrame")) - render_end = int(cmds.getAttr("defaultRenderGlobals.endFrame")) - - if int(render_start) != inst_start: - errors.append("Render settings start frame is set to [ {} ] " - "and doesn't match the one set on " - "asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - int(render_start), - frame_start_handle, - handle_start, frame_start, frame_end, - handle_end - )) - - if int(render_end) != inst_end: - errors.append("Render settings end frame is set to [ {} ] " - "and doesn't match the one set on " - "asset [ {} ]: " - "{}/{}/{}/{} (handle/start/end/handle)".format( - int(render_end), - frame_end_handle, - handle_start, frame_start, frame_end, - handle_end - )) - for e in errors: self.log.error(e) @@ -103,12 +76,8 @@ class ValidateFrameRange(pyblish.api.InstancePlugin): @classmethod def repair(cls, instance): """ - Repair by calling avalon reset frame range function. This will set - timeline frame range, render settings range and frame information - on instance container to match asset data. + Repair instance container to match asset data. """ - import avalon.maya.interactive - avalon.maya.interactive.reset_frame_range() cmds.setAttr( "{}.frameStart".format(instance.data["name"]), instance.context.data.get("frameStartHandle")) From 4511b915035cdb509562b84d088f7f8834c04c8a Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 23 Mar 2020 18:32:20 +0100 Subject: [PATCH 102/118] set avalon project on publish job by environment variable --- .../global/publish/submit_publish_job.py | 3 ++- pype/scripts/publish_filesequence.py | 18 ------------------ 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 556132cd77..9cfeb0762e 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -222,9 +222,10 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): # Transfer the environment from the original job to this dependent # job so they use the same environment - environment = job["Props"].get("Env", {}) environment["PYPE_METADATA_FILE"] = metadata_path + environment["AVALON_PROJECT"] = api.Session.get("AVALON_PROJECT") + i = 0 for index, key in enumerate(environment): if key.upper() in self.enviro_filter: diff --git a/pype/scripts/publish_filesequence.py b/pype/scripts/publish_filesequence.py index fe795564a5..a41d97668e 100644 --- a/pype/scripts/publish_filesequence.py +++ b/pype/scripts/publish_filesequence.py @@ -25,18 +25,6 @@ log.setLevel(logging.DEBUG) error_format = "Failed {plugin.__name__}: {error} -- {error.traceback}" -def _load_json(path): - assert os.path.isfile(path), ("path to json file doesn't exist") - data = None - with open(path, "r") as json_file: - try: - data = json.load(json_file) - except Exception as exc: - log.error( - "Error loading json: " - "{} - Exception: {}".format(path, exc) - ) - return data def __main__(): parser = argparse.ArgumentParser() @@ -90,12 +78,6 @@ def __main__(): paths = kwargs.paths or [os.environ.get("PYPE_METADATA_FILE")] or [os.getcwd()] # noqa - for path in paths: - data = _load_json(path) - log.info("Setting session using data from file") - os.environ["AVALON_PROJECT"] = data["session"]["AVALON_PROJECT"] - break - args = [ os.path.join(pype_root, pype_command), "publish", From f1241415f991f41f06e440b7ce9a677bfa079530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Samohel?= <33513211+antirotor@users.noreply.github.com> Date: Mon, 23 Mar 2020 18:38:03 +0100 Subject: [PATCH 103/118] fixing flake8 configuration there was duplicated ignore option --- .flake8 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index f28d8cbfc3..b04062ceab 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,6 @@ [flake8] # ignore = D203 -ignore = BLK100 -ignore = W504 +ignore = BLK100, W504 max-line-length = 79 exclude = .git, From 139add3a45d353c211ef39264d03b589227d9d62 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 24 Mar 2020 17:23:36 +0100 Subject: [PATCH 104/118] it is possible to use `source_timecode` in burnins --- pype/scripts/otio_burnin.py | 48 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index 8b52216968..7c94006466 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -36,7 +36,8 @@ TIMECODE = ( MISSING_KEY_VALUE = "N/A" CURRENT_FRAME_KEY = "{current_frame}" CURRENT_FRAME_SPLITTER = "_-_CURRENT_FRAME_-_" -TIME_CODE_KEY = "{timecode}" +TIMECODE_KEY = "{timecode}" +SOURCE_TIMECODE_KEY = "{source_timecode}" def _streams(source): @@ -188,10 +189,13 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): if not options.get("fps"): options["fps"] = self.frame_rate - options["timecode"] = ffmpeg_burnins._frames_to_timecode( - frame_start_tc, - self.frame_rate - ) + if isinstance(frame_start_tc, str): + options["timecode"] = frame_start_tc + else: + options["timecode"] = ffmpeg_burnins._frames_to_timecode( + frame_start_tc, + self.frame_rate + ) self._add_burnin(text, align, options, TIMECODE) @@ -412,7 +416,14 @@ def burnins_from_data( data[CURRENT_FRAME_KEY[1:-1]] = CURRENT_FRAME_SPLITTER if frame_start_tc is not None: - data[TIME_CODE_KEY[1:-1]] = TIME_CODE_KEY + data[TIMECODE_KEY[1:-1]] = TIMECODE_KEY + + source_timecode = stream.get("timecode") + if source_timecode is None: + source_timecode = stream.get("tags", {}).get("timecode") + + if source_timecode is not None: + data[SOURCE_TIMECODE_KEY[1:-1]] = SOURCE_TIMECODE_KEY for align_text, value in presets.get('burnins', {}).items(): if not value: @@ -425,8 +436,6 @@ def burnins_from_data( " (Make sure you have new burnin presets)." ).format(str(type(value)), str(value))) - has_timecode = TIME_CODE_KEY in value - align = None align_text = align_text.strip().lower() if align_text == "top_left": @@ -442,6 +451,7 @@ def burnins_from_data( elif align_text == "bottom_right": align = ModifiedBurnins.BOTTOM_RIGHT + has_timecode = TIMECODE_KEY in value # Replace with missing key value if frame_start_tc is not set if frame_start_tc is None and has_timecode: has_timecode = False @@ -449,7 +459,13 @@ def burnins_from_data( "`frame_start` and `frame_start_tc`" " are not set in entered data." ) - value = value.replace(TIME_CODE_KEY, MISSING_KEY_VALUE) + value = value.replace(TIMECODE_KEY, MISSING_KEY_VALUE) + + has_source_timecode = SOURCE_TIMECODE_KEY in value + if source_timecode is None and has_source_timecode: + has_source_timecode = False + log.warning("Source does not have set timecode value.") + value = value.replace(SOURCE_TIMECODE_KEY, MISSING_KEY_VALUE) key_pattern = re.compile(r"(\{.*?[^{0]*\})") @@ -465,10 +481,20 @@ def burnins_from_data( value = value.replace(key, MISSING_KEY_VALUE) # Handle timecode differently + if has_source_timecode: + args = [align, frame_start, frame_end, source_timecode] + if not value.startswith(SOURCE_TIMECODE_KEY): + value_items = value.split(SOURCE_TIMECODE_KEY) + text = value_items[0].format(**data) + args.append(text) + + burnin.add_timecode(*args) + continue + if has_timecode: args = [align, frame_start, frame_end, frame_start_tc] - if not value.startswith(TIME_CODE_KEY): - value_items = value.split(TIME_CODE_KEY) + if not value.startswith(TIMECODE_KEY): + value_items = value.split(TIMECODE_KEY) text = value_items[0].format(**data) args.append(text) From 570d336f05ad5fb3b2fb8249e06bf1e3cee6ec1b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Mar 2020 15:29:50 +0100 Subject: [PATCH 105/118] fix invalid iterations in integrate master version --- pype/plugins/global/publish/integrate_master_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index 3c7838b708..af6e7707e4 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -111,13 +111,13 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): all_copied_files = [] transfers = instance.data.get("transfers", list()) - for dst in transfers.values(): + for _src, dst in transfers: dst = os.path.normpath(dst) if dst not in all_copied_files: all_copied_files.append(dst) hardlinks = instance.data.get("hardlinks", list()) - for dst in hardlinks.values(): + for _src, dst in hardlinks: dst = os.path.normpath(dst) if dst not in all_copied_files: all_copied_files.append(dst) From c15437f04a0bf215871a2df205ba61e52227521e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Mar 2020 16:32:06 +0100 Subject: [PATCH 106/118] template_data stores frame used for anatomy filling --- pype/plugins/global/publish/integrate_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index ae946f0696..32504a64b3 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -326,6 +326,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): test_dest_files.append( os.path.normpath(template_filled) ) + # Store used frame value to template data + template_data["frame"] = repre_context["frame"] self.log.debug( "test_dest_files: {}".format(str(test_dest_files))) From ac47874d3c7a1f9c6a527236fd29a6344c48e3de Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Mar 2020 16:32:34 +0100 Subject: [PATCH 107/118] template_name could not be overriden during representation loop --- pype/plugins/global/publish/integrate_new.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 32504a64b3..d4e4ae0b87 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -89,6 +89,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "project", "asset", "task", "subset", "version", "representation", "family", "hierarchy", "task", "username" ] + default_template_name = "publish" def process(self, instance): @@ -261,7 +262,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): # Each should be a single representation (as such, a single extension) representations = [] destination_list = [] - template_name = 'publish' + if 'transfers' not in instance.data: instance.data['transfers'] = [] @@ -288,8 +289,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): files = repre['files'] if repre.get('stagingDir'): stagingdir = repre['stagingDir'] - if repre.get('anatomy_template'): - template_name = repre['anatomy_template'] + + template_name = ( + repre.get('anatomy_template') or self.default_template_name + ) if repre.get("outputName"): template_data["output"] = repre['outputName'] From 0cb481706ca0d88e0e5fbb2aa7938b904440707d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Wed, 25 Mar 2020 16:37:55 +0100 Subject: [PATCH 108/118] Now is used corrent frame --- pype/plugins/global/publish/integrate_new.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index d4e4ae0b87..768970ccdc 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -329,8 +329,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): test_dest_files.append( os.path.normpath(template_filled) ) - # Store used frame value to template data - template_data["frame"] = repre_context["frame"] self.log.debug( "test_dest_files: {}".format(str(test_dest_files))) @@ -387,6 +385,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if not dst_start_frame: dst_start_frame = dst_padding + # Store used frame value to template data + template_data["frame"] = dst_start_frame dst = "{0}{1}{2}".format( dst_head, dst_start_frame, From 9577c236243d7caa42a4bdd1409e469d2f1cd0d2 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 18:07:12 +0100 Subject: [PATCH 109/118] fix(global): slate was on even if render on farm --- pype/plugins/global/publish/extract_review_slate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 8c33a0d853..da94c7714a 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -11,7 +11,9 @@ class ExtractReviewSlate(pype.api.Extractor): label = "Review with Slate frame" order = pyblish.api.ExtractorOrder + 0.031 - families = ["slate"] + families = ["slate", "review"] + match = pyblish.api.Subset + hosts = ["nuke", "maya", "shell"] optional = True From 570269f0247331460ad66baee0a76728bba0242b Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 18:56:17 +0100 Subject: [PATCH 110/118] fix(global, nuke): setting families filter --- .../publish/integrate_ftrack_instances.py | 1 + pype/plugins/global/publish/integrate_new.py | 2 -- pype/plugins/nuke/load/load_mov.py | 1 + pype/plugins/nuke/load/load_sequence.py | 4 ++-- .../nuke/publish/increment_script_version.py | 21 +++++-------------- .../nuke/publish/validate_rendered_frames.py | 2 +- 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py index db257e901a..59fb507788 100644 --- a/pype/plugins/ftrack/publish/integrate_ftrack_instances.py +++ b/pype/plugins/ftrack/publish/integrate_ftrack_instances.py @@ -22,6 +22,7 @@ class IntegrateFtrackInstance(pyblish.api.InstancePlugin): 'setdress': 'setdress', 'pointcache': 'cache', 'render': 'render', + 'render2d': 'render', 'nukescript': 'comp', 'write': 'render', 'review': 'mov', diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 127bff90f0..0ceac1f4a7 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -64,9 +64,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): "scene", "vrayproxy", "render", - "render.local", "prerender", - "prerender.local", "imagesequence", "review", "rendersetup", diff --git a/pype/plugins/nuke/load/load_mov.py b/pype/plugins/nuke/load/load_mov.py index 88e65156cb..5d15efcd3a 100644 --- a/pype/plugins/nuke/load/load_mov.py +++ b/pype/plugins/nuke/load/load_mov.py @@ -92,6 +92,7 @@ class LoadMov(api.Loader): "source", "plate", "render", + "prerender", "review"] + presets["families"] representations = [ diff --git a/pype/plugins/nuke/load/load_sequence.py b/pype/plugins/nuke/load/load_sequence.py index 690f074c3f..083cc86474 100644 --- a/pype/plugins/nuke/load/load_sequence.py +++ b/pype/plugins/nuke/load/load_sequence.py @@ -70,7 +70,7 @@ def loader_shift(node, frame, relative=True): class LoadSequence(api.Loader): """Load image sequence into Nuke""" - families = ["render2d", "source", "plate", "render"] + families = ["render2d", "source", "plate", "render", "prerender"] representations = ["exr", "dpx", "jpg", "jpeg", "png"] label = "Load sequence" @@ -87,7 +87,7 @@ class LoadSequence(api.Loader): version = context['version'] version_data = version.get("data", {}) repr_id = context["representation"]["_id"] - + self.log.info("version_data: {}\n".format(version_data)) self.log.debug( "Representation id `{}` ".format(repr_id)) diff --git a/pype/plugins/nuke/publish/increment_script_version.py b/pype/plugins/nuke/publish/increment_script_version.py index 6e3ce08276..c76083eb1e 100644 --- a/pype/plugins/nuke/publish/increment_script_version.py +++ b/pype/plugins/nuke/publish/increment_script_version.py @@ -9,6 +9,7 @@ class IncrementScriptVersion(pyblish.api.ContextPlugin): order = pyblish.api.IntegratorOrder + 0.9 label = "Increment Script Version" optional = True + families = ["workfile", "render", "render.local", "render.farm"] hosts = ['nuke'] def process(self, context): @@ -16,19 +17,7 @@ class IncrementScriptVersion(pyblish.api.ContextPlugin): assert all(result["success"] for result in context.data["results"]), ( "Publishing not succesfull so version is not increased.") - instances = context[:] - - prerender_check = list() - families_check = list() - for instance in instances: - if ("prerender" in str(instance)) and instance.data.get("families", None): - prerender_check.append(instance) - if instance.data.get("families", None): - families_check.append(True) - - - if len(prerender_check) != len(families_check): - from pype.lib import version_up - path = context.data["currentFile"] - nuke.scriptSaveAs(version_up(path)) - self.log.info('Incrementing script version') + from pype.lib import version_up + path = context.data["currentFile"] + nuke.scriptSaveAs(version_up(path)) + self.log.info('Incrementing script version') diff --git a/pype/plugins/nuke/publish/validate_rendered_frames.py b/pype/plugins/nuke/publish/validate_rendered_frames.py index 6e9b91dd72..425789f18a 100644 --- a/pype/plugins/nuke/publish/validate_rendered_frames.py +++ b/pype/plugins/nuke/publish/validate_rendered_frames.py @@ -28,7 +28,7 @@ class ValidateRenderedFrames(pyblish.api.InstancePlugin): """ Validates file output. """ order = pyblish.api.ValidatorOrder + 0.1 - families = ["render"] + families = ["render", "prerender"] label = "Validate rendered frame" hosts = ["nuke", "nukestudio"] From f25dea58a808e80876acf5115c41a7fd2c2b3272 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 18:57:20 +0100 Subject: [PATCH 111/118] fix(nuke, global): shuffling families after plugins finish processing --- pype/plugins/nuke/publish/collect_writes.py | 24 ++++++++++++++----- .../nuke/publish/extract_render_local.py | 18 ++++++++++---- .../nuke/publish/submit_nuke_deadline.py | 10 ++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/pype/plugins/nuke/publish/collect_writes.py b/pype/plugins/nuke/publish/collect_writes.py index f3f33b7a6d..6379a1db87 100644 --- a/pype/plugins/nuke/publish/collect_writes.py +++ b/pype/plugins/nuke/publish/collect_writes.py @@ -12,9 +12,11 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): hosts = ["nuke", "nukeassist"] families = ["write"] + # preset attributes + sync_workfile_version = True + def process(self, instance): - # adding 2d focused rendering - instance.data["families"].append("render2d") + families = instance.data["families"] node = None for x in instance: @@ -52,10 +54,13 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): output_dir = os.path.dirname(path) self.log.debug('output dir: {}'.format(output_dir)) - # get version to instance for integration - instance.data['version'] = instance.context.data["version"] + if not next((f for f in families + if "prerender" in f), + None) and self.sync_workfile_version: + # get version to instance for integration + instance.data['version'] = instance.context.data["version"] - self.log.debug('Write Version: %s' % instance.data('version')) + self.log.debug('Write Version: %s' % instance.data('version')) # create label name = node.name() @@ -66,7 +71,7 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): int(last_frame) ) - if [fm for fm in instance.data['families'] + if [fm for fm in families if fm in ["render", "prerender"]]: if "representations" not in instance.data: instance.data["representations"] = list() @@ -145,6 +150,13 @@ class CollectNukeWrites(pyblish.api.InstancePlugin): "deadlineChunkSize": deadlineChunkSize, "deadlinePriority": deadlinePriority }) + + if "prerender" in families: + instance.data.update({ + "family": "prerender", + "families": [] + }) + self.log.debug("families: {}".format(families)) self.log.debug("instance.data: {}".format(instance.data)) diff --git a/pype/plugins/nuke/publish/extract_render_local.py b/pype/plugins/nuke/publish/extract_render_local.py index 1dad413ee5..b7aa59a457 100644 --- a/pype/plugins/nuke/publish/extract_render_local.py +++ b/pype/plugins/nuke/publish/extract_render_local.py @@ -20,6 +20,8 @@ class NukeRenderLocal(pype.api.Extractor): families = ["render.local", "prerender.local"] def process(self, instance): + families = instance.data["families"] + node = None for x in instance: if x.Class() == "Write": @@ -30,7 +32,7 @@ class NukeRenderLocal(pype.api.Extractor): first_frame = instance.data.get("frameStartHandle", None) # exception for slate workflow - if "slate" in instance.data["families"]: + if "slate" in families: first_frame -= 1 last_frame = instance.data.get("frameEndHandle", None) @@ -53,7 +55,7 @@ class NukeRenderLocal(pype.api.Extractor): ) # exception for slate workflow - if "slate" in instance.data["families"]: + if "slate" in families: first_frame += 1 path = node['file'].value() @@ -79,8 +81,16 @@ class NukeRenderLocal(pype.api.Extractor): out_dir )) - instance.data['family'] = 'render' - instance.data['families'].append('render') + # redefinition of families + if "render.local" in families: + instance.data['family'] = 'render2d' + families.remove('render.local') + families.insert(0, "render") + elif "prerender.local" in families: + instance.data['family'] = 'prerender' + families.remove('prerender.local') + families.insert(0, "prerender") + instance.data["families"] = families collections, remainder = clique.assemble(collected_frames) self.log.info('collections: {}'.format(str(collections))) diff --git a/pype/plugins/nuke/publish/submit_nuke_deadline.py b/pype/plugins/nuke/publish/submit_nuke_deadline.py index 3da2e58e4d..7990c20112 100644 --- a/pype/plugins/nuke/publish/submit_nuke_deadline.py +++ b/pype/plugins/nuke/publish/submit_nuke_deadline.py @@ -28,6 +28,7 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): deadline_chunk_size = 1 def process(self, instance): + families = instance.data["families"] node = instance[0] context = instance.context @@ -82,6 +83,15 @@ class NukeSubmitDeadline(pyblish.api.InstancePlugin): instance.data["deadlineSubmissionJob"] = resp.json() instance.data["publishJobState"] = "Suspended" + # redefinition of families + if "render.farm" in families: + instance.data['family'] = 'write' + families.insert(0, "render2d") + elif "prerender.farm" in families: + instance.data['family'] = 'write' + families.insert(0, "prerender") + instance.data["families"] = families + def payload_submit(self, instance, script_path, From 93761e3010743d139f272022a578af3d90e4acfc Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 18:57:52 +0100 Subject: [PATCH 112/118] fix(global): add correct frame number to repre.data --- pype/plugins/global/publish/integrate_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 0ceac1f4a7..44f91a343c 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -383,6 +383,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if not dst_start_frame: dst_start_frame = dst_padding + template_data["frame"] = dst_start_frame + dst = "{0}{1}{2}".format( dst_head, dst_start_frame, From 5f616f14f1a97db551b713a1ab73415d9ab6ecd4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 18:58:18 +0100 Subject: [PATCH 113/118] fix(nuke): submitting with correct family --- .../global/publish/submit_publish_job.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 144a0822a2..134f8e9098 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -141,7 +141,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): hosts = ["fusion", "maya", "nuke"] - families = ["render.farm", "prerender.farm", "renderlayer", "imagesequence"] + families = ["render.farm", "prerener", "renderlayer", "imagesequence"] aov_filter = {"maya": ["beauty"]} @@ -168,9 +168,9 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): instance_transfer = { "slate": ["slateFrame"], "review": ["lutPath"], - "render.farm": ["bakeScriptPath", "bakeRenderPath", - "bakeWriteNodeName", "version"] - } + "render2d": ["bakeScriptPath", "bakeRenderPath", + "bakeWriteNodeName", "version"] + } # list of family names to transfer to new family if present families_transfer = ["render3d", "render2d", "ftrack", "slate"] @@ -586,17 +586,24 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "resolutionHeight": data.get("resolutionHeight", 1080), } - if "prerender.farm" in instance.data["families"]: + if "prerender" in instance.data["families"]: instance_skeleton_data.update({ "family": "prerender", - "families": ["prerender"] - }) + "families": [] + }) # transfer specific families from original instance to new render for item in self.families_transfer: if item in instance.data.get("families", []): instance_skeleton_data["families"] += [item] + if "render.farm" in instance.data["families"]: + instance_skeleton_data.update({ + "family": "render2d", + "families": ["render"] + [f for f in instance.data["families"] + if "render.farm" not in f] + }) + # transfer specific properties from original instance based on # mapping dictionary `instance_transfer` for key, values in self.instance_transfer.items(): From 129eb5db6931832c8a3592295e352dbdd4700ea9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 19:04:03 +0100 Subject: [PATCH 114/118] fix(nuke): hound pep8 --- pype/plugins/nuke/create/create_write_prerender.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pype/plugins/nuke/create/create_write_prerender.py b/pype/plugins/nuke/create/create_write_prerender.py index 6e242f886c..210c84e0cd 100644 --- a/pype/plugins/nuke/create/create_write_prerender.py +++ b/pype/plugins/nuke/create/create_write_prerender.py @@ -1,8 +1,7 @@ from collections import OrderedDict from pype.nuke import ( plugin, - lib as pnlib - ) + lib as pnlib) import nuke @@ -81,15 +80,15 @@ class CreateWritePrerender(plugin.PypeCreator): else: self.log.info("Adding template path from plugin") write_data.update({ - "fpath_template": "{work}/prerenders/nuke/{subset}/{subset}.{frame}.{ext}"}) + "fpath_template": ("{work}/prerenders/nuke/{subset}" + "/{subset}.{frame}.{ext}")}) write_node = pnlib.create_write_node( self.data["subset"], write_data, input=selected_node, prenodes=[], - review=False - ) + review=False) # relinking to collected connections for i, input in enumerate(inputs): From 8d9d965d6d4f8b6fa8a217e45415f3eee611f866 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 19:06:19 +0100 Subject: [PATCH 115/118] fix(nuke): hound pep8 --- pype/plugins/nuke/create/create_write_render.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/plugins/nuke/create/create_write_render.py b/pype/plugins/nuke/create/create_write_render.py index c3b60ba2b0..06ef237305 100644 --- a/pype/plugins/nuke/create/create_write_render.py +++ b/pype/plugins/nuke/create/create_write_render.py @@ -1,8 +1,7 @@ from collections import OrderedDict from pype.nuke import ( plugin, - lib as pnlib - ) + lib as pnlib) import nuke @@ -82,7 +81,8 @@ class CreateWriteRender(plugin.PypeCreator): else: self.log.info("Adding template path from plugin") write_data.update({ - "fpath_template": "{work}/renders/nuke/{subset}/{subset}.{frame}.{ext}"}) + "fpath_template": ("{work}/renders/nuke/{subset}" + "/{subset}.{frame}.{ext}")}) write_node = pnlib.create_write_node( self.data["subset"], From 48d7489621f36fe6743207747f093df43f8512f9 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Wed, 25 Mar 2020 19:12:34 +0100 Subject: [PATCH 116/118] fix(global): hound pep8 --- .../global/publish/extract_review_slate.py | 15 +++++++++------ pype/plugins/global/publish/submit_publish_job.py | 5 ++--- .../nuke/publish/extract_review_data_mov.py | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index da94c7714a..aaa67bde68 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -36,7 +36,8 @@ class ExtractReviewSlate(pype.api.Extractor): fps = inst_data.get("fps") # defining image ratios - resolution_ratio = (float(resolution_width) * pixel_aspect) / resolution_height + resolution_ratio = ((float(resolution_width) * pixel_aspect) / + resolution_height) delivery_ratio = float(to_width) / float(to_height) self.log.debug("__ resolution_ratio: `{}`".format(resolution_ratio)) self.log.debug("__ delivery_ratio: `{}`".format(delivery_ratio)) @@ -91,7 +92,7 @@ class ExtractReviewSlate(pype.api.Extractor): input_args.extend([ "-r {}".format(fps), "-t 0.04"] - ) + ) # output args codec_args = repre["_profile"].get('codec', []) @@ -113,7 +114,7 @@ class ExtractReviewSlate(pype.api.Extractor): self.log.debug("lower then delivery") width_scale = int(to_width * scale_factor) width_half_pad = int(( - to_width - width_scale)/2) + to_width - width_scale) / 2) height_scale = to_height height_half_pad = 0 else: @@ -126,7 +127,7 @@ class ExtractReviewSlate(pype.api.Extractor): height_scale = int( resolution_height * scale_factor) height_half_pad = int( - (to_height - height_scale)/2) + (to_height - height_scale) / 2) self.log.debug( "__ width_scale: `{}`".format(width_scale)) @@ -137,8 +138,10 @@ class ExtractReviewSlate(pype.api.Extractor): self.log.debug( "__ height_half_pad: `{}`".format(height_half_pad)) - scaling_arg = "scale={0}x{1}:flags=lanczos,pad={2}:{3}:{4}:{5}:black,setsar=1".format( - width_scale, height_scale, to_width, to_height, width_half_pad, height_half_pad + scaling_arg = ("scale={0}x{1}:flags=lanczos," + "pad={2}:{3}:{4}:{5}:black,setsar=1").format( + width_scale, height_scale, to_width, to_height, + width_half_pad, height_half_pad ) vf_back = self.add_video_filter_args( diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 134f8e9098..af16b11db6 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -170,7 +170,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): "review": ["lutPath"], "render2d": ["bakeScriptPath", "bakeRenderPath", "bakeWriteNodeName", "version"] - } + } # list of family names to transfer to new family if present families_transfer = ["render3d", "render2d", "ftrack", "slate"] @@ -589,8 +589,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): if "prerender" in instance.data["families"]: instance_skeleton_data.update({ "family": "prerender", - "families": [] - }) + "families": []}) # transfer specific families from original instance to new render for item in self.families_transfer: diff --git a/pype/plugins/nuke/publish/extract_review_data_mov.py b/pype/plugins/nuke/publish/extract_review_data_mov.py index 0bd5394548..7c56dc8b92 100644 --- a/pype/plugins/nuke/publish/extract_review_data_mov.py +++ b/pype/plugins/nuke/publish/extract_review_data_mov.py @@ -3,7 +3,7 @@ import pyblish.api from avalon.nuke import lib as anlib from pype.nuke import lib as pnlib import pype -reload(pnlib) + class ExtractReviewDataMov(pype.api.Extractor): """Extracts movie and thumbnail with baked in luts From f152b9b6a7c691d962344519be36cf1d12b84b6f Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Thu, 26 Mar 2020 16:19:08 +0100 Subject: [PATCH 117/118] do not allow token in Redshift --- .../maya/publish/validate_rendersettings.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pype/plugins/maya/publish/validate_rendersettings.py b/pype/plugins/maya/publish/validate_rendersettings.py index c98f0f8cdc..67239d4790 100644 --- a/pype/plugins/maya/publish/validate_rendersettings.py +++ b/pype/plugins/maya/publish/validate_rendersettings.py @@ -13,13 +13,17 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin): """Validates the global render settings * File Name Prefix must start with: `maya/` - all other token are customizable but sane values are: + all other token are customizable but sane values for Arnold are: `maya///_` - token is supported also, usefull for multiple renderable + token is supported also, useful for multiple renderable cameras per render layer. + For Redshift omit token. Redshift will append it + automatically if AOVs are enabled and if you user Multipart EXR + it doesn't make much sense. + * Frame Padding must be: * default: 4 @@ -127,8 +131,13 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin): # no vray checks implemented yet pass elif renderer == "redshift": - # no redshift check implemented yet - pass + if re.search(cls.R_AOV_TOKEN, prefix): + invalid = True + cls.log.error("Do not use AOV token [ {} ] - " + "Redshift automatically append AOV name and " + "it doesn't make much sense with " + "Multipart EXR".format(prefix)) + elif renderer == "renderman": file_prefix = cmds.getAttr("rmanGlobals.imageFileFormat") dir_prefix = cmds.getAttr("rmanGlobals.imageOutputDir") @@ -143,8 +152,8 @@ class ValidateRenderSettings(pyblish.api.InstancePlugin): dir_prefix)) else: - multichannel = cmds.getAttr("defaultArnoldDriver.mergeAOVs") - if multichannel: + multipart = cmds.getAttr("defaultArnoldDriver.mergeAOVs") + if multipart: if re.search(cls.R_AOV_TOKEN, prefix): invalid = True cls.log.error("Wrong image prefix [ {} ] - " From 41e6eb05caf8bf053bc2d8183a484be278436f20 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 26 Mar 2020 17:06:00 +0100 Subject: [PATCH 118/118] fix(nuke): build first workfile was not accepting jpeg sequences --- pype/nuke/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/nuke/lib.py b/pype/nuke/lib.py index ad2d576da3..3ff9c6d397 100644 --- a/pype/nuke/lib.py +++ b/pype/nuke/lib.py @@ -1135,7 +1135,7 @@ class BuildWorkfile(WorkfileSettings): regex_filter=None, version=None, representations=["exr", "dpx", "lutJson", "mov", - "preview", "png"]): + "preview", "png", "jpeg", "jpg"]): """ A short description.