diff --git a/docs/source/pype.aport.rst b/docs/source/pype.aport.rst deleted file mode 100644 index 4a96b1e619..0000000000 --- a/docs/source/pype.aport.rst +++ /dev/null @@ -1,20 +0,0 @@ -pype.aport package -================== - -.. automodule:: pype.aport - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -pype.aport.api module ---------------------- - -.. automodule:: pype.aport.api - :members: - :undoc-members: - :show-inheritance: - - diff --git a/docs/source/pype.rst b/docs/source/pype.rst index 7409ee62ee..a480cc3180 100644 --- a/docs/source/pype.rst +++ b/docs/source/pype.rst @@ -11,7 +11,6 @@ Subpackages .. toctree:: - pype.aport pype.avalon_apps pype.clockify pype.ftrack diff --git a/pype/api.py b/pype/api.py index c1bf84b4ef..b88be4cc88 100644 --- a/pype/api.py +++ b/pype/api.py @@ -39,13 +39,9 @@ from .action import ( from .lib import ( version_up, get_asset, - get_project, get_hierarchy, - get_subsets, get_version_from_path, get_last_version_from_path, - modified_environ, - add_tool_to_environment, source_hash, get_latest_version ) @@ -88,14 +84,10 @@ __all__ = [ # get contextual data "version_up", - "get_project", "get_hierarchy", "get_asset", - "get_subsets", "get_version_from_path", "get_last_version_from_path", - "modified_environ", - "add_tool_to_environment", "source_hash", "subprocess", diff --git a/pype/hosts/fusion/lib.py b/pype/hosts/fusion/lib.py index f2846c966a..77866fde9d 100644 --- a/pype/hosts/fusion/lib.py +++ b/pype/hosts/fusion/lib.py @@ -2,7 +2,7 @@ import sys from avalon.vendor.Qt import QtGui import avalon.fusion - +from avalon import io self = sys.modules[__name__] self._project = None @@ -59,3 +59,84 @@ def get_additional_data(container): return {"color": QtGui.QColor.fromRgbF(tile_color["R"], tile_color["G"], tile_color["B"])} + + +def switch_item(container, + asset_name=None, + subset_name=None, + representation_name=None): + """Switch container asset, subset or representation of a container by name. + + It'll always switch to the latest version - of course a different + approach could be implemented. + + Args: + container (dict): data of the item to switch with + asset_name (str): name of the asset + subset_name (str): name of the subset + representation_name (str): name of the representation + + Returns: + dict + + """ + + if all(not x for x in [asset_name, subset_name, representation_name]): + raise ValueError("Must have at least one change provided to switch.") + + # Collect any of current asset, subset and representation if not provided + # so we can use the original name from those. + if any(not x for x in [asset_name, subset_name, representation_name]): + _id = io.ObjectId(container["representation"]) + representation = io.find_one({"type": "representation", "_id": _id}) + version, subset, asset, project = io.parenthood(representation) + + if asset_name is None: + asset_name = asset["name"] + + if subset_name is None: + subset_name = subset["name"] + + if representation_name is None: + representation_name = representation["name"] + + # Find the new one + asset = io.find_one({ + "name": asset_name, + "type": "asset" + }) + assert asset, ("Could not find asset in the database with the name " + "'%s'" % asset_name) + + subset = io.find_one({ + "name": subset_name, + "type": "subset", + "parent": asset["_id"] + }) + assert subset, ("Could not find subset in the database with the name " + "'%s'" % subset_name) + + version = io.find_one( + { + "type": "version", + "parent": subset["_id"] + }, + sort=[('name', -1)] + ) + + assert version, "Could not find a version for {}.{}".format( + asset_name, subset_name + ) + + representation = io.find_one({ + "name": representation_name, + "type": "representation", + "parent": version["_id"]} + ) + + assert representation, ("Could not find representation in the database " + "with the name '%s'" % representation_name) + + avalon.api.switch(container, representation) + + return representation diff --git a/pype/hosts/fusion/scripts/fusion_switch_shot.py b/pype/hosts/fusion/scripts/fusion_switch_shot.py index a3f2116db8..ed657cb612 100644 --- a/pype/hosts/fusion/scripts/fusion_switch_shot.py +++ b/pype/hosts/fusion/scripts/fusion_switch_shot.py @@ -234,7 +234,7 @@ def switch(asset_name, filepath=None, new=True): representations = [] for container in containers: try: - representation = pype.switch_item( + representation = fusion_lib.switch_item( container, asset_name=asset_name) representations.append(representation) diff --git a/pype/hosts/hiero/lib.py b/pype/hosts/hiero/lib.py index db7199a190..a508343bfa 100644 --- a/pype/hosts/hiero/lib.py +++ b/pype/hosts/hiero/lib.py @@ -4,6 +4,7 @@ import sys import hiero import pyblish.api import avalon.api as avalon +import avalon.io from avalon.vendor.Qt import (QtWidgets, QtGui) import pype.api as pype from pype.api import Logger, Anatomy @@ -58,7 +59,8 @@ def sync_avalon_data_to_workfile(): project.setProjectRoot(active_project_root) # get project data from avalon db - project_data = pype.get_project()["data"] + project_doc = avalon.io.find_one({"type": "project"}) + project_data = project_doc["data"] log.debug("project_data: {}".format(project_data)) diff --git a/pype/hosts/maya/expected_files.py b/pype/hosts/maya/expected_files.py index dab4c18c2b..8d225bc13d 100644 --- a/pype/hosts/maya/expected_files.py +++ b/pype/hosts/maya/expected_files.py @@ -378,14 +378,8 @@ class AExpectedFiles: renderable = False if self.maya_is_true(cmds.getAttr("{}.renderable".format(cam))): renderable = True - - for override in self.get_layer_overrides( - "{}.renderable".format(cam), self.layer - ): - renderable = self.maya_is_true(override) - - if renderable: renderable_cameras.append(cam) + return renderable_cameras def maya_is_true(self, attr_val): diff --git a/pype/hosts/maya/lib.py b/pype/hosts/maya/lib.py index 2dda198d45..37e4ccf915 100644 --- a/pype/hosts/maya/lib.py +++ b/pype/hosts/maya/lib.py @@ -8,6 +8,7 @@ import math import bson import json import logging +import itertools import contextlib from collections import OrderedDict, defaultdict from math import ceil @@ -122,6 +123,12 @@ def float_round(num, places=0, direction=ceil): return direction(num * (10**places)) / float(10**places) +def pairwise(iterable): + """s -> (s0,s1), (s2,s3), (s4, s5), ...""" + a = iter(iterable) + return itertools.izip(a, a) + + def unique(name): assert isinstance(name, string_types), "`name` must be string" @@ -419,12 +426,12 @@ def empty_sets(sets, force=False): plugs=True, connections=True) or [] original_connections.extend(connections) - for dest, src in lib.pairwise(connections): + for dest, src in pairwise(connections): cmds.disconnectAttr(src, dest) yield finally: - for dest, src in lib.pairwise(original_connections): + for dest, src in pairwise(original_connections): cmds.connectAttr(src, dest) # Restore original members @@ -1857,8 +1864,8 @@ def set_context_settings(): """ # Todo (Wijnand): apply renderer and resolution of project - - project_data = lib.get_project()["data"] + project_doc = io.find_one({"type": "project"}) + project_data = project_doc["data"] asset_data = lib.get_asset()["data"] # Set project fps diff --git a/pype/hosts/nuke/lib.py b/pype/hosts/nuke/lib.py index b4795aec13..50b9697d8e 100644 --- a/pype/hosts/nuke/lib.py +++ b/pype/hosts/nuke/lib.py @@ -195,7 +195,7 @@ def format_anatomy(data): if not version: file = script_name() data["version"] = pype.get_version_from_path(file) - project_document = pype.get_project() + project_document = io.find_one({"type": "project"}) data.update({ "subset": data["avalon"]["subset"], "asset": data["avalon"]["asset"], diff --git a/pype/hosts/tvpaint/__init__.py b/pype/hosts/tvpaint/__init__.py index cb02c87151..7027f0fb55 100644 --- a/pype/hosts/tvpaint/__init__.py +++ b/pype/hosts/tvpaint/__init__.py @@ -2,6 +2,7 @@ import os import logging from avalon.tvpaint.communication_server import register_localization_file +from avalon.tvpaint import pipeline import avalon.api import pyblish.api from pype import PLUGINS_DIR @@ -13,6 +14,23 @@ LOAD_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "load") CREATE_PATH = os.path.join(PLUGINS_DIR, "tvpaint", "create") +def on_instance_toggle(instance, old_value, new_value): + instance_id = instance.data["uuid"] + found_idx = None + current_instances = pipeline.list_instances() + for idx, workfile_instance in enumerate(current_instances): + if workfile_instance["uuid"] == instance_id: + found_idx = idx + break + + if found_idx is None: + return + + if "active" in current_instances[found_idx]: + current_instances[found_idx]["active"] = new_value + pipeline._write_instances(current_instances) + + def install(): log.info("Pype - Installing TVPaint integration") current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -23,6 +41,12 @@ def install(): avalon.api.register_plugin_path(avalon.api.Loader, LOAD_PATH) avalon.api.register_plugin_path(avalon.api.Creator, CREATE_PATH) + registered_callbacks = ( + pyblish.api.registered_callbacks().get("instanceToggled") or [] + ) + if on_instance_toggle not in registered_callbacks: + pyblish.api.register_callback("instanceToggled", on_instance_toggle) + def uninstall(): log.info("Pype - Uninstalling TVPaint integration") diff --git a/pype/lib.py b/pype/lib.py deleted file mode 100644 index cf2457e00d..0000000000 --- a/pype/lib.py +++ /dev/null @@ -1,1915 +0,0 @@ -import os -import sys -import types -import re -import uuid -import json -import collections -import logging -import itertools -import copy -import contextlib -import subprocess -import getpass -import inspect -import acre -import platform -from abc import ABCMeta, abstractmethod - -from avalon import io, pipeline -import six -import avalon.api -from .api import config, Anatomy, Logger - -log = logging.getLogger(__name__) - - -def get_paths_from_environ(env_key, return_first=False): - """Return existing paths from specific envirnment variable. - - :param env_key: Environment key where should look for paths. - :type env_key: str - :param return_first: Return first path on `True`, list of all on `False`. - :type return_first: boolean - - Difference when none of paths exists: - - when `return_first` is set to `False` then function returns empty list. - - when `return_first` is set to `True` then function returns `None`. - """ - - existing_paths = [] - paths = os.environ.get(env_key) or "" - path_items = paths.split(os.pathsep) - for path in path_items: - # Skip empty string - if not path: - continue - # Normalize path - path = os.path.normpath(path) - # Check if path exists - if os.path.exists(path): - # Return path if `return_first` is set to True - if return_first: - return path - # Store path - existing_paths.append(path) - - # Return None if none of paths exists - if return_first: - return None - # Return all existing paths from environment variable - return existing_paths - - -def get_ffmpeg_tool_path(tool="ffmpeg"): - """Find path to ffmpeg tool in FFMPEG_PATH paths. - - Function looks for tool in paths set in FFMPEG_PATH environment. If tool - exists then returns it's full path. - - Returns tool name itself when tool path was not found. (FFmpeg path may be - set in PATH environment variable) - """ - - dir_paths = get_paths_from_environ("FFMPEG_PATH") - for dir_path in dir_paths: - for file_name in os.listdir(dir_path): - base, ext = os.path.splitext(file_name) - if base.lower() == tool.lower(): - return os.path.join(dir_path, tool) - return tool - - -# Special naming case for subprocess since its a built-in method. -def _subprocess(*args, **kwargs): - """Convenience method for getting output errors for subprocess. - - Entered arguments and keyword arguments are passed to subprocess Popen. - - Args: - *args: Variable length arument list passed to Popen. - **kwargs : Arbitary keyword arguments passed to Popen. Is possible to - pass `logging.Logger` object under "logger" if want to use - different than lib's logger. - - Returns: - str: Full output of subprocess concatenated stdout and stderr. - - Raises: - RuntimeError: Exception is raised if process finished with nonzero - return code. - """ - - # Get environents from kwarg or use current process environments if were - # not passed. - env = kwargs.get("env") or os.environ - # Make sure environment contains only strings - filtered_env = {k: str(v) for k, v in env.items()} - - # Use lib's logger if was not passed with kwargs. - logger = kwargs.pop("logger", log) - - # set overrides - kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) - kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) - kwargs['env'] = filtered_env - - proc = subprocess.Popen(*args, **kwargs) - - full_output = "" - _stdout, _stderr = proc.communicate() - if _stdout: - _stdout = _stdout.decode("utf-8") - full_output += _stdout - logger.debug(_stdout) - - if _stderr: - _stderr = _stderr.decode("utf-8") - # Add additional line break if output already containt stdout - if full_output: - full_output += "\n" - full_output += _stderr - logger.warning(_stderr) - - if proc.returncode != 0: - exc_msg = "Executing arguments was not successful: \"{}\"".format(args) - if _stdout: - exc_msg += "\n\nOutput:\n{}".format(_stdout) - - if _stderr: - exc_msg += "Error:\n{}".format(_stderr) - - raise RuntimeError(exc_msg) - - return full_output - - -def get_hierarchy(asset_name=None): - """ - Obtain asset hierarchy path string from mongo db - - Returns: - string: asset hierarchy path - - """ - if not asset_name: - asset_name = io.Session.get("AVALON_ASSET", os.environ["AVALON_ASSET"]) - - asset_entity = io.find_one({ - "type": 'asset', - "name": asset_name - }) - - not_set = "PARENTS_NOT_SET" - entity_parents = asset_entity.get("data", {}).get("parents", not_set) - - # If entity already have parents then just return joined - if entity_parents != not_set: - return "/".join(entity_parents) - - # Else query parents through visualParents and store result to entity - hierarchy_items = [] - entity = asset_entity - while True: - parent_id = entity.get("data", {}).get("visualParent") - if not parent_id: - break - entity = io.find_one({"_id": parent_id}) - hierarchy_items.append(entity["name"]) - - # Add parents to entity data for next query - entity_data = asset_entity.get("data", {}) - entity_data["parents"] = hierarchy_items - io.update_many( - {"_id": asset_entity["_id"]}, - {"$set": {"data": entity_data}} - ) - - return "/".join(hierarchy_items) - - -def add_tool_to_environment(tools): - """ - It is adding dynamic environment to os environment. - - Args: - tool (list, tuple): list of tools, name should corespond to json/toml - - Returns: - os.environ[KEY]: adding to os.environ - """ - - import acre - tools_env = acre.get_tools(tools) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - os.environ.update(env) - - -@contextlib.contextmanager -def modified_environ(*remove, **update): - """ - Temporarily updates the ``os.environ`` dictionary in-place. - - The ``os.environ`` dictionary is updated in-place so that the modification - is sure to work in all situations. - - :param remove: Environment variables to remove. - :param update: Dictionary of environment variables - and values to add/update. - """ - env = os.environ - update = update or {} - remove = remove or [] - - # List of environment variables being updated or removed. - stomped = (set(update.keys()) | set(remove)) & set(env.keys()) - # Environment variables and values to restore on exit. - update_after = {k: env[k] for k in stomped} - # Environment variables and values to remove on exit. - remove_after = frozenset(k for k in update if k not in env) - - try: - env.update(update) - [env.pop(k, None) for k in remove] - yield - finally: - env.update(update_after) - [env.pop(k) for k in remove_after] - - -def pairwise(iterable): - """s -> (s0,s1), (s2,s3), (s4, s5), ...""" - a = iter(iterable) - return itertools.izip(a, a) - - -def grouper(iterable, n, fillvalue=None): - """Collect data into fixed-length chunks or blocks - - Examples: - grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx - - """ - - args = [iter(iterable)] * n - return itertools.izip_longest(fillvalue=fillvalue, *args) - - -def is_latest(representation): - """Return whether the representation is from latest version - - Args: - representation (dict): The representation document from the database. - - Returns: - bool: Whether the representation is of latest version. - - """ - - version = io.find_one({"_id": representation['parent']}) - if version["type"] == "master_version": - return True - - # Get highest version under the parent - highest_version = io.find_one({ - "type": "version", - "parent": version["parent"] - }, sort=[("name", -1)], projection={"name": True}) - - if version['name'] == highest_version['name']: - return True - else: - return False - - -def any_outdated(): - """Return whether the current scene has any outdated content""" - - checked = set() - host = avalon.api.registered_host() - for container in host.ls(): - representation = container['representation'] - if representation in checked: - continue - - representation_doc = io.find_one( - { - "_id": io.ObjectId(representation), - "type": "representation" - }, - projection={"parent": True} - ) - if representation_doc and not is_latest(representation_doc): - return True - elif not representation_doc: - log.debug("Container '{objectName}' has an invalid " - "representation, it is missing in the " - "database".format(**container)) - - checked.add(representation) - return False - - -def _rreplace(s, a, b, n=1): - """Replace a with b in string s from right side n times""" - return b.join(s.rsplit(a, n)) - - -def version_up(filepath): - """Version up filepath to a new non-existing version. - - Parses for a version identifier like `_v001` or `.v001` - When no version present _v001 is appended as suffix. - - Returns: - str: filepath with increased version number - - """ - - dirname = os.path.dirname(filepath) - basename, ext = os.path.splitext(os.path.basename(filepath)) - - regex = r"[._]v\d+" - matches = re.findall(regex, str(basename), re.IGNORECASE) - if not matches: - log.info("Creating version...") - new_label = "_v{version:03d}".format(version=1) - new_basename = "{}{}".format(basename, new_label) - else: - label = matches[-1] - version = re.search(r"\d+", label).group() - padding = len(version) - - new_version = int(version) + 1 - new_version = '{version:0{padding}d}'.format(version=new_version, - padding=padding) - new_label = label.replace(version, new_version, 1) - new_basename = _rreplace(basename, label, new_label) - - if not new_basename.endswith(new_label): - index = (new_basename.find(new_label)) - index += len(new_label) - new_basename = new_basename[:index] - - new_filename = "{}{}".format(new_basename, ext) - new_filename = os.path.join(dirname, new_filename) - new_filename = os.path.normpath(new_filename) - - if new_filename == filepath: - raise RuntimeError("Created path is the same as current file," - "this is a bug") - - for file in os.listdir(dirname): - if file.endswith(ext) and file.startswith(new_basename): - log.info("Skipping existing version %s" % new_label) - return version_up(new_filename) - - log.info("New version %s" % new_label) - return new_filename - - -def switch_item(container, - asset_name=None, - subset_name=None, - representation_name=None): - """Switch container asset, subset or representation of a container by name. - - It'll always switch to the latest version - of course a different - approach could be implemented. - - Args: - container (dict): data of the item to switch with - asset_name (str): name of the asset - subset_name (str): name of the subset - representation_name (str): name of the representation - - Returns: - dict - - """ - - if all(not x for x in [asset_name, subset_name, representation_name]): - raise ValueError("Must have at least one change provided to switch.") - - # Collect any of current asset, subset and representation if not provided - # so we can use the original name from those. - if any(not x for x in [asset_name, subset_name, representation_name]): - _id = io.ObjectId(container["representation"]) - representation = io.find_one({"type": "representation", "_id": _id}) - version, subset, asset, project = io.parenthood(representation) - - if asset_name is None: - asset_name = asset["name"] - - if subset_name is None: - subset_name = subset["name"] - - if representation_name is None: - representation_name = representation["name"] - - # Find the new one - asset = io.find_one({ - "name": asset_name, - "type": "asset" - }) - assert asset, ("Could not find asset in the database with the name " - "'%s'" % asset_name) - - subset = io.find_one({ - "name": subset_name, - "type": "subset", - "parent": asset["_id"] - }) - assert subset, ("Could not find subset in the database with the name " - "'%s'" % subset_name) - - version = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[('name', -1)] - ) - - assert version, "Could not find a version for {}.{}".format( - asset_name, subset_name - ) - - representation = io.find_one({ - "name": representation_name, - "type": "representation", - "parent": version["_id"]} - ) - - assert representation, ("Could not find representation in the database " - "with the name '%s'" % representation_name) - - avalon.api.switch(container, representation) - - return representation - - -def _get_host_name(): - - _host = avalon.api.registered_host() - # This covers nested module name like avalon.maya - return _host.__name__.rsplit(".", 1)[-1] - - -def get_asset(asset_name=None): - """ Returning asset document from database """ - if not asset_name: - asset_name = avalon.api.Session["AVALON_ASSET"] - - asset_document = io.find_one({ - "name": asset_name, - "type": "asset" - }) - - if not asset_document: - raise TypeError("Entity \"{}\" was not found in DB".format(asset_name)) - - return asset_document - - -def get_project(): - io.install() - return io.find_one({"type": "project"}) - - -def get_version_from_path(file): - """ - Finds version number in file path string - - Args: - file (string): file path - - Returns: - v: version number in string ('001') - - """ - pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) - try: - return pattern.findall(file)[0] - except IndexError: - log.error( - "templates:get_version_from_workfile:" - "`{}` missing version string." - "Example `v004`".format(file) - ) - - -def get_last_version_from_path(path_dir, filter): - """ - Finds last version of given directory content - - Args: - path_dir (string): directory path - filter (list): list of strings used as file name filter - - Returns: - string: file name with last version - - Example: - last_version_file = get_last_version_from_path( - "/project/shots/shot01/work", ["shot01", "compositing", "nk"]) - """ - - assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory" - assert isinstance(filter, list) and ( - len(filter) != 0), "`filter` argument needs to be list and not empty" - - filtred_files = list() - - # form regex for filtering - patern = r".*".join(filter) - - for f in os.listdir(path_dir): - if not re.findall(patern, f): - continue - filtred_files.append(f) - - if filtred_files: - sorted(filtred_files) - return filtred_files[-1] - else: - return None - - -def get_avalon_database(): - if io._database is None: - set_io_database() - return io._database - - -def set_io_database(): - required_keys = ["AVALON_PROJECT", "AVALON_ASSET", "AVALON_SILO"] - for key in required_keys: - os.environ[key] = os.environ.get(key, "") - io.install() - - -def filter_pyblish_plugins(plugins): - """ - This servers as plugin filter / modifier for pyblish. It will load plugin - definitions from presets and filter those needed to be excluded. - - :param plugins: Dictionary of plugins produced by :mod:`pyblish-base` - `discover()` method. - :type plugins: Dict - """ - from pyblish import api - - host = api.current_host() - - presets = config.get_presets().get('plugins', {}) - - # iterate over plugins - for plugin in plugins[:]: - # skip if there are no presets to process - if not presets: - continue - - file = os.path.normpath(inspect.getsourcefile(plugin)) - file = os.path.normpath(file) - - # host determined from path - host_from_file = file.split(os.path.sep)[-3:-2][0] - plugin_kind = file.split(os.path.sep)[-2:-1][0] - - try: - config_data = presets[host]["publish"][plugin.__name__] - except KeyError: - try: - config_data = presets[host_from_file][plugin_kind][plugin.__name__] # noqa: E501 - except KeyError: - continue - - for option, value in config_data.items(): - if option == "enabled" and value is False: - log.info('removing plugin {}'.format(plugin.__name__)) - plugins.remove(plugin) - else: - log.info('setting {}:{} on plugin {}'.format( - option, value, plugin.__name__)) - - setattr(plugin, option, value) - - -def get_subsets(asset_name, - regex_filter=None, - version=None, - representations=["exr", "dpx"]): - """ - Query subsets with filter on name. - - The method will return all found subsets and its defined version - and subsets. Version could be specified with number. Representation - can be filtered. - - Arguments: - asset_name (str): asset (shot) name - regex_filter (raw): raw string with filter pattern - version (str or int): `last` or number of version - representations (list): list for all representations - - Returns: - dict: subsets with version and representaions in keys - """ - - # query asset from db - asset_io = io.find_one({"type": "asset", "name": asset_name}) - - # check if anything returned - assert asset_io, ( - "Asset not existing. Check correct name: `{}`").format(asset_name) - - # create subsets query filter - filter_query = {"type": "subset", "parent": asset_io["_id"]} - - # add reggex filter string into query filter - if regex_filter: - filter_query.update({"name": {"$regex": r"{}".format(regex_filter)}}) - else: - filter_query.update({"name": {"$regex": r'.*'}}) - - # query all assets - subsets = [s for s in io.find(filter_query)] - - assert subsets, ("No subsets found. Check correct filter. " - "Try this for start `r'.*'`: " - "asset: `{}`").format(asset_name) - - output_dict = {} - # Process subsets - for subset in subsets: - if not version: - version_sel = io.find_one( - { - "type": "version", - "parent": subset["_id"] - }, - sort=[("name", -1)] - ) - else: - assert isinstance(version, int), "version needs to be `int` type" - version_sel = io.find_one({ - "type": "version", - "parent": subset["_id"], - "name": int(version) - }) - - find_dict = {"type": "representation", - "parent": version_sel["_id"]} - - filter_repr = {"name": {"$in": representations}} - - find_dict.update(filter_repr) - repres_out = [i for i in io.find(find_dict)] - - if len(repres_out) > 0: - output_dict[subset["name"]] = {"version": version_sel, - "representations": repres_out} - - return output_dict - - -class CustomNone: - """Created object can be used as custom None (not equal to None). - - WARNING: Multiple created objects are not equal either. - Exmple: - >>> a = CustomNone() - >>> a == None - False - >>> b = CustomNone() - >>> a == b - False - >>> a == a - True - """ - - def __init__(self): - """Create uuid as identifier for custom None.""" - self.identifier = str(uuid.uuid4()) - - def __bool__(self): - """Return False (like default None).""" - return False - - def __eq__(self, other): - """Equality is compared by identifier value.""" - if type(other) == type(self): - if other.identifier == self.identifier: - return True - return False - - def __str__(self): - """Return value of identifier when converted to string.""" - return self.identifier - - def __repr__(self): - """Representation of custom None.""" - return "".format(str(self.identifier)) - - -def execute_hook(hook, *args, **kwargs): - """ - This will load hook file, instantiate class and call `execute` method - on it. Hook must be in a form: - - `$PYPE_SETUP_PATH/repos/pype/path/to/hook.py/HookClass` - - This will load `hook.py`, instantiate HookClass and then execute_hook - `execute(*args, **kwargs)` - - :param hook: path to hook class - :type hook: str - """ - - class_name = hook.split("/")[-1] - - abspath = os.path.join(os.getenv('PYPE_SETUP_PATH'), - 'repos', 'pype', *hook.split("/")[:-1]) - - mod_name, mod_ext = os.path.splitext(os.path.basename(abspath)) - - if not mod_ext == ".py": - return False - - module = types.ModuleType(mod_name) - module.__file__ = abspath - - try: - with open(abspath) as f: - six.exec_(f.read(), module.__dict__) - - sys.modules[abspath] = module - - except Exception as exp: - log.exception("loading hook failed: {}".format(exp), - exc_info=True) - return False - - obj = getattr(module, class_name) - hook_obj = obj() - ret_val = hook_obj.execute(*args, **kwargs) - return ret_val - - -@six.add_metaclass(ABCMeta) -class PypeHook: - - def __init__(self): - pass - - @abstractmethod - def execute(self, *args, **kwargs): - pass - - -def get_linked_assets(asset_entity): - """Return linked assets for `asset_entity`.""" - inputs = asset_entity["data"].get("inputs", []) - inputs = [io.find_one({"_id": x}) for x in inputs] - return inputs - - -def map_subsets_by_family(subsets): - subsets_by_family = collections.defaultdict(list) - for subset in subsets: - family = subset["data"].get("family") - if not family: - families = subset["data"].get("families") - if not families: - continue - family = families[0] - - subsets_by_family[family].append(subset) - return subsets_by_family - - -class BuildWorkfile: - """Wrapper for build workfile process. - - Load representations for current context by build presets. Build presets - are host related, since each host has it's loaders. - """ - - def process(self): - """Main method of this wrapper. - - Building of workfile is triggered and is possible to implement - post processing of loaded containers if necessary. - """ - containers = self.build_workfile() - - return containers - - def build_workfile(self): - """Prepares and load containers into workfile. - - Loads latest versions of current and linked assets to workfile by logic - stored in Workfile profiles from presets. Profiles are set by host, - filtered by current task name and used by families. - - Each family can specify representation names and loaders for - representations and first available and successful loaded - representation is returned as container. - - At the end you'll get list of loaded containers per each asset. - - loaded_containers [{ - "asset_entity": , - "containers": [, , ...] - }, { - "asset_entity": , - "containers": [, ...] - }, { - ... - }] - """ - # Get current asset name and entity - current_asset_name = io.Session["AVALON_ASSET"] - current_asset_entity = io.find_one({ - "type": "asset", - "name": current_asset_name - }) - - # Skip if asset was not found - if not current_asset_entity: - print("Asset entity with name `{}` was not found".format( - current_asset_name - )) - return - - # Prepare available loaders - loaders_by_name = {} - for loader in avalon.api.discover(avalon.api.Loader): - loader_name = loader.__name__ - if loader_name in loaders_by_name: - raise KeyError( - "Duplicated loader name {0}!".format(loader_name) - ) - loaders_by_name[loader_name] = loader - - # Skip if there are any loaders - if not loaders_by_name: - log.warning("There are no registered loaders.") - return - - # Get current task name - current_task_name = io.Session["AVALON_TASK"] - - # Load workfile presets for task - self.build_presets = self.get_build_presets(current_task_name) - - # Skip if there are any presets for task - if not self.build_presets: - log.warning( - "Current task `{}` does not have any loading preset.".format( - current_task_name - ) - ) - return - - # Get presets for loading current asset - current_context_profiles = self.build_presets.get("current_context") - # Get presets for loading linked assets - link_context_profiles = self.build_presets.get("linked_assets") - # Skip if both are missing - if not current_context_profiles and not link_context_profiles: - log.warning("Current task `{}` has empty loading preset.".format( - current_task_name - )) - return - - elif not current_context_profiles: - log.warning(( - "Current task `{}` doesn't have any loading" - " preset for it's context." - ).format(current_task_name)) - - elif not link_context_profiles: - log.warning(( - "Current task `{}` doesn't have any" - "loading preset for it's linked assets." - ).format(current_task_name)) - - # Prepare assets to process by workfile presets - assets = [] - current_asset_id = None - if current_context_profiles: - # Add current asset entity if preset has current context set - assets.append(current_asset_entity) - current_asset_id = current_asset_entity["_id"] - - if link_context_profiles: - # Find and append linked assets if preset has set linked mapping - link_assets = get_linked_assets(current_asset_entity) - if link_assets: - assets.extend(link_assets) - - # Skip if there are no assets. This can happen if only linked mapping - # is set and there are no links for his asset. - if not assets: - log.warning( - "Asset does not have linked assets. Nothing to process." - ) - return - - # Prepare entities from database for assets - prepared_entities = self._collect_last_version_repres(assets) - - # Load containers by prepared entities and presets - loaded_containers = [] - # - Current asset containers - if current_asset_id and current_asset_id in prepared_entities: - current_context_data = prepared_entities.pop(current_asset_id) - loaded_data = self.load_containers_by_asset_data( - current_context_data, current_context_profiles, loaders_by_name - ) - if loaded_data: - loaded_containers.append(loaded_data) - - # - Linked assets container - for linked_asset_data in prepared_entities.values(): - loaded_data = self.load_containers_by_asset_data( - linked_asset_data, link_context_profiles, loaders_by_name - ) - if loaded_data: - loaded_containers.append(loaded_data) - - # Return list of loaded containers - return loaded_containers - - def get_build_presets(self, task_name): - """ Returns presets to build workfile for task name. - - Presets are loaded for current project set in - io.Session["AVALON_PROJECT"], filtered by registered host - and entered task name. - - :param task_name: Task name used for filtering build presets. - :type task_name: str - :return: preset per eneter task - :rtype: dict | None - """ - host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1] - presets = config.get_presets(io.Session["AVALON_PROJECT"]) - # Get presets for host - build_presets = ( - presets["plugins"] - .get(host_name, {}) - .get("workfile_build") - ) - if not build_presets: - return - - task_name_low = task_name.lower() - per_task_preset = None - for preset in build_presets: - preset_tasks = preset.get("tasks") or [] - preset_tasks_low = [task.lower() for task in preset_tasks] - if task_name_low in preset_tasks_low: - per_task_preset = preset - break - - return per_task_preset - - def _filter_build_profiles(self, build_profiles, loaders_by_name): - """ Filter build profiles by loaders and prepare process data. - - Valid profile must have "loaders", "families" and "repre_names" keys - with valid values. - - "loaders" expects list of strings representing possible loaders. - - "families" expects list of strings for filtering - by main subset family. - - "repre_names" expects list of strings for filtering by - representation name. - - Lowered "families" and "repre_names" are prepared for each profile with - all required keys. - - :param build_profiles: Profiles for building workfile. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Filtered and prepared profiles. - :rtype: list - """ - valid_profiles = [] - for profile in build_profiles: - # Check loaders - profile_loaders = profile.get("loaders") - if not profile_loaders: - log.warning(( - "Build profile has missing loaders configuration: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check if any loader is available - loaders_match = False - for loader_name in profile_loaders: - if loader_name in loaders_by_name: - loaders_match = True - break - - if not loaders_match: - log.warning(( - "All loaders from Build profile are not available: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check families - profile_families = profile.get("families") - if not profile_families: - log.warning(( - "Build profile is missing families configuration: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Check representation names - profile_repre_names = profile.get("repre_names") - if not profile_repre_names: - log.warning(( - "Build profile is missing" - " representation names filtering: {0}" - ).format(json.dumps(profile, indent=4))) - continue - - # Prepare lowered families and representation names - profile["families_lowered"] = [ - fam.lower() for fam in profile_families - ] - profile["repre_names_lowered"] = [ - name.lower() for name in profile_repre_names - ] - - valid_profiles.append(profile) - - return valid_profiles - - def _prepare_profile_for_subsets(self, subsets, profiles): - """Select profile for each subset byt it's data. - - Profiles are filtered for each subset individually. - Profile is filtered by subset's family, optionally by name regex and - representation names set in profile. - It is possible to not find matching profile for subset, in that case - subset is skipped and it is possible that none of subsets have - matching profile. - - :param subsets: Subset documents. - :type subsets: list - :param profiles: Build profiles. - :type profiles: dict - :return: Profile by subset's id. - :rtype: dict - """ - # Prepare subsets - subsets_by_family = map_subsets_by_family(subsets) - - profiles_per_subset_id = {} - for family, subsets in subsets_by_family.items(): - family_low = family.lower() - for profile in profiles: - # Skip profile if does not contain family - if family_low not in profile["families_lowered"]: - continue - - # Precompile name filters as regexes - profile_regexes = profile.get("subset_name_filters") - if profile_regexes: - _profile_regexes = [] - for regex in profile_regexes: - _profile_regexes.append(re.compile(regex)) - profile_regexes = _profile_regexes - - # TODO prepare regex compilation - for subset in subsets: - # Verify regex filtering (optional) - if profile_regexes: - valid = False - for pattern in profile_regexes: - if re.match(pattern, subset["name"]): - valid = True - break - - if not valid: - continue - - profiles_per_subset_id[subset["_id"]] = profile - - # break profiles loop on finding the first matching profile - break - return profiles_per_subset_id - - def load_containers_by_asset_data( - self, asset_entity_data, build_profiles, loaders_by_name - ): - """Load containers for entered asset entity by Build profiles. - - :param asset_entity_data: Prepared data with subsets, last version - and representations for specific asset. - :type asset_entity_data: dict - :param build_profiles: Build profiles. - :type build_profiles: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Output contains asset document and loaded containers. - :rtype: dict - """ - - # Make sure all data are not empty - if not asset_entity_data or not build_profiles or not loaders_by_name: - return - - asset_entity = asset_entity_data["asset_entity"] - - valid_profiles = self._filter_build_profiles( - build_profiles, loaders_by_name - ) - if not valid_profiles: - log.warning( - "There are not valid Workfile profiles. Skipping process." - ) - return - - log.debug("Valid Workfile profiles: {}".format(valid_profiles)) - - subsets_by_id = {} - version_by_subset_id = {} - repres_by_version_id = {} - for subset_id, in_data in asset_entity_data["subsets"].items(): - subset_entity = in_data["subset_entity"] - subsets_by_id[subset_entity["_id"]] = subset_entity - - version_data = in_data["version"] - version_entity = version_data["version_entity"] - version_by_subset_id[subset_id] = version_entity - repres_by_version_id[version_entity["_id"]] = ( - version_data["repres"] - ) - - if not subsets_by_id: - log.warning("There are not subsets for asset {0}".format( - asset_entity["name"] - )) - return - - profiles_per_subset_id = self._prepare_profile_for_subsets( - subsets_by_id.values(), valid_profiles - ) - if not profiles_per_subset_id: - log.warning("There are not valid subsets.") - return - - valid_repres_by_subset_id = collections.defaultdict(list) - for subset_id, profile in profiles_per_subset_id.items(): - profile_repre_names = profile["repre_names_lowered"] - - version_entity = version_by_subset_id[subset_id] - version_id = version_entity["_id"] - repres = repres_by_version_id[version_id] - for repre in repres: - repre_name_low = repre["name"].lower() - if repre_name_low in profile_repre_names: - valid_repres_by_subset_id[subset_id].append(repre) - - # DEBUG message - msg = "Valid representations for Asset: `{}`".format( - asset_entity["name"] - ) - for subset_id, repres in valid_repres_by_subset_id.items(): - subset = subsets_by_id[subset_id] - msg += "\n# Subset Name/ID: `{}`/{}".format( - subset["name"], subset_id - ) - for repre in repres: - msg += "\n## Repre name: `{}`".format(repre["name"]) - - log.debug(msg) - - containers = self._load_containers( - valid_repres_by_subset_id, subsets_by_id, - profiles_per_subset_id, loaders_by_name - ) - - return { - "asset_entity": asset_entity, - "containers": containers - } - - def _load_containers( - self, repres_by_subset_id, subsets_by_id, - profiles_per_subset_id, loaders_by_name - ): - """Real load by collected data happens here. - - Loading of representations per subset happens here. Each subset can - loads one representation. Loading is tried in specific order. - Representations are tried to load by names defined in configuration. - If subset has representation matching representation name each loader - is tried to load it until any is successful. If none of them was - successful then next reprensentation name is tried. - Subset process loop ends when any representation is loaded or - all matching representations were already tried. - - :param repres_by_subset_id: Available representations mapped - by their parent (subset) id. - :type repres_by_subset_id: dict - :param subsets_by_id: Subset documents mapped by their id. - :type subsets_by_id: dict - :param profiles_per_subset_id: Build profiles mapped by subset id. - :type profiles_per_subset_id: dict - :param loaders_by_name: Available loaders per name. - :type loaders_by_name: dict - :return: Objects of loaded containers. - :rtype: list - """ - loaded_containers = [] - - # Get subset id order from build presets. - build_presets = self.build_presets.get("current_context", []) - build_presets += self.build_presets.get("linked_assets", []) - subset_ids_ordered = [] - for preset in build_presets: - for preset_family in preset["families"]: - for id, subset in subsets_by_id.items(): - if preset_family not in subset["data"].get("families", []): - continue - - subset_ids_ordered.append(id) - - # Order representations from subsets. - print("repres_by_subset_id", repres_by_subset_id) - representations_ordered = [] - representations = [] - for id in subset_ids_ordered: - for subset_id, repres in repres_by_subset_id.items(): - if repres in representations: - continue - - if id == subset_id: - representations_ordered.append((subset_id, repres)) - representations.append(repres) - - print("representations", representations) - - # Load ordered reprensentations. - for subset_id, repres in representations_ordered: - subset_name = subsets_by_id[subset_id]["name"] - - profile = profiles_per_subset_id[subset_id] - loaders_last_idx = len(profile["loaders"]) - 1 - repre_names_last_idx = len(profile["repre_names_lowered"]) - 1 - - repre_by_low_name = { - repre["name"].lower(): repre for repre in repres - } - - is_loaded = False - for repre_name_idx, profile_repre_name in enumerate( - profile["repre_names_lowered"] - ): - # Break iteration if representation was already loaded - if is_loaded: - break - - repre = repre_by_low_name.get(profile_repre_name) - if not repre: - continue - - for loader_idx, loader_name in enumerate(profile["loaders"]): - if is_loaded: - break - - loader = loaders_by_name.get(loader_name) - if not loader: - continue - try: - container = avalon.api.load( - loader, - repre["_id"], - name=subset_name - ) - loaded_containers.append(container) - is_loaded = True - - except Exception as exc: - if exc == pipeline.IncompatibleLoaderError: - log.info(( - "Loader `{}` is not compatible with" - " representation `{}`" - ).format(loader_name, repre["name"])) - - else: - log.error( - "Unexpected error happened during loading", - exc_info=True - ) - - msg = "Loading failed." - if loader_idx < loaders_last_idx: - msg += " Trying next loader." - elif repre_name_idx < repre_names_last_idx: - msg += ( - " Loading of subset `{}` was not successful." - ).format(subset_name) - else: - msg += " Trying next representation." - log.info(msg) - - return loaded_containers - - def _collect_last_version_repres(self, asset_entities): - """Collect subsets, versions and representations for asset_entities. - - :param asset_entities: Asset entities for which want to find data - :type asset_entities: list - :return: collected entities - :rtype: dict - - Example output: - ``` - { - {Asset ID}: { - "asset_entity": , - "subsets": { - {Subset ID}: { - "subset_entity": , - "version": { - "version_entity": , - "repres": [ - , , ... - ] - } - }, - ... - } - }, - ... - } - output[asset_id]["subsets"][subset_id]["version"]["repres"] - ``` - """ - - if not asset_entities: - return {} - - asset_entity_by_ids = {asset["_id"]: asset for asset in asset_entities} - - subsets = list(io.find({ - "type": "subset", - "parent": {"$in": asset_entity_by_ids.keys()} - })) - subset_entity_by_ids = {subset["_id"]: subset for subset in subsets} - - sorted_versions = list(io.find({ - "type": "version", - "parent": {"$in": subset_entity_by_ids.keys()} - }).sort("name", -1)) - - subset_id_with_latest_version = [] - last_versions_by_id = {} - for version in sorted_versions: - subset_id = version["parent"] - if subset_id in subset_id_with_latest_version: - continue - subset_id_with_latest_version.append(subset_id) - last_versions_by_id[version["_id"]] = version - - repres = io.find({ - "type": "representation", - "parent": {"$in": last_versions_by_id.keys()} - }) - - output = {} - for repre in repres: - version_id = repre["parent"] - version = last_versions_by_id[version_id] - - subset_id = version["parent"] - subset = subset_entity_by_ids[subset_id] - - asset_id = subset["parent"] - asset = asset_entity_by_ids[asset_id] - - if asset_id not in output: - output[asset_id] = { - "asset_entity": asset, - "subsets": {} - } - - if subset_id not in output[asset_id]["subsets"]: - output[asset_id]["subsets"][subset_id] = { - "subset_entity": subset, - "version": { - "version_entity": version, - "repres": [] - } - } - - output[asset_id]["subsets"][subset_id]["version"]["repres"].append( - repre - ) - - return output - - -def ffprobe_streams(path_to_file, logger=None): - """Load streams from entered filepath via ffprobe.""" - if not logger: - logger = log - logger.info( - "Getting information about input \"{}\".".format(path_to_file) - ) - args = [ - "\"{}\"".format(get_ffmpeg_tool_path("ffprobe")), - "-v quiet", - "-print_format json", - "-show_format", - "-show_streams", - "\"{}\"".format(path_to_file) - ] - command = " ".join(args) - logger.debug("FFprobe command: \"{}\"".format(command)) - popen = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - popen_stdout, popen_stderr = popen.communicate() - if popen_stdout: - logger.debug("ffprobe stdout: {}".format(popen_stdout)) - - if popen_stderr: - logger.debug("ffprobe stderr: {}".format(popen_stderr)) - return json.loads(popen_stdout)["streams"] - - -def source_hash(filepath, *args): - """Generate simple identifier for a source file. - This is used to identify whether a source file has previously been - processe into the pipeline, e.g. a texture. - The hash is based on source filepath, modification time and file size. - This is only used to identify whether a specific source file was already - published before from the same location with the same modification date. - We opt to do it this way as opposed to Avalanch C4 hash as this is much - faster and predictable enough for all our production use cases. - Args: - filepath (str): The source file path. - You can specify additional arguments in the function - to allow for specific 'processing' values to be included. - """ - # We replace dots with comma because . cannot be a key in a pymongo dict. - file_name = os.path.basename(filepath) - time = str(os.path.getmtime(filepath)) - size = str(os.path.getsize(filepath)) - return "|".join([file_name, time, size] + list(args)).replace(".", ",") - - -def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): - """Retrieve latest version from `asset_name`, and `subset_name`. - - Do not use if you want to query more than 5 latest versions as this method - query 3 times to mongo for each call. For those cases is better to use - more efficient way, e.g. with help of aggregations. - - Args: - asset_name (str): Name of asset. - subset_name (str): Name of subset. - dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection - with Session. - project_name (str, optional): Find latest version in specific project. - - Returns: - None: If asset, subset or version were not found. - dict: Last version document for entered . - """ - - if not dbcon: - log.debug("Using `avalon.io` for query.") - dbcon = io - # Make sure is installed - io.install() - - if project_name and project_name != dbcon.Session.get("AVALON_PROJECT"): - # `avalon.io` has only `_database` attribute - # but `AvalonMongoDB` has `database` - database = getattr(dbcon, "database", dbcon._database) - collection = database[project_name] - else: - project_name = dbcon.Session.get("AVALON_PROJECT") - collection = dbcon - - log.debug(( - "Getting latest version for Project: \"{}\" Asset: \"{}\"" - " and Subset: \"{}\"" - ).format(project_name, asset_name, subset_name)) - - # Query asset document id by asset name - asset_doc = collection.find_one( - {"type": "asset", "name": asset_name}, - {"_id": True} - ) - if not asset_doc: - log.info( - "Asset \"{}\" was not found in Database.".format(asset_name) - ) - return None - - subset_doc = collection.find_one( - {"type": "subset", "name": subset_name, "parent": asset_doc["_id"]}, - {"_id": True} - ) - if not subset_doc: - log.info( - "Subset \"{}\" was not found in Database.".format(subset_name) - ) - return None - - version_doc = collection.find_one( - {"type": "version", "parent": subset_doc["_id"]}, - sort=[("name", -1)], - ) - if not version_doc: - log.info( - "Subset \"{}\" does not have any version yet.".format(subset_name) - ) - return None - return version_doc - - -class ApplicationLaunchFailed(Exception): - pass - - -def launch_application(project_name, asset_name, task_name, app_name): - database = get_avalon_database() - project_document = database[project_name].find_one({"type": "project"}) - asset_document = database[project_name].find_one({ - "type": "asset", - "name": asset_name - }) - - asset_doc_parents = asset_document["data"].get("parents") - hierarchy = "/".join(asset_doc_parents) - - app_def = avalon.lib.get_application(app_name) - app_label = app_def.get("ftrack_label", app_def.get("label", app_name)) - - host_name = app_def["application_dir"] - data = { - "project": { - "name": project_document["name"], - "code": project_document["data"].get("code") - }, - "task": task_name, - "asset": asset_name, - "app": host_name, - "hierarchy": hierarchy - } - - try: - anatomy = Anatomy(project_name) - anatomy_filled = anatomy.format(data) - workdir = os.path.normpath(anatomy_filled["work"]["folder"]) - - except Exception as exc: - raise ApplicationLaunchFailed( - "Error in anatomy.format: {}".format(str(exc)) - ) - - try: - os.makedirs(workdir) - except FileExistsError: - pass - - last_workfile_path = None - extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(host_name) - if extensions: - # Find last workfile - file_template = anatomy.templates["work"]["file"] - data.update({ - "version": 1, - "user": os.environ.get("PYPE_USERNAME") or getpass.getuser(), - "ext": extensions[0] - }) - - last_workfile_path = avalon.api.last_workfile( - workdir, file_template, data, extensions, True - ) - - # set environments for Avalon - prep_env = copy.deepcopy(os.environ) - prep_env.update({ - "AVALON_PROJECT": project_name, - "AVALON_ASSET": asset_name, - "AVALON_TASK": task_name, - "AVALON_APP": host_name, - "AVALON_APP_NAME": app_name, - "AVALON_HIERARCHY": hierarchy, - "AVALON_WORKDIR": workdir - }) - - start_last_workfile = avalon.api.should_start_last_workfile( - project_name, host_name, task_name - ) - # Store boolean as "0"(False) or "1"(True) - prep_env["AVALON_OPEN_LAST_WORKFILE"] = ( - str(int(bool(start_last_workfile))) - ) - - if ( - start_last_workfile - and last_workfile_path - and os.path.exists(last_workfile_path) - ): - prep_env["AVALON_LAST_WORKFILE"] = last_workfile_path - - prep_env.update(anatomy.roots_obj.root_environments()) - - # collect all the 'environment' attributes from parents - tools_attr = [prep_env["AVALON_APP"], prep_env["AVALON_APP_NAME"]] - tools_env = asset_document["data"].get("tools_env") or [] - tools_attr.extend(tools_env) - - tools_env = acre.get_tools(tools_attr) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(prep_env)) - - # Get path to execute - st_temp_path = os.environ["PYPE_CONFIG"] - os_plat = platform.system().lower() - - # Path to folder with launchers - path = os.path.join(st_temp_path, "launchers", os_plat) - - # Full path to executable launcher - execfile = None - - launch_hook = app_def.get("launch_hook") - if launch_hook: - log.info("launching hook: {}".format(launch_hook)) - ret_val = execute_hook(launch_hook, env=env) - if not ret_val: - raise ApplicationLaunchFailed( - "Hook didn't finish successfully {}".format(app_label) - ) - - if sys.platform == "win32": - for ext in os.environ["PATHEXT"].split(os.pathsep): - fpath = os.path.join(path.strip('"'), app_def["executable"] + ext) - if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - execfile = fpath - break - - # Run SW if was found executable - if execfile is None: - raise ApplicationLaunchFailed( - "We didn't find launcher for {}".format(app_label) - ) - - popen = avalon.lib.launch( - executable=execfile, args=[], environment=env - ) - - elif ( - sys.platform.startswith("linux") - or sys.platform.startswith("darwin") - ): - execfile = os.path.join(path.strip('"'), app_def["executable"]) - # Run SW if was found executable - if execfile is None: - raise ApplicationLaunchFailed( - "We didn't find launcher for {}".format(app_label) - ) - - if not os.path.isfile(execfile): - raise ApplicationLaunchFailed( - "Launcher doesn't exist - {}".format(execfile) - ) - - try: - fp = open(execfile) - except PermissionError as perm_exc: - raise ApplicationLaunchFailed( - "Access denied on launcher {} - {}".format(execfile, perm_exc) - ) - - fp.close() - # check executable permission - if not os.access(execfile, os.X_OK): - raise ApplicationLaunchFailed( - "No executable permission - {}".format(execfile) - ) - - popen = avalon.lib.launch( # noqa: F841 - "/usr/bin/env", args=["bash", execfile], environment=env - ) - return popen - - -class ApplicationAction(avalon.api.Action): - """Default application launcher - - This is a convenience application Action that when "config" refers to a - parsed application `.toml` this can launch the application. - - """ - _log = None - config = None - group = None - variant = None - required_session_keys = ( - "AVALON_PROJECT", - "AVALON_ASSET", - "AVALON_TASK" - ) - - @property - def log(self): - if self._log is None: - self._log = Logger().get_logger(self.__class__.__name__) - return self._log - - def is_compatible(self, session): - for key in self.required_session_keys: - if key not in session: - return False - return True - - def process(self, session, **kwargs): - """Process the full Application action""" - - project_name = session["AVALON_PROJECT"] - asset_name = session["AVALON_ASSET"] - task_name = session["AVALON_TASK"] - launch_application( - project_name, asset_name, task_name, self.name - ) - - self._ftrack_after_launch_procedure( - project_name, asset_name, task_name - ) - - def _ftrack_after_launch_procedure( - self, project_name, asset_name, task_name - ): - # TODO move to launch hook - required_keys = ("FTRACK_SERVER", "FTRACK_API_USER", "FTRACK_API_KEY") - for key in required_keys: - if not os.environ.get(key): - self.log.debug(( - "Missing required environment \"{}\"" - " for Ftrack after launch procedure." - ).format(key)) - return - - try: - import ftrack_api - session = ftrack_api.Session(auto_connect_event_hub=True) - self.log.debug("Ftrack session created") - except Exception: - self.log.warning("Couldn't create Ftrack session") - return - - try: - entity = self._find_ftrack_task_entity( - session, project_name, asset_name, task_name - ) - self._ftrack_status_change(session, entity, project_name) - self._start_timer(session, entity, ftrack_api) - except Exception: - self.log.warning( - "Couldn't finish Ftrack procedure.", exc_info=True - ) - return - - finally: - session.close() - - def _find_ftrack_task_entity( - self, session, project_name, asset_name, task_name - ): - project_entity = session.query( - "Project where full_name is \"{}\"".format(project_name) - ).first() - if not project_entity: - self.log.warning( - "Couldn't find project \"{}\" in Ftrack.".format(project_name) - ) - return - - potential_task_entities = session.query(( - "TypedContext where parent.name is \"{}\" and project_id is \"{}\"" - ).format(asset_name, project_entity["id"])).all() - filtered_entities = [] - for _entity in potential_task_entities: - if ( - _entity.entity_type.lower() == "task" - and _entity["name"] == task_name - ): - filtered_entities.append(_entity) - - if not filtered_entities: - self.log.warning(( - "Couldn't find task \"{}\" under parent \"{}\" in Ftrack." - ).format(task_name, asset_name)) - return - - if len(filtered_entities) > 1: - self.log.warning(( - "Found more than one task \"{}\"" - " under parent \"{}\" in Ftrack." - ).format(task_name, asset_name)) - return - - return filtered_entities[0] - - def _ftrack_status_change(self, session, entity, project_name): - presets = config.get_presets(project_name)["ftrack"]["ftrack_config"] - statuses = presets.get("status_update") - if not statuses: - return - - actual_status = entity["status"]["name"].lower() - already_tested = set() - ent_path = "/".join( - [ent["name"] for ent in entity["link"]] - ) - while True: - next_status_name = None - for key, value in statuses.items(): - if key in already_tested: - continue - if actual_status in value or "_any_" in value: - if key != "_ignore_": - next_status_name = key - already_tested.add(key) - break - already_tested.add(key) - - if next_status_name is None: - break - - try: - query = "Status where name is \"{}\"".format( - next_status_name - ) - status = session.query(query).one() - - entity["status"] = status - session.commit() - self.log.debug("Changing status to \"{}\" <{}>".format( - next_status_name, ent_path - )) - break - - except Exception: - session.rollback() - msg = ( - "Status \"{}\" in presets wasn't found" - " on Ftrack entity type \"{}\"" - ).format(next_status_name, entity.entity_type) - self.log.warning(msg) - - def _start_timer(self, session, entity, _ftrack_api): - self.log.debug("Triggering timer start.") - - user_entity = session.query("User where username is \"{}\"".format( - os.environ["FTRACK_API_USER"] - )).first() - if not user_entity: - self.log.warning( - "Couldn't find user with username \"{}\" in Ftrack".format( - os.environ["FTRACK_API_USER"] - ) - ) - return - - source = { - "user": { - "id": user_entity["id"], - "username": user_entity["username"] - } - } - event_data = { - "actionIdentifier": "start.timer", - "selection": [{"entityId": entity["id"], "entityType": "task"}] - } - session.event_hub.publish( - _ftrack_api.event.base.Event( - topic="ftrack.action.launch", - data=event_data, - source=source - ), - on_error="ignore" - ) - self.log.debug("Timer start triggered successfully.") diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py new file mode 100644 index 0000000000..1ade97cd0e --- /dev/null +++ b/pype/lib/__init__.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Pype lib module.""" + +from .deprecated import ( + get_avalon_database, + set_io_database +) + +from .avalon_context import ( + is_latest, + any_outdated, + get_asset, + get_hierarchy, + get_linked_assets, + get_latest_version, + BuildWorkfile +) + +from .hooks import PypeHook, execute_hook + +from .applications import ( + ApplicationLaunchFailed, + launch_application, + ApplicationAction, + _subprocess +) + +from .plugin_tools import filter_pyblish_plugins, source_hash + +from .path_tools import ( + version_up, + get_version_from_path, + get_last_version_from_path, + get_paths_from_environ, + get_ffmpeg_tool_path +) + +from .ffmpeg_utils import ffprobe_streams + +__all__ = [ + "get_avalon_database", + "set_io_database", + + "is_latest", + "any_outdated", + "get_asset", + "get_hierarchy", + "get_linked_assets", + "get_latest_version", + "BuildWorkfile", + + "PypeHook", + "execute_hook", + + "ApplicationLaunchFailed", + "launch_application", + "ApplicationAction", + + "filter_pyblish_plugins", + + "version_up", + "get_version_from_path", + "get_last_version_from_path", + "get_paths_from_environ", + "get_ffmpeg_tool_path", + + "ffprobe_streams", + + "source_hash", + "_subprocess" +] diff --git a/pype/lib/applications.py b/pype/lib/applications.py new file mode 100644 index 0000000000..159840ceb5 --- /dev/null +++ b/pype/lib/applications.py @@ -0,0 +1,457 @@ +import os +import sys +import getpass +import copy +import platform +import logging +import subprocess + +import acre + +import avalon.lib + +from ..api import Anatomy, Logger, config +from .hooks import execute_hook +from .deprecated import get_avalon_database + +log = logging.getLogger(__name__) + + +class ApplicationLaunchFailed(Exception): + pass + + +def launch_application(project_name, asset_name, task_name, app_name): + """Launch host application with filling required environments. + + TODO(iLLiCiT): This should be split into more parts. + """ + # `get_avalon_database` is in Pype 3 replaced with using `AvalonMongoDB` + database = get_avalon_database() + project_document = database[project_name].find_one({"type": "project"}) + asset_document = database[project_name].find_one({ + "type": "asset", + "name": asset_name + }) + + asset_doc_parents = asset_document["data"].get("parents") + hierarchy = "/".join(asset_doc_parents) + + app_def = avalon.lib.get_application(app_name) + app_label = app_def.get("ftrack_label", app_def.get("label", app_name)) + + host_name = app_def["application_dir"] + # Workfile data collection may be special function? + data = { + "project": { + "name": project_document["name"], + "code": project_document["data"].get("code") + }, + "task": task_name, + "asset": asset_name, + "app": host_name, + "hierarchy": hierarchy + } + + try: + anatomy = Anatomy(project_name) + anatomy_filled = anatomy.format(data) + workdir = os.path.normpath(anatomy_filled["work"]["folder"]) + + except Exception as exc: + raise ApplicationLaunchFailed( + "Error in anatomy.format: {}".format(str(exc)) + ) + + try: + os.makedirs(workdir) + except FileExistsError: + pass + + last_workfile_path = None + extensions = avalon.api.HOST_WORKFILE_EXTENSIONS.get(host_name) + if extensions: + # Find last workfile + file_template = anatomy.templates["work"]["file"] + data.update({ + "version": 1, + "user": os.environ.get("PYPE_USERNAME") or getpass.getuser(), + "ext": extensions[0] + }) + + last_workfile_path = avalon.api.last_workfile( + workdir, file_template, data, extensions, True + ) + + # set environments for Avalon + prep_env = copy.deepcopy(os.environ) + prep_env.update({ + "AVALON_PROJECT": project_name, + "AVALON_ASSET": asset_name, + "AVALON_TASK": task_name, + "AVALON_APP": host_name, + "AVALON_APP_NAME": app_name, + "AVALON_HIERARCHY": hierarchy, + "AVALON_WORKDIR": workdir + }) + + start_last_workfile = avalon.api.should_start_last_workfile( + project_name, host_name, task_name + ) + # Store boolean as "0"(False) or "1"(True) + prep_env["AVALON_OPEN_LAST_WORKFILE"] = ( + str(int(bool(start_last_workfile))) + ) + + if ( + start_last_workfile + and last_workfile_path + and os.path.exists(last_workfile_path) + ): + prep_env["AVALON_LAST_WORKFILE"] = last_workfile_path + + prep_env.update(anatomy.roots_obj.root_environments()) + + # collect all the 'environment' attributes from parents + tools_attr = [prep_env["AVALON_APP"], prep_env["AVALON_APP_NAME"]] + tools_env = asset_document["data"].get("tools_env") or [] + tools_attr.extend(tools_env) + + tools_env = acre.get_tools(tools_attr) + env = acre.compute(tools_env) + env = acre.merge(env, current_env=dict(prep_env)) + + # Get path to execute + st_temp_path = os.environ["PYPE_CONFIG"] + os_plat = platform.system().lower() + + # Path to folder with launchers + path = os.path.join(st_temp_path, "launchers", os_plat) + + # Full path to executable launcher + execfile = None + + launch_hook = app_def.get("launch_hook") + if launch_hook: + log.info("launching hook: {}".format(launch_hook)) + ret_val = execute_hook(launch_hook, env=env) + if not ret_val: + raise ApplicationLaunchFailed( + "Hook didn't finish successfully {}".format(app_label) + ) + + if sys.platform == "win32": + for ext in os.environ["PATHEXT"].split(os.pathsep): + fpath = os.path.join(path.strip('"'), app_def["executable"] + ext) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + execfile = fpath + break + + # Run SW if was found executable + if execfile is None: + raise ApplicationLaunchFailed( + "We didn't find launcher for {}".format(app_label) + ) + + popen = avalon.lib.launch( + executable=execfile, args=[], environment=env + ) + + elif ( + sys.platform.startswith("linux") + or sys.platform.startswith("darwin") + ): + execfile = os.path.join(path.strip('"'), app_def["executable"]) + # Run SW if was found executable + if execfile is None: + raise ApplicationLaunchFailed( + "We didn't find launcher for {}".format(app_label) + ) + + if not os.path.isfile(execfile): + raise ApplicationLaunchFailed( + "Launcher doesn't exist - {}".format(execfile) + ) + + try: + fp = open(execfile) + except PermissionError as perm_exc: + raise ApplicationLaunchFailed( + "Access denied on launcher {} - {}".format(execfile, perm_exc) + ) + + fp.close() + # check executable permission + if not os.access(execfile, os.X_OK): + raise ApplicationLaunchFailed( + "No executable permission - {}".format(execfile) + ) + + popen = avalon.lib.launch( # noqa: F841 + "/usr/bin/env", args=["bash", execfile], environment=env + ) + return popen + + +class ApplicationAction(avalon.api.Action): + """Default application launcher + + This is a convenience application Action that when "config" refers to a + parsed application `.toml` this can launch the application. + + """ + _log = None + config = None + group = None + variant = None + required_session_keys = ( + "AVALON_PROJECT", + "AVALON_ASSET", + "AVALON_TASK" + ) + + @property + def log(self): + if self._log is None: + self._log = Logger().get_logger(self.__class__.__name__) + return self._log + + def is_compatible(self, session): + for key in self.required_session_keys: + if key not in session: + return False + return True + + def process(self, session, **kwargs): + """Process the full Application action""" + + project_name = session["AVALON_PROJECT"] + asset_name = session["AVALON_ASSET"] + task_name = session["AVALON_TASK"] + launch_application( + project_name, asset_name, task_name, self.name + ) + + self._ftrack_after_launch_procedure( + project_name, asset_name, task_name + ) + + def _ftrack_after_launch_procedure( + self, project_name, asset_name, task_name + ): + # TODO move to launch hook + required_keys = ("FTRACK_SERVER", "FTRACK_API_USER", "FTRACK_API_KEY") + for key in required_keys: + if not os.environ.get(key): + self.log.debug(( + "Missing required environment \"{}\"" + " for Ftrack after launch procedure." + ).format(key)) + return + + try: + import ftrack_api + session = ftrack_api.Session(auto_connect_event_hub=True) + self.log.debug("Ftrack session created") + except Exception: + self.log.warning("Couldn't create Ftrack session") + return + + try: + entity = self._find_ftrack_task_entity( + session, project_name, asset_name, task_name + ) + self._ftrack_status_change(session, entity, project_name) + self._start_timer(session, entity, ftrack_api) + except Exception: + self.log.warning( + "Couldn't finish Ftrack procedure.", exc_info=True + ) + return + + finally: + session.close() + + def _find_ftrack_task_entity( + self, session, project_name, asset_name, task_name + ): + project_entity = session.query( + "Project where full_name is \"{}\"".format(project_name) + ).first() + if not project_entity: + self.log.warning( + "Couldn't find project \"{}\" in Ftrack.".format(project_name) + ) + return + + potential_task_entities = session.query(( + "TypedContext where parent.name is \"{}\" and project_id is \"{}\"" + ).format(asset_name, project_entity["id"])).all() + filtered_entities = [] + for _entity in potential_task_entities: + if ( + _entity.entity_type.lower() == "task" + and _entity["name"] == task_name + ): + filtered_entities.append(_entity) + + if not filtered_entities: + self.log.warning(( + "Couldn't find task \"{}\" under parent \"{}\" in Ftrack." + ).format(task_name, asset_name)) + return + + if len(filtered_entities) > 1: + self.log.warning(( + "Found more than one task \"{}\"" + " under parent \"{}\" in Ftrack." + ).format(task_name, asset_name)) + return + + return filtered_entities[0] + + def _ftrack_status_change(self, session, entity, project_name): + presets = config.get_presets(project_name)["ftrack"]["ftrack_config"] + statuses = presets.get("status_update") + if not statuses: + return + + actual_status = entity["status"]["name"].lower() + already_tested = set() + ent_path = "/".join( + [ent["name"] for ent in entity["link"]] + ) + while True: + next_status_name = None + for key, value in statuses.items(): + if key in already_tested: + continue + if actual_status in value or "_any_" in value: + if key != "_ignore_": + next_status_name = key + already_tested.add(key) + break + already_tested.add(key) + + if next_status_name is None: + break + + try: + query = "Status where name is \"{}\"".format( + next_status_name + ) + status = session.query(query).one() + + entity["status"] = status + session.commit() + self.log.debug("Changing status to \"{}\" <{}>".format( + next_status_name, ent_path + )) + break + + except Exception: + session.rollback() + msg = ( + "Status \"{}\" in presets wasn't found" + " on Ftrack entity type \"{}\"" + ).format(next_status_name, entity.entity_type) + self.log.warning(msg) + + def _start_timer(self, session, entity, _ftrack_api): + self.log.debug("Triggering timer start.") + + user_entity = session.query("User where username is \"{}\"".format( + os.environ["FTRACK_API_USER"] + )).first() + if not user_entity: + self.log.warning( + "Couldn't find user with username \"{}\" in Ftrack".format( + os.environ["FTRACK_API_USER"] + ) + ) + return + + source = { + "user": { + "id": user_entity["id"], + "username": user_entity["username"] + } + } + event_data = { + "actionIdentifier": "start.timer", + "selection": [{"entityId": entity["id"], "entityType": "task"}] + } + session.event_hub.publish( + _ftrack_api.event.base.Event( + topic="ftrack.action.launch", + data=event_data, + source=source + ), + on_error="ignore" + ) + self.log.debug("Timer start triggered successfully.") + + +# Special naming case for subprocess since its a built-in method. +def _subprocess(*args, **kwargs): + """Convenience method for getting output errors for subprocess. + + Entered arguments and keyword arguments are passed to subprocess Popen. + + Args: + *args: Variable length arument list passed to Popen. + **kwargs : Arbitary keyword arguments passed to Popen. Is possible to + pass `logging.Logger` object under "logger" if want to use + different than lib's logger. + + Returns: + str: Full output of subprocess concatenated stdout and stderr. + + Raises: + RuntimeError: Exception is raised if process finished with nonzero + return code. + """ + + # Get environents from kwarg or use current process environments if were + # not passed. + env = kwargs.get("env") or os.environ + # Make sure environment contains only strings + filtered_env = {k: str(v) for k, v in env.items()} + + # Use lib's logger if was not passed with kwargs. + logger = kwargs.pop("logger", log) + + # set overrides + kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) + kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) + kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) + kwargs['env'] = filtered_env + + proc = subprocess.Popen(*args, **kwargs) + + full_output = "" + _stdout, _stderr = proc.communicate() + if _stdout: + _stdout = _stdout.decode("utf-8") + full_output += _stdout + logger.debug(_stdout) + + if _stderr: + _stderr = _stderr.decode("utf-8") + # Add additional line break if output already containt stdout + if full_output: + full_output += "\n" + full_output += _stderr + logger.warning(_stderr) + + if proc.returncode != 0: + exc_msg = "Executing arguments was not successful: \"{}\"".format(args) + if _stdout: + exc_msg += "\n\nOutput:\n{}".format(_stdout) + + if _stderr: + exc_msg += "Error:\n{}".format(_stderr) + + raise RuntimeError(exc_msg) + + return full_output diff --git a/pype/lib/avalon_context.py b/pype/lib/avalon_context.py new file mode 100644 index 0000000000..6cecdb93e3 --- /dev/null +++ b/pype/lib/avalon_context.py @@ -0,0 +1,870 @@ +import os +import json +import re +import logging +import collections + +from avalon import io, pipeline +from ..api import config +import avalon.api + +log = logging.getLogger("AvalonContext") + + +def is_latest(representation): + """Return whether the representation is from latest version + + Args: + representation (dict): The representation document from the database. + + Returns: + bool: Whether the representation is of latest version. + + """ + + version = io.find_one({"_id": representation['parent']}) + if version["type"] == "master_version": + return True + + # Get highest version under the parent + highest_version = io.find_one({ + "type": "version", + "parent": version["parent"] + }, sort=[("name", -1)], projection={"name": True}) + + if version['name'] == highest_version['name']: + return True + else: + return False + + +def any_outdated(): + """Return whether the current scene has any outdated content""" + + checked = set() + host = avalon.api.registered_host() + for container in host.ls(): + representation = container['representation'] + if representation in checked: + continue + + representation_doc = io.find_one( + { + "_id": io.ObjectId(representation), + "type": "representation" + }, + projection={"parent": True} + ) + if representation_doc and not is_latest(representation_doc): + return True + elif not representation_doc: + log.debug("Container '{objectName}' has an invalid " + "representation, it is missing in the " + "database".format(**container)) + + checked.add(representation) + return False + + +def get_asset(asset_name=None): + """ Returning asset document from database by its name. + + Doesn't count with duplicities on asset names! + + Args: + asset_name (str) + + Returns: + (MongoDB document) + """ + if not asset_name: + asset_name = avalon.api.Session["AVALON_ASSET"] + + asset_document = io.find_one({ + "name": asset_name, + "type": "asset" + }) + + if not asset_document: + raise TypeError("Entity \"{}\" was not found in DB".format(asset_name)) + + return asset_document + + +def get_hierarchy(asset_name=None): + """ + Obtain asset hierarchy path string from mongo db + + Args: + asset_name (str) + + Returns: + (string): asset hierarchy path + + """ + if not asset_name: + asset_name = io.Session.get("AVALON_ASSET", os.environ["AVALON_ASSET"]) + + asset_entity = io.find_one({ + "type": 'asset', + "name": asset_name + }) + + not_set = "PARENTS_NOT_SET" + entity_parents = asset_entity.get("data", {}).get("parents", not_set) + + # If entity already have parents then just return joined + if entity_parents != not_set: + return "/".join(entity_parents) + + # Else query parents through visualParents and store result to entity + hierarchy_items = [] + entity = asset_entity + while True: + parent_id = entity.get("data", {}).get("visualParent") + if not parent_id: + break + entity = io.find_one({"_id": parent_id}) + hierarchy_items.append(entity["name"]) + + # Add parents to entity data for next query + entity_data = asset_entity.get("data", {}) + entity_data["parents"] = hierarchy_items + io.update_many( + {"_id": asset_entity["_id"]}, + {"$set": {"data": entity_data}} + ) + + return "/".join(hierarchy_items) + + +def get_linked_assets(asset_entity): + """Return linked assets for `asset_entity` from DB + + Args: + asset_entity (dict): asset document from DB + + Returns: + (list) of MongoDB documents + """ + inputs = asset_entity["data"].get("inputs", []) + inputs = [io.find_one({"_id": x}) for x in inputs] + return inputs + + +def get_latest_version(asset_name, subset_name, dbcon=None, project_name=None): + """Retrieve latest version from `asset_name`, and `subset_name`. + + Do not use if you want to query more than 5 latest versions as this method + query 3 times to mongo for each call. For those cases is better to use + more efficient way, e.g. with help of aggregations. + + Args: + asset_name (str): Name of asset. + subset_name (str): Name of subset. + dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection + with Session. + project_name (str, optional): Find latest version in specific project. + + Returns: + None: If asset, subset or version were not found. + dict: Last version document for entered . + """ + + if not dbcon: + log.debug("Using `avalon.io` for query.") + dbcon = io + # Make sure is installed + io.install() + + if project_name and project_name != dbcon.Session.get("AVALON_PROJECT"): + # `avalon.io` has only `_database` attribute + # but `AvalonMongoDB` has `database` + database = getattr(dbcon, "database", dbcon._database) + collection = database[project_name] + else: + project_name = dbcon.Session.get("AVALON_PROJECT") + collection = dbcon + + log.debug(( + "Getting latest version for Project: \"{}\" Asset: \"{}\"" + " and Subset: \"{}\"" + ).format(project_name, asset_name, subset_name)) + + # Query asset document id by asset name + asset_doc = collection.find_one( + {"type": "asset", "name": asset_name}, + {"_id": True} + ) + if not asset_doc: + log.info( + "Asset \"{}\" was not found in Database.".format(asset_name) + ) + return None + + subset_doc = collection.find_one( + {"type": "subset", "name": subset_name, "parent": asset_doc["_id"]}, + {"_id": True} + ) + if not subset_doc: + log.info( + "Subset \"{}\" was not found in Database.".format(subset_name) + ) + return None + + version_doc = collection.find_one( + {"type": "version", "parent": subset_doc["_id"]}, + sort=[("name", -1)], + ) + if not version_doc: + log.info( + "Subset \"{}\" does not have any version yet.".format(subset_name) + ) + return None + return version_doc + + +class BuildWorkfile: + """Wrapper for build workfile process. + + Load representations for current context by build presets. Build presets + are host related, since each host has it's loaders. + """ + + log = logging.getLogger("BuildWorkfile") + + @staticmethod + def map_subsets_by_family(subsets): + subsets_by_family = collections.defaultdict(list) + for subset in subsets: + family = subset["data"].get("family") + if not family: + families = subset["data"].get("families") + if not families: + continue + family = families[0] + + subsets_by_family[family].append(subset) + return subsets_by_family + + def process(self): + """Main method of this wrapper. + + Building of workfile is triggered and is possible to implement + post processing of loaded containers if necessary. + """ + containers = self.build_workfile() + + return containers + + def build_workfile(self): + """Prepares and load containers into workfile. + + Loads latest versions of current and linked assets to workfile by logic + stored in Workfile profiles from presets. Profiles are set by host, + filtered by current task name and used by families. + + Each family can specify representation names and loaders for + representations and first available and successful loaded + representation is returned as container. + + At the end you'll get list of loaded containers per each asset. + + loaded_containers [{ + "asset_entity": , + "containers": [, , ...] + }, { + "asset_entity": , + "containers": [, ...] + }, { + ... + }] + """ + # Get current asset name and entity + current_asset_name = io.Session["AVALON_ASSET"] + current_asset_entity = io.find_one({ + "type": "asset", + "name": current_asset_name + }) + + # Skip if asset was not found + if not current_asset_entity: + print("Asset entity with name `{}` was not found".format( + current_asset_name + )) + return + + # Prepare available loaders + loaders_by_name = {} + for loader in avalon.api.discover(avalon.api.Loader): + loader_name = loader.__name__ + if loader_name in loaders_by_name: + raise KeyError( + "Duplicated loader name {0}!".format(loader_name) + ) + loaders_by_name[loader_name] = loader + + # Skip if there are any loaders + if not loaders_by_name: + self.log.warning("There are no registered loaders.") + return + + # Get current task name + current_task_name = io.Session["AVALON_TASK"] + + # Load workfile presets for task + self.build_presets = self.get_build_presets(current_task_name) + + # Skip if there are any presets for task + if not self.build_presets: + self.log.warning( + "Current task `{}` does not have any loading preset.".format( + current_task_name + ) + ) + return + + # Get presets for loading current asset + current_context_profiles = self.build_presets.get("current_context") + # Get presets for loading linked assets + link_context_profiles = self.build_presets.get("linked_assets") + # Skip if both are missing + if not current_context_profiles and not link_context_profiles: + self.log.warning( + "Current task `{}` has empty loading preset.".format( + current_task_name + ) + ) + return + + elif not current_context_profiles: + self.log.warning(( + "Current task `{}` doesn't have any loading" + " preset for it's context." + ).format(current_task_name)) + + elif not link_context_profiles: + self.log.warning(( + "Current task `{}` doesn't have any" + "loading preset for it's linked assets." + ).format(current_task_name)) + + # Prepare assets to process by workfile presets + assets = [] + current_asset_id = None + if current_context_profiles: + # Add current asset entity if preset has current context set + assets.append(current_asset_entity) + current_asset_id = current_asset_entity["_id"] + + if link_context_profiles: + # Find and append linked assets if preset has set linked mapping + link_assets = get_linked_assets(current_asset_entity) + if link_assets: + assets.extend(link_assets) + + # Skip if there are no assets. This can happen if only linked mapping + # is set and there are no links for his asset. + if not assets: + self.log.warning( + "Asset does not have linked assets. Nothing to process." + ) + return + + # Prepare entities from database for assets + prepared_entities = self._collect_last_version_repres(assets) + + # Load containers by prepared entities and presets + loaded_containers = [] + # - Current asset containers + if current_asset_id and current_asset_id in prepared_entities: + current_context_data = prepared_entities.pop(current_asset_id) + loaded_data = self.load_containers_by_asset_data( + current_context_data, current_context_profiles, loaders_by_name + ) + if loaded_data: + loaded_containers.append(loaded_data) + + # - Linked assets container + for linked_asset_data in prepared_entities.values(): + loaded_data = self.load_containers_by_asset_data( + linked_asset_data, link_context_profiles, loaders_by_name + ) + if loaded_data: + loaded_containers.append(loaded_data) + + # Return list of loaded containers + return loaded_containers + + def get_build_presets(self, task_name): + """ Returns presets to build workfile for task name. + + Presets are loaded for current project set in + io.Session["AVALON_PROJECT"], filtered by registered host + and entered task name. + + Args: + task_name (str): Task name used for filtering build presets. + + Returns: + (dict): preset per entered task name + """ + host_name = avalon.api.registered_host().__name__.rsplit(".", 1)[-1] + presets = config.get_presets(io.Session["AVALON_PROJECT"]) + # Get presets for host + build_presets = ( + presets["plugins"] + .get(host_name, {}) + .get("workfile_build") + ) + if not build_presets: + return + + task_name_low = task_name.lower() + per_task_preset = None + for preset in build_presets: + preset_tasks = preset.get("tasks") or [] + preset_tasks_low = [task.lower() for task in preset_tasks] + if task_name_low in preset_tasks_low: + per_task_preset = preset + break + + return per_task_preset + + def _filter_build_profiles(self, build_profiles, loaders_by_name): + """ Filter build profiles by loaders and prepare process data. + + Valid profile must have "loaders", "families" and "repre_names" keys + with valid values. + - "loaders" expects list of strings representing possible loaders. + - "families" expects list of strings for filtering + by main subset family. + - "repre_names" expects list of strings for filtering by + representation name. + + Lowered "families" and "repre_names" are prepared for each profile with + all required keys. + + Args: + build_profiles (dict): Profiles for building workfile. + loaders_by_name (dict): Available loaders per name. + + Returns: + (list): Filtered and prepared profiles. + """ + valid_profiles = [] + for profile in build_profiles: + # Check loaders + profile_loaders = profile.get("loaders") + if not profile_loaders: + self.log.warning(( + "Build profile has missing loaders configuration: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check if any loader is available + loaders_match = False + for loader_name in profile_loaders: + if loader_name in loaders_by_name: + loaders_match = True + break + + if not loaders_match: + self.log.warning(( + "All loaders from Build profile are not available: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check families + profile_families = profile.get("families") + if not profile_families: + self.log.warning(( + "Build profile is missing families configuration: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Check representation names + profile_repre_names = profile.get("repre_names") + if not profile_repre_names: + self.log.warning(( + "Build profile is missing" + " representation names filtering: {0}" + ).format(json.dumps(profile, indent=4))) + continue + + # Prepare lowered families and representation names + profile["families_lowered"] = [ + fam.lower() for fam in profile_families + ] + profile["repre_names_lowered"] = [ + name.lower() for name in profile_repre_names + ] + + valid_profiles.append(profile) + + return valid_profiles + + def _prepare_profile_for_subsets(self, subsets, profiles): + """Select profile for each subset byt it's data. + + Profiles are filtered for each subset individually. + Profile is filtered by subset's family, optionally by name regex and + representation names set in profile. + It is possible to not find matching profile for subset, in that case + subset is skipped and it is possible that none of subsets have + matching profile. + + Args: + subsets (list): Subset documents. + profiles (dict): Build profiles. + + Returns: + (dict) Profile by subset's id. + """ + # Prepare subsets + subsets_by_family = self.map_subsets_by_family(subsets) + + profiles_per_subset_id = {} + for family, subsets in subsets_by_family.items(): + family_low = family.lower() + for profile in profiles: + # Skip profile if does not contain family + if family_low not in profile["families_lowered"]: + continue + + # Precompile name filters as regexes + profile_regexes = profile.get("subset_name_filters") + if profile_regexes: + _profile_regexes = [] + for regex in profile_regexes: + _profile_regexes.append(re.compile(regex)) + profile_regexes = _profile_regexes + + # TODO prepare regex compilation + for subset in subsets: + # Verify regex filtering (optional) + if profile_regexes: + valid = False + for pattern in profile_regexes: + if re.match(pattern, subset["name"]): + valid = True + break + + if not valid: + continue + + profiles_per_subset_id[subset["_id"]] = profile + + # break profiles loop on finding the first matching profile + break + return profiles_per_subset_id + + def load_containers_by_asset_data( + self, asset_entity_data, build_profiles, loaders_by_name + ): + """Load containers for entered asset entity by Build profiles. + + Args: + asset_entity_data (dict): Prepared data with subsets, last version + and representations for specific asset. + build_profiles (dict): Build profiles. + loaders_by_name (dict): Available loaders per name. + + Returns: + (dict) Output contains asset document and loaded containers. + """ + + # Make sure all data are not empty + if not asset_entity_data or not build_profiles or not loaders_by_name: + return + + asset_entity = asset_entity_data["asset_entity"] + + valid_profiles = self._filter_build_profiles( + build_profiles, loaders_by_name + ) + if not valid_profiles: + self.log.warning( + "There are not valid Workfile profiles. Skipping process." + ) + return + + self.log.debug("Valid Workfile profiles: {}".format(valid_profiles)) + + subsets_by_id = {} + version_by_subset_id = {} + repres_by_version_id = {} + for subset_id, in_data in asset_entity_data["subsets"].items(): + subset_entity = in_data["subset_entity"] + subsets_by_id[subset_entity["_id"]] = subset_entity + + version_data = in_data["version"] + version_entity = version_data["version_entity"] + version_by_subset_id[subset_id] = version_entity + repres_by_version_id[version_entity["_id"]] = ( + version_data["repres"] + ) + + if not subsets_by_id: + self.log.warning("There are not subsets for asset {0}".format( + asset_entity["name"] + )) + return + + profiles_per_subset_id = self._prepare_profile_for_subsets( + subsets_by_id.values(), valid_profiles + ) + if not profiles_per_subset_id: + self.log.warning("There are not valid subsets.") + return + + valid_repres_by_subset_id = collections.defaultdict(list) + for subset_id, profile in profiles_per_subset_id.items(): + profile_repre_names = profile["repre_names_lowered"] + + version_entity = version_by_subset_id[subset_id] + version_id = version_entity["_id"] + repres = repres_by_version_id[version_id] + for repre in repres: + repre_name_low = repre["name"].lower() + if repre_name_low in profile_repre_names: + valid_repres_by_subset_id[subset_id].append(repre) + + # DEBUG message + msg = "Valid representations for Asset: `{}`".format( + asset_entity["name"] + ) + for subset_id, repres in valid_repres_by_subset_id.items(): + subset = subsets_by_id[subset_id] + msg += "\n# Subset Name/ID: `{}`/{}".format( + subset["name"], subset_id + ) + for repre in repres: + msg += "\n## Repre name: `{}`".format(repre["name"]) + + self.log.debug(msg) + + containers = self._load_containers( + valid_repres_by_subset_id, subsets_by_id, + profiles_per_subset_id, loaders_by_name + ) + + return { + "asset_entity": asset_entity, + "containers": containers + } + + def _load_containers( + self, repres_by_subset_id, subsets_by_id, + profiles_per_subset_id, loaders_by_name + ): + """Real load by collected data happens here. + + Loading of representations per subset happens here. Each subset can + loads one representation. Loading is tried in specific order. + Representations are tried to load by names defined in configuration. + If subset has representation matching representation name each loader + is tried to load it until any is successful. If none of them was + successful then next reprensentation name is tried. + Subset process loop ends when any representation is loaded or + all matching representations were already tried. + + Args: + repres_by_subset_id (dict): Available representations mapped + by their parent (subset) id. + subsets_by_id (dict): Subset documents mapped by their id. + profiles_per_subset_id (dict): Build profiles mapped by subset id. + loaders_by_name (dict): Available loaders per name. + + Returns: + (list) Objects of loaded containers. + """ + loaded_containers = [] + + # Get subset id order from build presets. + build_presets = self.build_presets.get("current_context", []) + build_presets += self.build_presets.get("linked_assets", []) + subset_ids_ordered = [] + for preset in build_presets: + for preset_family in preset["families"]: + for id, subset in subsets_by_id.items(): + if preset_family not in subset["data"].get("families", []): + continue + + subset_ids_ordered.append(id) + + # Order representations from subsets. + print("repres_by_subset_id", repres_by_subset_id) + representations_ordered = [] + representations = [] + for id in subset_ids_ordered: + for subset_id, repres in repres_by_subset_id.items(): + if repres in representations: + continue + + if id == subset_id: + representations_ordered.append((subset_id, repres)) + representations.append(repres) + + print("representations", representations) + + # Load ordered reprensentations. + for subset_id, repres in representations_ordered: + subset_name = subsets_by_id[subset_id]["name"] + + profile = profiles_per_subset_id[subset_id] + loaders_last_idx = len(profile["loaders"]) - 1 + repre_names_last_idx = len(profile["repre_names_lowered"]) - 1 + + repre_by_low_name = { + repre["name"].lower(): repre for repre in repres + } + + is_loaded = False + for repre_name_idx, profile_repre_name in enumerate( + profile["repre_names_lowered"] + ): + # Break iteration if representation was already loaded + if is_loaded: + break + + repre = repre_by_low_name.get(profile_repre_name) + if not repre: + continue + + for loader_idx, loader_name in enumerate(profile["loaders"]): + if is_loaded: + break + + loader = loaders_by_name.get(loader_name) + if not loader: + continue + try: + container = avalon.api.load( + loader, + repre["_id"], + name=subset_name + ) + loaded_containers.append(container) + is_loaded = True + + except Exception as exc: + if exc == pipeline.IncompatibleLoaderError: + self.log.info(( + "Loader `{}` is not compatible with" + " representation `{}`" + ).format(loader_name, repre["name"])) + + else: + self.log.error( + "Unexpected error happened during loading", + exc_info=True + ) + + msg = "Loading failed." + if loader_idx < loaders_last_idx: + msg += " Trying next loader." + elif repre_name_idx < repre_names_last_idx: + msg += ( + " Loading of subset `{}` was not successful." + ).format(subset_name) + else: + msg += " Trying next representation." + self.log.info(msg) + + return loaded_containers + + def _collect_last_version_repres(self, asset_entities): + """Collect subsets, versions and representations for asset_entities. + + Args: + asset_entities (list): Asset entities for which want to find data + + Returns: + (dict): collected entities + + Example output: + ``` + { + {Asset ID}: { + "asset_entity": , + "subsets": { + {Subset ID}: { + "subset_entity": , + "version": { + "version_entity": , + "repres": [ + , , ... + ] + } + }, + ... + } + }, + ... + } + output[asset_id]["subsets"][subset_id]["version"]["repres"] + ``` + """ + + if not asset_entities: + return {} + + asset_entity_by_ids = {asset["_id"]: asset for asset in asset_entities} + + subsets = list(io.find({ + "type": "subset", + "parent": {"$in": asset_entity_by_ids.keys()} + })) + subset_entity_by_ids = {subset["_id"]: subset for subset in subsets} + + sorted_versions = list(io.find({ + "type": "version", + "parent": {"$in": subset_entity_by_ids.keys()} + }).sort("name", -1)) + + subset_id_with_latest_version = [] + last_versions_by_id = {} + for version in sorted_versions: + subset_id = version["parent"] + if subset_id in subset_id_with_latest_version: + continue + subset_id_with_latest_version.append(subset_id) + last_versions_by_id[version["_id"]] = version + + repres = io.find({ + "type": "representation", + "parent": {"$in": last_versions_by_id.keys()} + }) + + output = {} + for repre in repres: + version_id = repre["parent"] + version = last_versions_by_id[version_id] + + subset_id = version["parent"] + subset = subset_entity_by_ids[subset_id] + + asset_id = subset["parent"] + asset = asset_entity_by_ids[asset_id] + + if asset_id not in output: + output[asset_id] = { + "asset_entity": asset, + "subsets": {} + } + + if subset_id not in output[asset_id]["subsets"]: + output[asset_id]["subsets"][subset_id] = { + "subset_entity": subset, + "version": { + "version_entity": version, + "repres": [] + } + } + + output[asset_id]["subsets"][subset_id]["version"]["repres"].append( + repre + ) + + return output diff --git a/pype/lib/deprecated.py b/pype/lib/deprecated.py new file mode 100644 index 0000000000..e7296f67ef --- /dev/null +++ b/pype/lib/deprecated.py @@ -0,0 +1,26 @@ +import os + +from avalon import io + + +def get_avalon_database(): + """Mongo database used in avalon's io. + + * Function is not used in pype 3.0 where was replaced with usage of + AvalonMongoDB. + """ + if io._database is None: + set_io_database() + return io._database + + +def set_io_database(): + """Set avalon's io context with environemnts. + + * Function is not used in pype 3.0 where was replaced with usage of + AvalonMongoDB. + """ + required_keys = ["AVALON_PROJECT", "AVALON_ASSET", "AVALON_SILO"] + for key in required_keys: + os.environ[key] = os.environ.get(key, "") + io.install() diff --git a/pype/lib/ffmpeg_utils.py b/pype/lib/ffmpeg_utils.py new file mode 100644 index 0000000000..0096fd13aa --- /dev/null +++ b/pype/lib/ffmpeg_utils.py @@ -0,0 +1,46 @@ +import logging +import json +import subprocess + +from . import get_ffmpeg_tool_path + +log = logging.getLogger("FFmpeg utils") + + +def ffprobe_streams(path_to_file, logger=None): + """Load streams from entered filepath via ffprobe. + + Args: + path_to_file (str): absolute path + logger (logging.getLogger): injected logger, if empty new is created + + """ + if not logger: + logger = log + logger.info( + "Getting information about input \"{}\".".format(path_to_file) + ) + args = [ + "\"{}\"".format(get_ffmpeg_tool_path("ffprobe")), + "-v quiet", + "-print_format json", + "-show_format", + "-show_streams", + "\"{}\"".format(path_to_file) + ] + command = " ".join(args) + logger.debug("FFprobe command: \"{}\"".format(command)) + popen = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + popen_stdout, popen_stderr = popen.communicate() + if popen_stdout: + logger.debug("ffprobe stdout: {}".format(popen_stdout)) + + if popen_stderr: + logger.debug("ffprobe stderr: {}".format(popen_stderr)) + return json.loads(popen_stdout)["streams"] diff --git a/pype/lib/hooks.py b/pype/lib/hooks.py new file mode 100644 index 0000000000..425ad36342 --- /dev/null +++ b/pype/lib/hooks.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +"""Package containing code for handling hooks.""" +import os +import sys +import types +import logging +from abc import ABCMeta, abstractmethod + +import six + + +log = logging.getLogger(__name__) + + +@six.add_metaclass(ABCMeta) +class PypeHook: + """Abstract class from all hooks should inherit.""" + + def __init__(self): + """Constructor.""" + pass + + @abstractmethod + def execute(self, *args, **kwargs): + """Abstract execute method.""" + pass + + +def execute_hook(hook, *args, **kwargs): + """Execute hook with arguments. + + This will load hook file, instantiate class and call + :meth:`PypeHook.execute` method on it. Hook must be in a form:: + + $PYPE_SETUP_PATH/repos/pype/path/to/hook.py/HookClass + + This will load `hook.py`, instantiate HookClass and then execute_hook + `execute(*args, **kwargs)` + + Args: + hook (str): path to hook class. + + """ + class_name = hook.split("/")[-1] + + abspath = os.path.join(os.getenv('PYPE_SETUP_PATH'), + 'repos', 'pype', *hook.split("/")[:-1]) + + mod_name, mod_ext = os.path.splitext(os.path.basename(abspath)) + + if not mod_ext == ".py": + return False + + module = types.ModuleType(mod_name) + module.__file__ = abspath + + try: + with open(abspath) as f: + six.exec_(f.read(), module.__dict__) + + sys.modules[abspath] = module + + except Exception as exp: + log.exception("loading hook failed: {}".format(exp), + exc_info=True) + return False + + obj = getattr(module, class_name) + hook_obj = obj() + ret_val = hook_obj.execute(*args, **kwargs) + return ret_val diff --git a/pype/lib/path_tools.py b/pype/lib/path_tools.py new file mode 100644 index 0000000000..ba383ea3ed --- /dev/null +++ b/pype/lib/path_tools.py @@ -0,0 +1,181 @@ +import os +import re +import logging + +log = logging.getLogger(__name__) + + +def get_paths_from_environ(env_key, return_first=False): + """Return existing paths from specific envirnment variable. + + Args: + env_key (str): Environment key where should look for paths. + + Returns: + (bool): Return first path on `True`, list of all on `False`. + + + Difference when none of paths exists: + - when `return_first` is set to `False` then function returns empty list. + - when `return_first` is set to `True` then function returns `None`. + """ + existing_paths = [] + paths = os.environ.get(env_key) or "" + path_items = paths.split(os.pathsep) + for path in path_items: + # Skip empty string + if not path: + continue + # Normalize path + path = os.path.normpath(path) + # Check if path exists + if os.path.exists(path): + # Return path if `return_first` is set to True + if return_first: + return path + # Store path + existing_paths.append(path) + + # Return None if none of paths exists + if return_first: + return None + # Return all existing paths from environment variable + return existing_paths + + +def get_ffmpeg_tool_path(tool="ffmpeg"): + """Find path to ffmpeg tool in FFMPEG_PATH paths. + + Function looks for tool in paths set in FFMPEG_PATH environment. If tool + exists then returns it's full path. + + Args: + tool (string): tool name + + Returns: + (str): tool name itself when tool path was not found. (FFmpeg path + may be set in PATH environment variable) + """ + dir_paths = get_paths_from_environ("FFMPEG_PATH") + for dir_path in dir_paths: + for file_name in os.listdir(dir_path): + base, _ext = os.path.splitext(file_name) + if base.lower() == tool.lower(): + return os.path.join(dir_path, tool) + return tool + + +def _rreplace(s, a, b, n=1): + """Replace a with b in string s from right side n times.""" + return b.join(s.rsplit(a, n)) + + +def version_up(filepath): + """Version up filepath to a new non-existing version. + + Parses for a version identifier like `_v001` or `.v001` + When no version present _v001 is appended as suffix. + + Args: + filepath (str): full url + + Returns: + (str): filepath with increased version number + + """ + dirname = os.path.dirname(filepath) + basename, ext = os.path.splitext(os.path.basename(filepath)) + + regex = r"[._]v\d+" + matches = re.findall(regex, str(basename), re.IGNORECASE) + if not matches: + log.info("Creating version...") + new_label = "_v{version:03d}".format(version=1) + new_basename = "{}{}".format(basename, new_label) + else: + label = matches[-1] + version = re.search(r"\d+", label).group() + padding = len(version) + + new_version = int(version) + 1 + new_version = '{version:0{padding}d}'.format(version=new_version, + padding=padding) + new_label = label.replace(version, new_version, 1) + new_basename = _rreplace(basename, label, new_label) + + if not new_basename.endswith(new_label): + index = (new_basename.find(new_label)) + index += len(new_label) + new_basename = new_basename[:index] + + new_filename = "{}{}".format(new_basename, ext) + new_filename = os.path.join(dirname, new_filename) + new_filename = os.path.normpath(new_filename) + + if new_filename == filepath: + raise RuntimeError("Created path is the same as current file," + "this is a bug") + + for file in os.listdir(dirname): + if file.endswith(ext) and file.startswith(new_basename): + log.info("Skipping existing version %s" % new_label) + return version_up(new_filename) + + log.info("New version %s" % new_label) + return new_filename + + +def get_version_from_path(file): + """Find version number in file path string. + + Args: + file (string): file path + + Returns: + v: version number in string ('001') + + """ + pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) + try: + return pattern.findall(file)[0] + except IndexError: + log.error( + "templates:get_version_from_workfile:" + "`{}` missing version string." + "Example `v004`".format(file) + ) + + +def get_last_version_from_path(path_dir, filter): + """Find last version of given directory content. + + Args: + path_dir (string): directory path + filter (list): list of strings used as file name filter + + Returns: + string: file name with last version + + Example: + last_version_file = get_last_version_from_path( + "/project/shots/shot01/work", ["shot01", "compositing", "nk"]) + """ + assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory" + assert isinstance(filter, list) and ( + len(filter) != 0), "`filter` argument needs to be list and not empty" + + filtred_files = list() + + # form regex for filtering + patern = r".*".join(filter) + + for file in os.listdir(path_dir): + if not re.findall(patern, file): + continue + filtred_files.append(file) + + if filtred_files: + sorted(filtred_files) + return filtred_files[-1] + + return None diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py new file mode 100644 index 0000000000..0b6ace807e --- /dev/null +++ b/pype/lib/plugin_tools.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +"""Avalon/Pyblish plugin tools.""" +import os +import inspect +import logging + +from ..api import config + + +log = logging.getLogger(__name__) + + +def filter_pyblish_plugins(plugins): + """Filter pyblish plugins by presets. + + This servers as plugin filter / modifier for pyblish. It will load plugin + definitions from presets and filter those needed to be excluded. + + Args: + plugins (dict): Dictionary of plugins produced by :mod:`pyblish-base` + `discover()` method. + + """ + from pyblish import api + + host = api.current_host() + + presets = config.get_presets().get('plugins', {}) + + # iterate over plugins + for plugin in plugins[:]: + # skip if there are no presets to process + if not presets: + continue + + file = os.path.normpath(inspect.getsourcefile(plugin)) + file = os.path.normpath(file) + + # host determined from path + host_from_file = file.split(os.path.sep)[-3:-2][0] + plugin_kind = file.split(os.path.sep)[-2:-1][0] + + try: + config_data = presets[host]["publish"][plugin.__name__] + except KeyError: + try: + config_data = presets[host_from_file][plugin_kind][plugin.__name__] # noqa: E501 + except KeyError: + continue + + for option, value in config_data.items(): + if option == "enabled" and value is False: + log.info('removing plugin {}'.format(plugin.__name__)) + plugins.remove(plugin) + else: + log.info('setting {}:{} on plugin {}'.format( + option, value, plugin.__name__)) + + setattr(plugin, option, value) + + +def source_hash(filepath, *args): + """Generate simple identifier for a source file. + This is used to identify whether a source file has previously been + processe into the pipeline, e.g. a texture. + The hash is based on source filepath, modification time and file size. + This is only used to identify whether a specific source file was already + published before from the same location with the same modification date. + We opt to do it this way as opposed to Avalanch C4 hash as this is much + faster and predictable enough for all our production use cases. + Args: + filepath (str): The source file path. + You can specify additional arguments in the function + to allow for specific 'processing' values to be included. + """ + # We replace dots with comma because . cannot be a key in a pymongo dict. + file_name = os.path.basename(filepath) + time = str(os.path.getmtime(filepath)) + size = str(os.path.getsize(filepath)) + return "|".join([file_name, time, size] + list(args)).replace(".", ",") diff --git a/pype/modules/ftrack/events/event_push_frame_values_to_task.py b/pype/modules/ftrack/events/event_push_frame_values_to_task.py index 32993ef938..00457c8bfc 100644 --- a/pype/modules/ftrack/events/event_push_frame_values_to_task.py +++ b/pype/modules/ftrack/events/event_push_frame_values_to_task.py @@ -1,4 +1,6 @@ import collections +import datetime + import ftrack_api from pype.modules.ftrack import BaseEvent @@ -10,17 +12,24 @@ class PushFrameValuesToTaskEvent(BaseEvent): cust_attrs_query = ( "select id, key, object_type_id, is_hierarchical, default" " from CustomAttributeConfiguration" - " where key in ({}) and object_type_id in ({})" + " where key in ({}) and" + " (object_type_id in ({}) or is_hierarchical is true)" + ) + + cust_attr_query = ( + "select value, entity_id from ContextCustomAttributeValue " + "where entity_id in ({}) and configuration_id in ({})" ) - interest_entity_types = {"Shot"} - interest_attributes = {"frameStart", "frameEnd"} - interest_attr_mapping = { - "frameStart": "fstart", - "frameEnd": "fend" - } _cached_task_object_id = None _cached_interest_object_ids = None + _cached_user_id = None + _cached_changes = [] + _max_delta = 30 + + # Configrable (lists) + interest_entity_types = {"Shot"} + interest_attributes = {"frameStart", "frameEnd"} @staticmethod def join_keys(keys): @@ -49,8 +58,18 @@ class PushFrameValuesToTaskEvent(BaseEvent): ) return cls._cached_interest_object_ids + def session_user_id(self, session): + if self._cached_user_id is None: + user = session.query( + "User where username is \"{}\"".format(session.api_user) + ).one() + self._cached_user_id = user["id"] + return self._cached_user_id + def launch(self, session, event): - interesting_data = self.extract_interesting_data(session, event) + interesting_data, changed_keys_by_object_id = ( + self.extract_interesting_data(session, event) + ) if not interesting_data: return @@ -66,92 +85,165 @@ class PushFrameValuesToTaskEvent(BaseEvent): if entity_id not in entities_by_id: interesting_data.pop(entity_id) - task_entities = self.get_task_entities(session, interesting_data) + attrs_by_obj_id, hier_attrs = self.attrs_configurations(session) + + task_object_id = self.task_object_id(session) + task_attrs = attrs_by_obj_id.get(task_object_id) + # Skip keys that are not both in hierachical and type specific + for object_id, keys in changed_keys_by_object_id.items(): + object_id_attrs = attrs_by_obj_id.get(object_id) + for key in keys: + if key not in hier_attrs: + attrs_by_obj_id[object_id].pop(key) + continue + + if ( + (not object_id_attrs or key not in object_id_attrs) + and (not task_attrs or key not in task_attrs) + ): + hier_attrs.pop(key) + + # Clean up empty values + for key, value in tuple(attrs_by_obj_id.items()): + if not value: + attrs_by_obj_id.pop(key) - attrs_by_obj_id = self.attrs_configurations(session) if not attrs_by_obj_id: self.log.warning(( - "There is not created Custom Attributes {}" - " for \"Task\" entity type." - ).format(self.join_keys(self.interest_attributes))) + "There is not created Custom Attributes {} " + " for entity types: {}" + ).format( + self.join_keys(self.interest_attributes), + self.join_keys(self.interest_entity_types) + )) return - task_entities_by_parent_id = collections.defaultdict(list) + # Prepare task entities + task_entities = [] + # If task entity does not contain changed attribute then skip + if task_attrs: + task_entities = self.get_task_entities(session, interesting_data) + + task_entities_by_id = {} + parent_id_by_task_id = {} for task_entity in task_entities: - task_entities_by_parent_id[task_entity["parent_id"]].append( - task_entity - ) + task_entities_by_id[task_entity["id"]] = task_entity + parent_id_by_task_id[task_entity["id"]] = task_entity["parent_id"] - missing_keys_by_object_name = collections.defaultdict(set) - for parent_id, values in interesting_data.items(): - entities = task_entities_by_parent_id.get(parent_id) or [] - entities.append(entities_by_id[parent_id]) + changed_keys = set() + for keys in changed_keys_by_object_id.values(): + changed_keys |= set(keys) - for hier_key, value in values.items(): - changed_ids = [] - for entity in entities: - key = self.interest_attr_mapping[hier_key] - entity_attrs_mapping = ( - attrs_by_obj_id.get(entity["object_type_id"]) + attr_id_to_key = {} + for attr_confs in attrs_by_obj_id.values(): + for key in changed_keys: + custom_attr_id = attr_confs.get(key) + if custom_attr_id: + attr_id_to_key[custom_attr_id] = key + + for key in changed_keys: + custom_attr_id = hier_attrs.get(key) + if custom_attr_id: + attr_id_to_key[custom_attr_id] = key + + entity_ids = ( + set(interesting_data.keys()) | set(task_entities_by_id.keys()) + ) + attr_ids = set(attr_id_to_key.keys()) + + current_values_by_id = self.current_values( + session, attr_ids, entity_ids, task_entities_by_id, hier_attrs + ) + + for entity_id, current_values in current_values_by_id.items(): + parent_id = parent_id_by_task_id.get(entity_id) + if not parent_id: + parent_id = entity_id + values = interesting_data[parent_id] + + for attr_id, old_value in current_values.items(): + attr_key = attr_id_to_key.get(attr_id) + if not attr_key: + continue + + # Convert new value from string + new_value = values.get(attr_key) + if new_value is not None and old_value is not None: + try: + new_value = type(old_value)(new_value) + except Exception: + self.log.warning(( + "Couldn't convert from {} to {}." + " Skipping update values." + ).format(type(new_value), type(old_value))) + if new_value == old_value: + continue + + entity_key = collections.OrderedDict({ + "configuration_id": attr_id, + "entity_id": entity_id + }) + self._cached_changes.append({ + "attr_key": attr_key, + "entity_id": entity_id, + "value": new_value, + "time": datetime.datetime.now() + }) + if new_value is None: + op = ftrack_api.operation.DeleteEntityOperation( + "CustomAttributeValue", + entity_key + ) + else: + op = ftrack_api.operation.UpdateEntityOperation( + "ContextCustomAttributeValue", + entity_key, + "value", + ftrack_api.symbol.NOT_SET, + new_value ) - if not entity_attrs_mapping: - missing_keys_by_object_name[entity.entity_type].add( - key - ) - continue - configuration_id = entity_attrs_mapping.get(key) - if not configuration_id: - missing_keys_by_object_name[entity.entity_type].add( - key - ) - continue - - changed_ids.append(entity["id"]) - entity_key = collections.OrderedDict({ - "configuration_id": configuration_id, - "entity_id": entity["id"] - }) - if value is None: - op = ftrack_api.operation.DeleteEntityOperation( - "CustomAttributeValue", - entity_key - ) - else: - op = ftrack_api.operation.UpdateEntityOperation( - "ContextCustomAttributeValue", - entity_key, - "value", - ftrack_api.symbol.NOT_SET, - value - ) - - session.recorded_operations.push(op) + session.recorded_operations.push(op) self.log.info(( "Changing Custom Attribute \"{}\" to value" - " \"{}\" on entities: {}" - ).format(key, value, self.join_keys(changed_ids))) - try: - session.commit() - except Exception: - session.rollback() - self.log.warning( - "Changing of values failed.", - exc_info=True - ) - if not missing_keys_by_object_name: - return + " \"{}\" on entity: {}" + ).format(attr_key, new_value, entity_id)) + try: + session.commit() + except Exception: + session.rollback() + self.log.warning("Changing of values failed.", exc_info=True) - msg_items = [] - for object_name, missing_keys in missing_keys_by_object_name.items(): - msg_items.append( - "{}: ({})".format(object_name, self.join_keys(missing_keys)) + def current_values( + self, session, attr_ids, entity_ids, task_entities_by_id, hier_attrs + ): + current_values_by_id = {} + if not attr_ids or not entity_ids: + return current_values_by_id + joined_conf_ids = self.join_keys(attr_ids) + joined_entity_ids = self.join_keys(entity_ids) + + call_expr = [{ + "action": "query", + "expression": self.cust_attr_query.format( + joined_entity_ids, joined_conf_ids ) + }] + if hasattr(session, "call"): + [values] = session.call(call_expr) + else: + [values] = session._call(call_expr) - self.log.warning(( - "Missing Custom Attribute configuration" - " per specific object types: {}" - ).format(", ".join(msg_items))) + for item in values["data"]: + entity_id = item["entity_id"] + attr_id = item["configuration_id"] + if entity_id in task_entities_by_id and attr_id in hier_attrs: + continue + + if entity_id not in current_values_by_id: + current_values_by_id[entity_id] = {} + current_values_by_id[entity_id][attr_id] = item["value"] + return current_values_by_id def extract_interesting_data(self, session, event): # Filter if event contain relevant data @@ -159,7 +251,18 @@ class PushFrameValuesToTaskEvent(BaseEvent): if not entities_info: return + # for key, value in event["data"].items(): + # self.log.info("{}: {}".format(key, value)) + session_user_id = self.session_user_id(session) + user_data = event["data"].get("user") + changed_by_session = False + if user_data and user_data.get("userid") == session_user_id: + changed_by_session = True + + current_time = datetime.datetime.now() + interesting_data = {} + changed_keys_by_object_id = {} for entity_info in entities_info: # Care only about tasks if entity_info.get("entityType") != "task": @@ -176,16 +279,47 @@ class PushFrameValuesToTaskEvent(BaseEvent): if key in changes: entity_changes[key] = changes[key]["new"] + entity_id = entity_info["entityId"] + if changed_by_session: + for key, new_value in tuple(entity_changes.items()): + for cached in tuple(self._cached_changes): + if ( + cached["entity_id"] != entity_id + or cached["attr_key"] != key + ): + continue + + cached_value = cached["value"] + try: + new_value = type(cached_value)(new_value) + except Exception: + pass + + if cached_value == new_value: + self._cached_changes.remove(cached) + entity_changes.pop(key) + break + + delta = (current_time - cached["time"]).seconds + if delta > self._max_delta: + self._cached_changes.remove(cached) + if not entity_changes: continue # Do not care about "Task" entity_type task_object_id = self.task_object_id(session) - if entity_info.get("objectTypeId") == task_object_id: + object_id = entity_info.get("objectTypeId") + if not object_id or object_id == task_object_id: continue - interesting_data[entity_info["entityId"]] = entity_changes - return interesting_data + interesting_data[entity_id] = entity_changes + if object_id not in changed_keys_by_object_id: + changed_keys_by_object_id[object_id] = set() + + changed_keys_by_object_id[object_id] |= set(entity_changes.keys()) + + return interesting_data, changed_keys_by_object_id def get_entities(self, session, interesting_data): entities = session.query( @@ -213,17 +347,21 @@ class PushFrameValuesToTaskEvent(BaseEvent): object_ids.append(self.task_object_id(session)) attrs = session.query(self.cust_attrs_query.format( - self.join_keys(self.interest_attr_mapping.values()), + self.join_keys(self.interest_attributes), self.join_keys(object_ids) )).all() output = {} + hiearchical = {} for attr in attrs: + if attr["is_hierarchical"]: + hiearchical[attr["key"]] = attr["id"] + continue obj_id = attr["object_type_id"] if obj_id not in output: output[obj_id] = {} output[obj_id][attr["key"]] = attr["id"] - return output + return output, hiearchical def register(session, plugins_presets): diff --git a/pype/plugins/celaction/publish/collect_audio.py b/pype/plugins/celaction/publish/collect_audio.py index c29e212d80..bab1203642 100644 --- a/pype/plugins/celaction/publish/collect_audio.py +++ b/pype/plugins/celaction/publish/collect_audio.py @@ -1,7 +1,9 @@ -import pyblish.api import os +import collections + +import pyblish.api +from avalon import io -import pype.api as pype from pprint import pformat @@ -12,12 +14,13 @@ class AppendCelactionAudio(pyblish.api.ContextPlugin): def process(self, context): self.log.info('Collecting Audio Data') - asset_entity = context.data["assetEntity"] + asset_doc = context.data["assetEntity"] # get all available representations - subsets = pype.get_subsets(asset_entity["name"], - representations=["audio", "wav"] - ) + subsets = self.get_subsets( + asset_doc, + representations=["audio", "wav"] + ) self.log.info(f"subsets is: {pformat(subsets)}") if not subsets.get("audioMain"): @@ -39,3 +42,85 @@ class AppendCelactionAudio(pyblish.api.ContextPlugin): 'audio_file: {}, has been added to context'.format(audio_file)) else: self.log.warning("Couldn't find any audio file on Ftrack.") + + def get_subsets(self, asset_doc, representations): + """ + Query subsets with filter on name. + + The method will return all found subsets and its defined version + and subsets. Version could be specified with number. Representation + can be filtered. + + Arguments: + asset_doct (dict): Asset (shot) mongo document + representations (list): list for all representations + + Returns: + dict: subsets with version and representaions in keys + """ + + # Query all subsets for asset + subset_docs = io.find({ + "type": "subset", + "parent": asset_doc["_id"] + }) + # Collect all subset ids + subset_ids = [ + subset_doc["_id"] + for subset_doc in subset_docs + ] + + # Check if we found anything + assert subset_ids, ( + "No subsets found. Check correct filter. " + "Try this for start `r'.*'`: asset: `{}`" + ).format(asset_doc["name"]) + + # Last version aggregation + pipeline = [ + # Find all versions of those subsets + {"$match": { + "type": "version", + "parent": {"$in": subset_ids} + }}, + # Sorting versions all together + {"$sort": {"name": 1}}, + # Group them by "parent", but only take the last + {"$group": { + "_id": "$parent", + "_version_id": {"$last": "$_id"}, + "name": {"$last": "$name"} + }} + ] + last_versions_by_subset_id = dict() + for doc in io.aggregate(pipeline): + doc["parent"] = doc["_id"] + doc["_id"] = doc.pop("_version_id") + last_versions_by_subset_id[doc["parent"]] = doc + + version_docs_by_id = {} + for version_doc in last_versions_by_subset_id.values(): + version_docs_by_id[version_doc["_id"]] = version_doc + + repre_docs = io.find({ + "type": "representation", + "parent": {"$in": list(version_docs_by_id.keys())}, + "name": {"$in": representations} + }) + repre_docs_by_version_id = collections.defaultdict(list) + for repre_doc in repre_docs: + version_id = repre_doc["parent"] + repre_docs_by_version_id[version_id].append(repre_doc) + + output_dict = {} + for version_id, repre_docs in repre_docs_by_version_id.items(): + version_doc = version_docs_by_id[version_id] + subset_id = version_doc["parent"] + subset_doc = last_versions_by_subset_id[subset_id] + # Store queried docs by subset name + output_dict[subset_doc["name"]] = { + "representations": repre_docs, + "version": version_doc + } + + return output_dict diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 6b8a026fc9..5be5060590 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -2,6 +2,7 @@ import os import re import json import copy +import tempfile import pype.api import pyblish @@ -26,7 +27,7 @@ class ExtractBurnin(pype.api.Extractor): "hiero", "premiere", "standalonepublisher", - "harmony" + "harmony", "fusion" ] optional = True @@ -227,12 +228,30 @@ class ExtractBurnin(pype.api.Extractor): # Dump data to string dumped_script_data = json.dumps(script_data) + # Store dumped json to temporary file + temporary_json_file = tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) + temporary_json_file.write(dumped_script_data) + temporary_json_file.close() + temporary_json_filepath = temporary_json_file.name.replace( + "\\", "/" + ) + # Prepare subprocess arguments - args = [executable, scriptpath, dumped_script_data] - self.log.debug("Executing: {}".format(args)) + args = [ + "\"{}\"".format(executable), + "\"{}\"".format(scriptpath), + "\"{}\"".format(temporary_json_filepath) + ] + subprcs_cmd = " ".join(args) + self.log.debug("Executing: {}".format(subprcs_cmd)) # Run burnin script - pype.api.subprocess(args, shell=True, logger=self.log) + pype.api.subprocess(subprcs_cmd, shell=True, logger=self.log) + + # Remove the temporary json + os.remove(temporary_json_filepath) for filepath in temp_data["full_input_paths"]: filepath = filepath.replace("\\", "/") diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 34fe6f5e10..dd28e6fec3 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -30,7 +30,8 @@ class ExtractReview(pyblish.api.InstancePlugin): "premiere", "harmony", "standalonepublisher", - "fusion" + "fusion", + "tvpaint" ] # Supported extensions diff --git a/pype/plugins/global/publish/submit_publish_job.py b/pype/plugins/global/publish/submit_publish_job.py index 097761cf4e..444ad28b6c 100644 --- a/pype/plugins/global/publish/submit_publish_job.py +++ b/pype/plugins/global/publish/submit_publish_job.py @@ -151,6 +151,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): deadline_pool_secondary = "" deadline_group = "" deadline_chunk_size = 1 + deadline_priority = None # regex for finding frame number in string R_FRAME_NUMBER = re.compile(r'.+\.(?P[0-9]+)\..+') @@ -902,7 +903,11 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): render_job["Props"]["User"] = context.data.get( "deadlineUser", getpass.getuser()) # Priority is now not handled at all - render_job["Props"]["Pri"] = instance.data.get("priority") + + if self.deadline_priority: + render_job["Props"]["Pri"] = self.deadline_priority + else: + render_job["Props"]["Pri"] = instance.data.get("priority") render_job["Props"]["Env"] = { "FTRACK_API_USER": os.environ.get("FTRACK_API_USER"), @@ -1033,8 +1038,8 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): anatomy_filled = anatomy.format(template_data) - if "folder" in anatomy.templates["publish"]: - publish_folder = anatomy_filled["publish"]["folder"] + if "folder" in anatomy.templates["render"]: + publish_folder = anatomy_filled["render"]["folder"] else: # solve deprecated situation when `folder` key is not underneath # `publish` anatomy @@ -1044,7 +1049,7 @@ class ProcessSubmittedJobOnFarm(pyblish.api.InstancePlugin): " key underneath `publish` (in global of for project `{}`)." ).format(project_name)) - file_path = anatomy_filled["publish"]["path"] + file_path = anatomy_filled["render"]["path"] # Directory publish_folder = os.path.dirname(file_path) diff --git a/pype/plugins/global/publish/validate_ffmpeg_installed.py b/pype/plugins/global/publish/validate_ffmpeg_installed.py index 2e49650040..2951759fa4 100644 --- a/pype/plugins/global/publish/validate_ffmpeg_installed.py +++ b/pype/plugins/global/publish/validate_ffmpeg_installed.py @@ -29,6 +29,6 @@ class ValidateFFmpegInstalled(pyblish.api.ContextPlugin): def process(self, context): ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") self.log.info("ffmpeg path: `{}`".format(ffmpeg_path)) - if self.is_tool("\"{}\"".format(ffmpeg_path)) is False: + if self.is_tool("{}".format(ffmpeg_path)) is False: self.log.error("ffmpeg not found in PATH") raise RuntimeError('ffmpeg not installed.') diff --git a/pype/plugins/harmony/publish/extract_render.py b/pype/plugins/harmony/publish/extract_render.py index 7fb8ae79d2..c5744d43e2 100644 --- a/pype/plugins/harmony/publish/extract_render.py +++ b/pype/plugins/harmony/publish/extract_render.py @@ -90,7 +90,7 @@ class ExtractRender(pyblish.api.InstancePlugin): thumbnail_path = os.path.join(path, "thumbnail.png") ffmpeg_path = pype.lib.get_ffmpeg_tool_path("ffmpeg") args = [ - "\"{}\"".format(ffmpeg_path), "-y", + "{}".format(ffmpeg_path), "-y", "-i", os.path.join(path, list(collections[0])[0]), "-vf", "scale=300:-1", "-vframes", "1", diff --git a/pype/plugins/launcher/actions/Aport.py b/pype/plugins/launcher/actions/Aport.py deleted file mode 100644 index 0ecd07c49a..0000000000 --- a/pype/plugins/launcher/actions/Aport.py +++ /dev/null @@ -1,61 +0,0 @@ -import os -import acre - -from avalon import api, lib -import pype.api as pype -from pype.aport import lib as aportlib - -log = pype.Logger().get_logger(__name__, "aport") - - -class Aport(api.Action): - - name = "aport" - label = "Aport - Avalon's Server" - icon = "retweet" - order = 996 - - def is_compatible(self, session): - """Return whether the action is compatible with the session""" - if "AVALON_TASK" in session: - return True - return False - - def process(self, session, **kwargs): - """Implement the behavior for when the action is triggered - - Args: - session (dict): environment dictionary - - Returns: - Popen instance of newly spawned process - - """ - - with pype.modified_environ(**session): - # Get executable by name - print(self.name) - app = lib.get_application(self.name) - executable = lib.which(app["executable"]) - - # Run as server - arguments = [] - - tools_env = acre.get_tools([self.name]) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - - if not env.get('AVALON_WORKDIR', None): - os.environ["AVALON_WORKDIR"] = aportlib.get_workdir_template() - - env.update(dict(os.environ)) - - try: - lib.launch( - executable=executable, - args=arguments, - environment=env - ) - except Exception as e: - log.error(e) - return diff --git a/pype/plugins/launcher/actions/unused/PremierePro.py b/pype/plugins/launcher/actions/unused/PremierePro.py deleted file mode 100644 index e460af1451..0000000000 --- a/pype/plugins/launcher/actions/unused/PremierePro.py +++ /dev/null @@ -1,83 +0,0 @@ -import os -import acre - -from avalon import api, lib, io -import pype.api as pype - - -class PremierePro(api.Action): - - name = "premiere_2019" - label = "Premiere Pro" - icon = "premiere_icon" - order = 996 - - def is_compatible(self, session): - """Return whether the action is compatible with the session""" - if "AVALON_TASK" in session: - return True - return False - - def process(self, session, **kwargs): - """Implement the behavior for when the action is triggered - - Args: - session (dict): environment dictionary - - Returns: - Popen instance of newly spawned process - - """ - - with pype.modified_environ(**session): - # Get executable by name - app = lib.get_application(self.name) - executable = lib.which(app["executable"]) - - # Run as server - arguments = [] - - tools_env = acre.get_tools([self.name]) - env = acre.compute(tools_env) - env = acre.merge(env, current_env=dict(os.environ)) - - if not env.get('AVALON_WORKDIR', None): - project_name = env.get("AVALON_PROJECT") - anatomy = pype.Anatomy(project_name) - os.environ['AVALON_PROJECT'] = project_name - io.Session['AVALON_PROJECT'] = project_name - - task_name = os.environ.get( - "AVALON_TASK", io.Session["AVALON_TASK"] - ) - asset_name = os.environ.get( - "AVALON_ASSET", io.Session["AVALON_ASSET"] - ) - application = lib.get_application( - os.environ["AVALON_APP_NAME"] - ) - - project_doc = io.find_one({"type": "project"}) - data = { - "task": task_name, - "asset": asset_name, - "project": { - "name": project_doc["name"], - "code": project_doc["data"].get("code", '') - }, - "hierarchy": pype.get_hierarchy(), - "app": application["application_dir"] - } - anatomy_filled = anatomy.format(data) - workdir = anatomy_filled["work"]["folder"] - - os.environ["AVALON_WORKDIR"] = workdir - - env.update(dict(os.environ)) - - lib.launch( - executable=executable, - args=arguments, - environment=env - ) - return diff --git a/pype/plugins/maya/publish/collect_ftrack_family.py b/pype/plugins/maya/publish/collect_ftrack_family.py index c8896a5c9c..b2b66b1875 100644 --- a/pype/plugins/maya/publish/collect_ftrack_family.py +++ b/pype/plugins/maya/publish/collect_ftrack_family.py @@ -20,7 +20,8 @@ class CollectFtrackFamilies(pyblish.api.InstancePlugin): "model", "animation", "look", - "rig" + "rig", + "camera" ] def process(self, instance): diff --git a/pype/plugins/maya/publish/collect_remove_marked.py b/pype/plugins/maya/publish/collect_remove_marked.py index c842fe4ed0..a45c8e45a7 100644 --- a/pype/plugins/maya/publish/collect_remove_marked.py +++ b/pype/plugins/maya/publish/collect_remove_marked.py @@ -18,7 +18,12 @@ class CollectRemoveMarked(pyblish.api.ContextPlugin): def process(self, context): + self.log.debug(context) # make ftrack publishable + instances_to_remove = [] for instance in context: if instance.data.get('remove'): - context.remove(instance) + instances_to_remove.append(instance) + + for instance in instances_to_remove: + context.remove(instance) diff --git a/pype/plugins/maya/publish/collect_render.py b/pype/plugins/maya/publish/collect_render.py index 2848a55152..3dde3b1592 100644 --- a/pype/plugins/maya/publish/collect_render.py +++ b/pype/plugins/maya/publish/collect_render.py @@ -254,6 +254,10 @@ class CollectMayaRender(pyblish.api.ContextPlugin): if self.sync_workfile_version: data["version"] = context.data["version"] + for instance in context: + if instance.data['family'] == "workfile": + instance.data["version"] = context.data["version"] + # Apply each user defined attribute as data for attr in cmds.listAttr(layer, userDefined=True) or list(): try: diff --git a/pype/plugins/maya/publish/collect_review.py b/pype/plugins/maya/publish/collect_review.py index 886e3b82dd..60183341f9 100644 --- a/pype/plugins/maya/publish/collect_review.py +++ b/pype/plugins/maya/publish/collect_review.py @@ -43,33 +43,38 @@ class CollectReview(pyblish.api.InstancePlugin): i = 0 for inst in instance.context: - self.log.debug('processing {}'.format(inst)) - self.log.debug('processing2 {}'.format(instance.context[i])) + self.log.debug('filtering {}'.format(inst)) data = instance.context[i].data - if inst.name == reviewable_subset[0]: - if data.get('families'): - data['families'].append('review') - else: - data['families'] = ['review'] - self.log.debug('adding review family to {}'.format(reviewable_subset)) - data['review_camera'] = camera - # data["publish"] = False - data['frameStartFtrack'] = instance.data["frameStartHandle"] - data['frameEndFtrack'] = instance.data["frameEndHandle"] - data['frameStartHandle'] = instance.data["frameStartHandle"] - data['frameEndHandle'] = instance.data["frameEndHandle"] - data["frameStart"] = instance.data["frameStart"] - data["frameEnd"] = instance.data["frameEnd"] - data['handles'] = instance.data.get('handles', None) - data['step'] = instance.data['step'] - data['fps'] = instance.data['fps'] - data["isolate"] = instance.data["isolate"] - cmds.setAttr(str(instance) + '.active', 1) - self.log.debug('data {}'.format(instance.context[i].data)) - instance.context[i].data.update(data) - instance.data['remove'] = True - i += 1 + if inst.name != reviewable_subset[0]: + self.log.debug('subset name does not match {}'.format( + reviewable_subset[0])) + i += 1 + continue + + if data.get('families'): + data['families'].append('review') + else: + data['families'] = ['review'] + self.log.debug('adding review family to {}'.format( + reviewable_subset)) + data['review_camera'] = camera + # data["publish"] = False + data['frameStartFtrack'] = instance.data["frameStartHandle"] + data['frameEndFtrack'] = instance.data["frameEndHandle"] + data['frameStartHandle'] = instance.data["frameStartHandle"] + data['frameEndHandle'] = instance.data["frameEndHandle"] + data["frameStart"] = instance.data["frameStart"] + data["frameEnd"] = instance.data["frameEnd"] + data['handles'] = instance.data.get('handles', None) + data['step'] = instance.data['step'] + data['fps'] = instance.data['fps'] + data["isolate"] = instance.data["isolate"] + cmds.setAttr(str(instance) + '.active', 1) + self.log.debug('data {}'.format(instance.context[i].data)) + instance.context[i].data.update(data) + instance.data['remove'] = True + self.log.debug('isntance data {}'.format(instance.data)) else: if self.legacy: instance.data['subset'] = task + 'Review' @@ -82,8 +87,10 @@ class CollectReview(pyblish.api.InstancePlugin): instance.data['subset'] = subset instance.data['review_camera'] = camera - instance.data['frameStartFtrack'] = instance.data["frameStartHandle"] - instance.data['frameEndFtrack'] = instance.data["frameEndHandle"] + instance.data['frameStartFtrack'] = \ + instance.data["frameStartHandle"] + instance.data['frameEndFtrack'] = \ + instance.data["frameEndHandle"] # make ftrack publishable instance.data["families"] = ['ftrack'] diff --git a/pype/plugins/maya/publish/collect_yeti_rig.py b/pype/plugins/maya/publish/collect_yeti_rig.py index 8a7971f3ae..73a84b0073 100644 --- a/pype/plugins/maya/publish/collect_yeti_rig.py +++ b/pype/plugins/maya/publish/collect_yeti_rig.py @@ -6,7 +6,6 @@ from maya import cmds import pyblish.api from pype.hosts.maya import lib -from pype.lib import pairwise SETTINGS = {"renderDensity", @@ -78,7 +77,7 @@ class CollectYetiRig(pyblish.api.InstancePlugin): connections = cmds.ls(connections, long=True) # Ensure long names inputs = [] - for dest, src in pairwise(connections): + for dest, src in lib.pairwise(connections): source_node, source_attr = src.split(".", 1) dest_node, dest_attr = dest.split(".", 1) @@ -119,7 +118,7 @@ class CollectYetiRig(pyblish.api.InstancePlugin): texture_filenames = [] if image_search_paths: - + # TODO: Somehow this uses OS environment path separator, `:` vs `;` # Later on check whether this is pipeline OS cross-compatible. image_search_paths = [p for p in @@ -127,7 +126,7 @@ class CollectYetiRig(pyblish.api.InstancePlugin): # find all ${TOKEN} tokens and replace them with $TOKEN env. variable image_search_paths = self._replace_tokens(image_search_paths) - + # List all related textures texture_filenames = cmds.pgYetiCommand(node, listTextures=True) self.log.info("Found %i texture(s)" % len(texture_filenames)) diff --git a/pype/plugins/maya/publish/extract_camera_alembic.py b/pype/plugins/maya/publish/extract_camera_alembic.py index c61ec5e19e..c0538338dc 100644 --- a/pype/plugins/maya/publish/extract_camera_alembic.py +++ b/pype/plugins/maya/publish/extract_camera_alembic.py @@ -26,7 +26,15 @@ class ExtractCameraAlembic(pype.api.Extractor): # get settings framerange = [instance.data.get("frameStart", 1), instance.data.get("frameEnd", 1)] - handles = instance.data.get("handles", 0) + handle_start = instance.data.get("handleStart", 0) + handle_end = instance.data.get("handleEnd", 0) + + # TODO: deprecated attribute "handles" + + if handle_start is None: + handle_start = instance.data.get("handles", 0) + handle_end = instance.data.get("handles", 0) + step = instance.data.get("step", 1.0) bake_to_worldspace = instance.data("bakeToWorldSpace", True) @@ -55,8 +63,10 @@ class ExtractCameraAlembic(pype.api.Extractor): job_str = ' -selection -dataFormat "ogawa" ' job_str += ' -attrPrefix cb' - job_str += ' -frameRange {0} {1} '.format(framerange[0] - handles, - framerange[1] + handles) + job_str += ' -frameRange {0} {1} '.format(framerange[0] + - handle_start, + framerange[1] + + handle_end) job_str += ' -step {0} '.format(step) if bake_to_worldspace: diff --git a/pype/plugins/maya/publish/extract_camera_mayaScene.py b/pype/plugins/maya/publish/extract_camera_mayaScene.py index 1a0f4694d1..8fce48badf 100644 --- a/pype/plugins/maya/publish/extract_camera_mayaScene.py +++ b/pype/plugins/maya/publish/extract_camera_mayaScene.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- """Extract camera as Maya Scene.""" import os +import itertools from maya import cmds import avalon.maya import pype.api -from pype.lib import grouper from pype.hosts.maya import lib @@ -36,6 +36,17 @@ def massage_ma_file(path): f.close() +def grouper(iterable, n, fillvalue=None): + """Collect data into fixed-length chunks or blocks. + + Examples: + grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx + + """ + args = [iter(iterable)] * n + return itertools.izip_longest(fillvalue=fillvalue, *args) + + def unlock(plug): """Unlocks attribute and disconnects inputs for a plug. @@ -107,7 +118,18 @@ class ExtractCameraMayaScene(pype.api.Extractor): framerange = [instance.data.get("frameStart", 1), instance.data.get("frameEnd", 1)] - handles = instance.data.get("handles", 0) + handle_start = instance.data.get("handleStart", 0) + handle_end = instance.data.get("handleEnd", 0) + + # TODO: deprecated attribute "handles" + + if handle_start is None: + handle_start = instance.data.get("handles", 0) + handle_end = instance.data.get("handles", 0) + + range_with_handles = [framerange[0] - handle_start, + framerange[1] + handle_end] + step = instance.data.get("step", 1.0) bake_to_worldspace = instance.data("bakeToWorldSpace", True) @@ -121,9 +143,6 @@ class ExtractCameraMayaScene(pype.api.Extractor): cameras = cmds.ls(members, leaf=True, shapes=True, long=True, dag=True, type="camera") - range_with_handles = [framerange[0] - handles, - framerange[1] + handles] - # validate required settings assert len(cameras) == 1, "Single camera must be found in extraction" assert isinstance(step, float), "Step must be a float value" diff --git a/pype/plugins/nuke/load/load_sequence.py b/pype/plugins/nuke/load/load_sequence.py index 44b9cb4a34..411aec6ad6 100644 --- a/pype/plugins/nuke/load/load_sequence.py +++ b/pype/plugins/nuke/load/load_sequence.py @@ -41,7 +41,7 @@ def preserve_trim(node): "{}".format(script_start)) -def loader_shift(node, frame, relative=True): +def loader_shift(node, frame, relative=False): """Shift global in time by i preserving duration This moves the loader by i frames preserving global duration. When relative @@ -61,11 +61,12 @@ def loader_shift(node, frame, relative=True): script_start = nuke.root()["first_frame"].value() if relative: + node['frame_mode'].setValue("start at") + node['frame'].setValue(str(script_start)) + else: node['frame_mode'].setValue("start at") node['frame'].setValue(str(frame)) - return int(script_start) - class LoadSequence(api.Loader): """Load image sequence into Nuke""" @@ -73,10 +74,10 @@ class LoadSequence(api.Loader): families = ["render2d", "source", "plate", "render", "prerender", "review"] representations = ["exr", "dpx", "jpg", "jpeg", "png"] - label = "Load sequence" - order = -10 - icon = "code-fork" - color = "orange" + label = "Load Image Sequence" + order = -20 + icon = "file-video-o" + color = "white" def load(self, context, name, namespace, data): from avalon.nuke import ( diff --git a/pype/plugins/photoshop/load/load_image.py b/pype/plugins/photoshop/load/load_image.py index 301e60fbb1..44cc96c96f 100644 --- a/pype/plugins/photoshop/load/load_image.py +++ b/pype/plugins/photoshop/load/load_image.py @@ -11,7 +11,7 @@ class ImageLoader(api.Loader): Stores the imported asset in a container named after the asset. """ - families = ["image"] + families = ["image", "render"] representations = ["*"] def load(self, context, name=None, namespace=None, data=None): diff --git a/pype/plugins/photoshop/publish/extract_review.py b/pype/plugins/photoshop/publish/extract_review.py index 4f2c3c3d23..2753cbf366 100644 --- a/pype/plugins/photoshop/publish/extract_review.py +++ b/pype/plugins/photoshop/publish/extract_review.py @@ -54,7 +54,7 @@ class ExtractReview(pype.api.Extractor): # Generate thumbnail. thumbnail_path = os.path.join(staging_dir, "thumbnail.jpg") args = [ - "\"{}\"".format(ffmpeg_path), "-y", + "{}".format(ffmpeg_path), "-y", "-i", output_image_path, "-vf", "scale=300:-1", "-vframes", "1", diff --git a/pype/plugins/standalonepublisher/publish/collect_instance_data.py b/pype/plugins/standalonepublisher/publish/collect_instance_data.py index 58b81324f5..b594843812 100644 --- a/pype/plugins/standalonepublisher/publish/collect_instance_data.py +++ b/pype/plugins/standalonepublisher/publish/collect_instance_data.py @@ -18,7 +18,7 @@ class CollectInstanceData(pyblish.api.InstancePlugin): label = "Collect instance data" order = pyblish.api.CollectorOrder + 0.49 - families = ["render", "plate"] + families = ["render", "plate", "review"] hosts = ["standalonepublisher"] def process(self, instance): diff --git a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py index 249c3fddbe..533421e46d 100644 --- a/pype/plugins/standalonepublisher/publish/extract_thumbnail.py +++ b/pype/plugins/standalonepublisher/publish/extract_thumbnail.py @@ -57,8 +57,9 @@ class ExtractThumbnailSP(pyblish.api.InstancePlugin): elif is_jpeg: # use first frame as thumbnail if is sequence of jpegs - staging_dir = thumbnail_repre.get("stagingDir") - full_thumbnail_path = os.path.join(staging_dir, file) + full_thumbnail_path = os.path.join( + thumbnail_repre["stagingDir"], file + ) self.log.info( "For thumbnail is used file: {}".format(full_thumbnail_path) ) diff --git a/pype/plugins/tvpaint/create/create_render_layer.py b/pype/plugins/tvpaint/create/create_render_layer.py new file mode 100644 index 0000000000..c2921cebbe --- /dev/null +++ b/pype/plugins/tvpaint/create/create_render_layer.py @@ -0,0 +1,150 @@ +from avalon.tvpaint import pipeline, lib + + +class CreateRenderlayer(pipeline.Creator): + """Mark layer group as one instance.""" + name = "render_layer" + label = "RenderLayer" + family = "renderLayer" + icon = "cube" + defaults = ["Main"] + + rename_group = True + + subset_template = "{family}_{name}" + rename_script_template = ( + "tv_layercolor \"setcolor\"" + " {clip_id} {group_id} {r} {g} {b} \"{name}\"" + ) + + def process(self): + self.log.debug("Query data from workfile.") + instances = pipeline.list_instances() + layers_data = lib.layers_data() + + self.log.debug("Checking for selection groups.") + # Collect group ids from selection + group_ids = set() + for layer in layers_data: + if layer["selected"]: + group_ids.add(layer["group_id"]) + + # Raise if there is no selection + if not group_ids: + raise AssertionError("Nothing is selected.") + + # This creator should run only on one group + if len(group_ids) > 1: + raise AssertionError("More than one group is in selection.") + + group_id = tuple(group_ids)[0] + # If group id is `0` it is `default` group which is invalid + if group_id == 0: + raise AssertionError( + "Selection is not in group. Can't mark selection as Beauty." + ) + + self.log.debug(f"Selected group id is \"{group_id}\".") + self.data["group_id"] = group_id + + family = self.data["family"] + # Extract entered name + name = self.data["subset"][len(family):] + self.log.info(f"Extracted name from subset name \"{name}\".") + self.data["name"] = name + + # Change subset name by template + subset_name = self.subset_template.format(**{ + "family": self.family, + "name": name + }) + self.log.info(f"New subset name \"{subset_name}\".") + self.data["subset"] = subset_name + + # Check for instances of same group + existing_instance = None + existing_instance_idx = None + # Check if subset name is not already taken + same_subset_instance = None + same_subset_instance_idx = None + for idx, instance in enumerate(instances): + if instance["family"] == family: + if instance["group_id"] == group_id: + existing_instance = instance + existing_instance_idx = idx + elif instance["subset"] == subset_name: + same_subset_instance = instance + same_subset_instance_idx = idx + + if ( + same_subset_instance_idx is not None + and existing_instance_idx is not None + ): + break + + if same_subset_instance_idx is not None: + if self._ask_user_subset_override(same_subset_instance): + instances.pop(same_subset_instance_idx) + else: + return + + if existing_instance is not None: + self.log.info( + f"Beauty instance for group id {group_id} already exists" + ", overriding" + ) + instances[existing_instance_idx] = self.data + else: + instances.append(self.data) + + self.write_instances(instances) + + if not self.rename_group: + self.log.info("Group rename function is turned off. Skipping") + return + + self.log.debug("Querying groups data from workfile.") + groups_data = lib.groups_data() + + self.log.debug("Changing name of the group.") + selected_group = None + for group_data in groups_data: + if group_data["group_id"] == group_id: + selected_group = group_data + + # Rename TVPaint group (keep color same) + # - groups can't contain spaces + new_group_name = name.replace(" ", "_") + rename_script = self.rename_script_template.format( + clip_id=selected_group["clip_id"], + group_id=selected_group["group_id"], + r=selected_group["red"], + g=selected_group["green"], + b=selected_group["blue"], + name=new_group_name + ) + lib.execute_george_through_file(rename_script) + + self.log.info( + f"Name of group with index {group_id}" + f" was changed to \"{new_group_name}\"." + ) + + def _ask_user_subset_override(self, instance): + from Qt.QtWidgets import QMessageBox + + title = "Subset \"{}\" already exist".format(instance["subset"]) + text = ( + "Instance with subset name \"{}\" already exists." + "\n\nDo you want to override existing?" + ).format(instance["subset"]) + + dialog = QMessageBox() + dialog.setWindowTitle(title) + dialog.setText(text) + dialog.setStandardButtons(QMessageBox.Yes | QMessageBox.No) + dialog.setDefaultButton(QMessageBox.Yes) + dialog.exec_() + if dialog.result() == QMessageBox.Yes: + return True + return False diff --git a/pype/plugins/tvpaint/create/create_render_pass.py b/pype/plugins/tvpaint/create/create_render_pass.py new file mode 100644 index 0000000000..585da6ec00 --- /dev/null +++ b/pype/plugins/tvpaint/create/create_render_pass.py @@ -0,0 +1,105 @@ +from avalon.tvpaint import pipeline, lib + + +class CreateRenderPass(pipeline.Creator): + """Render pass is combination of one or more layers from same group. + + Requirement to create Render Pass is to have already created beauty + instance. Beauty instance is used as base for subset name. + """ + name = "render_pass" + label = "RenderPass" + family = "renderPass" + icon = "cube" + defaults = ["Main"] + + subset_template = "{family}_{render_layer}_{pass}" + + def process(self): + self.log.debug("Query data from workfile.") + instances = pipeline.list_instances() + layers_data = lib.layers_data() + + self.log.debug("Checking selection.") + # Get all selected layers and their group ids + group_ids = set() + selected_layers = [] + for layer in layers_data: + if layer["selected"]: + selected_layers.append(layer) + group_ids.add(layer["group_id"]) + + # Raise if nothing is selected + if not selected_layers: + raise AssertionError("Nothing is selected.") + + # Raise if layers from multiple groups are selected + if len(group_ids) != 1: + raise AssertionError("More than one group is in selection.") + + group_id = tuple(group_ids)[0] + self.log.debug(f"Selected group id is \"{group_id}\".") + + # Find beauty instance for selected layers + beauty_instance = None + for instance in instances: + if ( + instance["family"] == "renderLayer" + and instance["group_id"] == group_id + ): + beauty_instance = instance + break + + # Beauty is required for this creator so raise if was not found + if beauty_instance is None: + raise AssertionError("Beauty pass does not exist yet.") + + render_layer = beauty_instance["name"] + + # Extract entered name + family = self.data["family"] + name = self.data["subset"] + # Is this right way how to get name? + name = name[len(family):] + self.log.info(f"Extracted name from subset name \"{name}\".") + + self.data["group_id"] = group_id + self.data["pass"] = name + self.data["render_layer"] = render_layer + + # Collect selected layer ids to be stored into instance + layer_ids = [layer["layer_id"] for layer in selected_layers] + self.data["layer_ids"] = layer_ids + + # Replace `beauty` in beauty's subset name with entered name + subset_name = self.subset_template.format(**{ + "family": family, + "render_layer": render_layer, + "pass": name + }) + self.data["subset"] = subset_name + self.log.info(f"New subset name is \"{subset_name}\".") + + # Check if same instance already exists + existing_instance = None + existing_instance_idx = None + for idx, instance in enumerate(instances): + if ( + instance["family"] == family + and instance["group_id"] == group_id + and instance["pass"] == name + ): + existing_instance = instance + existing_instance_idx = idx + break + + if existing_instance is not None: + self.log.info( + f"Render pass instance for group id {group_id}" + f" and name \"{name}\" already exists, overriding." + ) + instances[existing_instance_idx] = self.data + else: + instances.append(self.data) + + self.write_instances(instances) diff --git a/pype/plugins/tvpaint/create/create_review.py b/pype/plugins/tvpaint/create/create_review.py new file mode 100644 index 0000000000..9f7ee1396e --- /dev/null +++ b/pype/plugins/tvpaint/create/create_review.py @@ -0,0 +1,18 @@ +from avalon.tvpaint import pipeline + + +class CreateReview(pipeline.Creator): + """Review for global review of all layers.""" + name = "review" + label = "Review" + family = "review" + icon = "cube" + defaults = ["Main"] + + def process(self): + instances = pipeline.list_instances() + for instance in instances: + if instance["family"] == self.family: + self.log.info("Review family is already Created.") + return + super(CreateReview, self).process() diff --git a/pype/plugins/tvpaint/load/load_image.py b/pype/plugins/tvpaint/load/load_image.py index 0d5a0b4aa1..f77fab87f8 100644 --- a/pype/plugins/tvpaint/load/load_image.py +++ b/pype/plugins/tvpaint/load/load_image.py @@ -1,9 +1,8 @@ -from avalon import api from avalon.vendor import qargparse -from avalon.tvpaint import CommunicatorWrapper +from avalon.tvpaint import lib, pipeline -class ImportImage(api.Loader): +class ImportImage(pipeline.Loader): """Load image or image sequence to TVPaint as new layer.""" families = ["render", "image", "background", "plate"] @@ -80,4 +79,4 @@ class ImportImage(api.Loader): layer_name, load_options_str ) - return CommunicatorWrapper.execute_george_through_file(george_script) + return lib.execute_george_through_file(george_script) diff --git a/pype/plugins/tvpaint/load/load_reference_image.py b/pype/plugins/tvpaint/load/load_reference_image.py new file mode 100644 index 0000000000..0fa4cefc51 --- /dev/null +++ b/pype/plugins/tvpaint/load/load_reference_image.py @@ -0,0 +1,244 @@ +from avalon.pipeline import get_representation_context +from avalon.vendor import qargparse +from avalon.tvpaint import lib, pipeline + + +class LoadImage(pipeline.Loader): + """Load image or image sequence to TVPaint as new layer.""" + + families = ["render", "image", "background", "plate"] + representations = ["*"] + + label = "Load Image" + order = 1 + icon = "image" + color = "white" + + import_script = ( + "filepath = \"{}\"\n" + "layer_name = \"{}\"\n" + "tv_loadsequence filepath {}PARSE layer_id\n" + "tv_layerrename layer_id layer_name" + ) + + defaults = { + "stretch": True, + "timestretch": True, + "preload": True + } + + options = [ + qargparse.Boolean( + "stretch", + label="Stretch to project size", + default=True, + help="Stretch loaded image/s to project resolution?" + ), + qargparse.Boolean( + "timestretch", + label="Stretch to timeline length", + default=True, + help="Clip loaded image/s to timeline length?" + ), + qargparse.Boolean( + "preload", + label="Preload loaded image/s", + default=True, + help="Preload image/s?" + ) + ] + + def load(self, context, name, namespace, options): + stretch = options.get("stretch", self.defaults["stretch"]) + timestretch = options.get("timestretch", self.defaults["timestretch"]) + preload = options.get("preload", self.defaults["preload"]) + + load_options = [] + if stretch: + load_options.append("\"STRETCH\"") + if timestretch: + load_options.append("\"TIMESTRETCH\"") + if preload: + load_options.append("\"PRELOAD\"") + + load_options_str = "" + for load_option in load_options: + load_options_str += (load_option + " ") + + # Prepare layer name + asset_name = context["asset"]["name"] + subset_name = context["subset"]["name"] + layer_name = self.get_unique_layer_name(asset_name, subset_name) + + # Fill import script with filename and layer name + # - filename mus not contain backwards slashes + george_script = self.import_script.format( + self.fname.replace("\\", "/"), + layer_name, + load_options_str + ) + + lib.execute_george_through_file(george_script) + + loaded_layer = None + layers = lib.layers_data() + for layer in layers: + if layer["name"] == layer_name: + loaded_layer = layer + break + + if loaded_layer is None: + raise AssertionError( + "Loading probably failed during execution of george script." + ) + + layer_ids = [loaded_layer["layer_id"]] + namespace = namespace or layer_name + return pipeline.containerise( + name=name, + namespace=namespace, + layer_ids=layer_ids, + context=context, + loader=self.__class__.__name__ + ) + + def _remove_layers(self, layer_ids, layers=None): + if not layer_ids: + return + + if layers is None: + layers = lib.layers_data() + + available_ids = set(layer["layer_id"] for layer in layers) + layer_ids_to_remove = [] + + for layer_id in layer_ids: + if layer_id in available_ids: + layer_ids_to_remove.append(layer_id) + + if not layer_ids_to_remove: + return + + george_script_lines = [] + for layer_id in layer_ids_to_remove: + line = "tv_layerkill {}".format(layer_id) + george_script_lines.append(line) + george_script = "\n".join(george_script_lines) + lib.execute_george_through_file(george_script) + + def remove(self, container): + layer_ids = self.layer_ids_from_container(container) + self._remove_layers(layer_ids) + + current_containers = pipeline.ls() + pop_idx = None + for idx, cur_con in enumerate(current_containers): + if cur_con["objectName"] == container["objectName"]: + pop_idx = idx + break + + if pop_idx is None: + self.log.warning( + "Didn't found container in workfile containers. {}".format( + container + ) + ) + return + + current_containers.pop(pop_idx) + pipeline.write_workfile_metadata( + pipeline.SECTION_NAME_CONTAINERS, current_containers + ) + + def switch(self, container, representation): + self.update(container, representation) + + def update(self, container, representation): + """Replace container with different version. + + New layers are loaded as first step. Then is tried to change data in + new layers with data from old layers. When that is done old layers are + removed. + """ + # Create new containers first + context = get_representation_context(representation) + name = container["name"] + namespace = container["namespace"] + new_container = self.load(context, name, namespace, {}) + new_layer_ids = self.layer_ids_from_container(new_container) + + # Get layer ids from previous container + old_layer_ids = self.layer_ids_from_container(container) + + layers = lib.layers_data() + layers_by_id = { + layer["layer_id"]: layer + for layer in layers + } + + old_layers = [] + new_layers = [] + for layer_id in old_layer_ids: + layer = layers_by_id.get(layer_id) + if layer: + old_layers.append(layer) + + for layer_id in new_layer_ids: + layer = layers_by_id.get(layer_id) + if layer: + new_layers.append(layer) + + # Prepare few data + new_start_position = None + new_group_id = None + for layer in old_layers: + position = layer["position"] + group_id = layer["group_id"] + if new_start_position is None: + new_start_position = position + elif new_start_position > position: + new_start_position = position + + if new_group_id is None: + new_group_id = group_id + elif new_group_id < 0: + continue + elif new_group_id != group_id: + new_group_id = -1 + + george_script_lines = [] + # Group new layers to same group as previous container layers had + # - all old layers must be under same group + if new_group_id is not None and new_group_id > 0: + for layer in new_layers: + line = "tv_layercolor \"set\" {} {}".format( + layer["layer_id"], new_group_id + ) + george_script_lines.append(line) + + # Rename new layer to have same name + # - only if both old and new have one layer + if len(old_layers) == 1 and len(new_layers) == 1: + layer_name = old_layers[0]["name"] + george_script_lines.append( + "tv_layerrename {} \"{}\"".format( + new_layers[0]["layer_id"], layer_name + ) + ) + + # Change position of new layer + # - this must be done before remove old layers + if len(new_layers) == 1 and new_start_position is not None: + new_layer = new_layers[0] + george_script_lines.extend([ + "tv_layerset {}".format(new_layer["layer_id"]), + "tv_layermove {}".format(new_start_position) + ]) + + # Execute george scripts if there are any + if george_script_lines: + george_script = "\n".join(george_script_lines) + lib.execute_george_through_file(george_script) + + # Remove old container + self.remove(container) diff --git a/pype/plugins/tvpaint/publish/collect_instances.py b/pype/plugins/tvpaint/publish/collect_instances.py new file mode 100644 index 0000000000..a9c19f9c0a --- /dev/null +++ b/pype/plugins/tvpaint/publish/collect_instances.py @@ -0,0 +1,172 @@ +import json +import copy +import pyblish.api +from avalon import io + + +class CollectInstances(pyblish.api.ContextPlugin): + label = "Collect Instances" + order = pyblish.api.CollectorOrder - 1 + hosts = ["tvpaint"] + + def process(self, context): + workfile_instances = context.data["workfileInstances"] + + self.log.debug("Collected ({}) instances:\n{}".format( + len(workfile_instances), + json.dumps(workfile_instances, indent=4) + )) + + for instance_data in workfile_instances: + instance_data["fps"] = context.data["fps"] + + # Store workfile instance data to instance data + instance_data["originData"] = copy.deepcopy(instance_data) + # Global instance data modifications + # Fill families + family = instance_data["family"] + # Add `review` family for thumbnail integration + instance_data["families"] = [family, "review"] + + # Instance name + subset_name = instance_data["subset"] + name = instance_data.get("name", subset_name) + instance_data["name"] = name + + active = instance_data.get("active", True) + instance_data["active"] = active + instance_data["publish"] = active + # Add representations key + instance_data["representations"] = [] + + # Different instance creation based on family + instance = None + if family == "review": + # Change subset name + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}".format(family, task_name.capitalize()) + instance_data["subset"] = new_subset_name + + instance = context.create_instance(**instance_data) + instance.data["layers"] = context.data["layersData"] + # Add ftrack family + instance.data["families"].append("ftrack") + + elif family == "renderLayer": + instance = self.create_render_layer_instance( + context, instance_data + ) + elif family == "renderPass": + instance = self.create_render_pass_instance( + context, instance_data + ) + else: + raise AssertionError( + "Instance with unknown family \"{}\": {}".format( + family, instance_data + ) + ) + + frame_start = context.data["frameStart"] + frame_end = frame_start + for layer in instance.data["layers"]: + _frame_end = layer["frame_end"] + if _frame_end > frame_end: + frame_end = _frame_end + + instance.data["frameStart"] = frame_start + instance.data["frameEnd"] = frame_end + + self.log.debug("Created instance: {}\n{}".format( + instance, json.dumps(instance.data, indent=4) + )) + + def create_render_layer_instance(self, context, instance_data): + name = instance_data["name"] + # Change label + subset_name = instance_data["subset"] + instance_data["label"] = "{}_Beauty".format(name) + + # Change subset name + # Final family of an instance will be `render` + new_family = "render" + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}_{}_Beauty".format( + new_family, task_name.capitalize(), name + ) + instance_data["subset"] = new_subset_name + self.log.debug("Changed subset name \"{}\"->\"{}\"".format( + subset_name, new_subset_name + )) + + # Get all layers for the layer + layers_data = context.data["layersData"] + group_id = instance_data["group_id"] + group_layers = [] + for layer in layers_data: + if layer["group_id"] == group_id and layer["visible"]: + group_layers.append(layer) + + if not group_layers: + # Should be handled here? + self.log.warning(( + f"Group with id {group_id} does not contain any layers." + f" Instance \"{name}\" not created." + )) + return None + + instance_data["layers"] = group_layers + + # Add ftrack family + instance_data["families"].append("ftrack") + + return context.create_instance(**instance_data) + + def create_render_pass_instance(self, context, instance_data): + pass_name = instance_data["pass"] + self.log.info( + "Creating render pass instance. \"{}\"".format(pass_name) + ) + # Change label + render_layer = instance_data["render_layer"] + instance_data["label"] = "{}_{}".format(render_layer, pass_name) + + # Change subset name + # Final family of an instance will be `render` + new_family = "render" + old_subset_name = instance_data["subset"] + task_name = io.Session["AVALON_TASK"] + new_subset_name = "{}{}_{}_{}".format( + new_family, task_name.capitalize(), render_layer, pass_name + ) + instance_data["subset"] = new_subset_name + self.log.debug("Changed subset name \"{}\"->\"{}\"".format( + old_subset_name, new_subset_name + )) + + layers_data = context.data["layersData"] + layers_by_id = { + layer["layer_id"]: layer + for layer in layers_data + } + + layer_ids = instance_data["layer_ids"] + render_pass_layers = [] + for layer_id in layer_ids: + layer = layers_by_id.get(layer_id) + if not layer: + self.log.warning(f"Layer with id {layer_id} was not found.") + continue + + render_pass_layers.append(layer) + + if not render_pass_layers: + name = instance_data["name"] + self.log.warning( + f"None of the layers from the RenderPass \"{name}\"" + " exist anymore. Instance not created." + ) + return None + + instance_data["layers"] = render_pass_layers + return context.create_instance(**instance_data) diff --git a/pype/plugins/tvpaint/publish/collect_workfile_data.py b/pype/plugins/tvpaint/publish/collect_workfile_data.py new file mode 100644 index 0000000000..31fd97ced4 --- /dev/null +++ b/pype/plugins/tvpaint/publish/collect_workfile_data.py @@ -0,0 +1,66 @@ +import json + +import pyblish.api +from avalon.tvpaint import pipeline, lib + + +class CollectWorkfileData(pyblish.api.ContextPlugin): + label = "Collect Workfile Data" + order = pyblish.api.CollectorOrder - 1.01 + hosts = ["tvpaint"] + + def process(self, context): + self.log.info("Collecting instance data from workfile") + instance_data = pipeline.list_instances() + self.log.debug( + "Instance data:\"{}".format(json.dumps(instance_data, indent=4)) + ) + context.data["workfileInstances"] = instance_data + + self.log.info("Collecting layers data from workfile") + layers_data = lib.layers_data() + self.log.debug( + "Layers data:\"{}".format(json.dumps(layers_data, indent=4)) + ) + context.data["layersData"] = layers_data + + self.log.info("Collecting groups data from workfile") + group_data = lib.groups_data() + self.log.debug( + "Group data:\"{}".format(json.dumps(group_data, indent=4)) + ) + context.data["groupsData"] = group_data + + self.log.info("Collecting scene data from workfile") + workfile_info_parts = lib.execute_george("tv_projectinfo").split(" ") + + frame_start = int(workfile_info_parts.pop(-1)) + field_order = workfile_info_parts.pop(-1) + frame_rate = float(workfile_info_parts.pop(-1)) + pixel_apsect = float(workfile_info_parts.pop(-1)) + height = int(workfile_info_parts.pop(-1)) + width = int(workfile_info_parts.pop(-1)) + workfile_path = " ".join(workfile_info_parts).replace("\"", "") + + # TODO This is not porper way of getting last frame + # - but don't know better + last_frame = frame_start + for layer in layers_data: + frame_end = layer["frame_end"] + if frame_end > last_frame: + last_frame = frame_end + + scene_data = { + "currentFile": workfile_path, + "sceneWidth": width, + "sceneHeight": height, + "pixelAspect": pixel_apsect, + "frameStart": frame_start, + "frameEnd": last_frame, + "fps": frame_rate, + "fieldOrder": field_order + } + self.log.debug( + "Scene data: {}".format(json.dumps(scene_data, indent=4)) + ) + context.data.update(scene_data) diff --git a/pype/plugins/tvpaint/publish/extract_sequence.py b/pype/plugins/tvpaint/publish/extract_sequence.py new file mode 100644 index 0000000000..d173ac287d --- /dev/null +++ b/pype/plugins/tvpaint/publish/extract_sequence.py @@ -0,0 +1,352 @@ +import os +import shutil +import tempfile + +import pyblish.api +from avalon.tvpaint import lib + + +class ExtractSequence(pyblish.api.Extractor): + label = "Extract Sequence" + hosts = ["tvpaint"] + families = ["review", "renderPass", "renderLayer"] + + save_mode_to_ext = { + "avi": ".avi", + "bmp": ".bmp", + "cin": ".cin", + "deep": ".dip", + "dps": ".dps", + "dpx": ".dpx", + "flc": ".fli", + "gif": ".gif", + "ilbm": ".iff", + "jpeg": ".jpg", + "pcx": ".pcx", + "png": ".png", + "psd": ".psd", + "qt": ".qt", + "rtv": ".rtv", + "sun": ".ras", + "tiff": ".tiff", + "tga": ".tga", + "vpb": ".vpb" + } + sequential_save_mode = { + "bmp", + "dpx", + "ilbm", + "jpeg", + "png", + "sun", + "tiff", + "tga" + } + + default_save_mode = "\"PNG\"" + save_mode_for_family = { + "review": "\"PNG\"", + "renderPass": "\"PNG\"", + "renderLayer": "\"PNG\"", + } + + def process(self, instance): + self.log.info( + "* Processing instance \"{}\"".format(instance.data["label"]) + ) + + # Get all layers and filter out not visible + layers = instance.data["layers"] + filtered_layers = [ + layer + for layer in layers + if layer["visible"] + ] + layer_ids = [str(layer["layer_id"]) for layer in filtered_layers] + if not layer_ids: + self.log.info( + f"None of the layers from the instance" + " are visible. Extraction skipped." + ) + return + + self.log.debug( + "Instance has {} layers with ids: {}".format( + len(layer_ids), ", ".join(layer_ids) + ) + ) + # This is plugin attribe cleanup method + self._prepare_save_modes() + + family_lowered = instance.data["family"].lower() + save_mode = self.save_mode_for_family.get( + family_lowered, self.default_save_mode + ) + save_mode_type = self._get_save_mode_type(save_mode) + + if not bool(save_mode_type in self.sequential_save_mode): + raise AssertionError(( + "Plugin can export only sequential frame output" + " but save mode for family \"{}\" is not for sequence > {} <" + ).format(instance.data["family"], save_mode)) + + frame_start = instance.data["frameStart"] + frame_end = instance.data["frameEnd"] + + filename_template = self._get_filename_template( + save_mode_type, save_mode, frame_end + ) + ext = os.path.splitext(filename_template)[1].replace(".", "") + + self.log.debug( + "Using save mode > {} < and file template \"{}\"".format( + save_mode, filename_template + ) + ) + + # Save to staging dir + output_dir = instance.data.get("stagingDir") + if not output_dir: + # Create temp folder if staging dir is not set + output_dir = tempfile.mkdtemp().replace("\\", "/") + instance.data["stagingDir"] = output_dir + + self.log.debug( + "Files will be rendered to folder: {}".format(output_dir) + ) + + thumbnail_filename = "thumbnail" + + # Render output + output_files_by_frame = self.render( + save_mode, filename_template, output_dir, + filtered_layers, frame_start, frame_end, thumbnail_filename + ) + thumbnail_fullpath = output_files_by_frame.pop( + thumbnail_filename, None + ) + + # Fill gaps in sequence + self.fill_missing_frames( + output_files_by_frame, + frame_start, + frame_end, + filename_template + ) + + # Fill tags and new families + tags = [] + if family_lowered in ("review", "renderlayer"): + # Add `ftrackreview` tag + tags.append("ftrackreview") + + repre_files = [ + os.path.basename(filepath) + for filepath in output_files_by_frame.values() + ] + new_repre = { + "name": ext, + "ext": ext, + "files": repre_files, + "stagingDir": output_dir, + "frameStart": frame_start, + "frameEnd": frame_end, + "tags": tags + } + self.log.debug("Creating new representation: {}".format(new_repre)) + + instance.data["representations"].append(new_repre) + + if family_lowered in ("renderpass", "renderlayer"): + # Change family to render + instance.data["family"] = "render" + + if not thumbnail_fullpath: + return + + # Create thumbnail representation + thumbnail_repre = { + "name": "thumbnail", + "ext": ext, + "files": os.path.basename(thumbnail_fullpath), + "stagingDir": output_dir, + "tags": ["thumbnail"] + } + instance.data["representations"].append(thumbnail_repre) + + def _prepare_save_modes(self): + """Lower family names in keys and skip empty values.""" + new_specifications = {} + for key, value in self.save_mode_for_family.items(): + if value: + new_specifications[key.lower()] = value + else: + self.log.warning(( + "Save mode for family \"{}\" has empty value." + " The family will use default save mode: > {} <." + ).format(key, self.default_save_mode)) + self.save_mode_for_family = new_specifications + + def _get_save_mode_type(self, save_mode): + """Extract type of save mode. + + Helps to define output files extension. + """ + save_mode_type = ( + save_mode.lower() + .split(" ")[0] + .replace("\"", "") + ) + self.log.debug("Save mode type is \"{}\"".format(save_mode_type)) + return save_mode_type + + def _get_filename_template(self, save_mode_type, save_mode, frame_end): + """Get filetemplate for rendered files. + + This is simple template contains `{frame}{ext}` for sequential outputs + and `single_file{ext}` for single file output. Output is rendered to + temporary folder so filename should not matter as integrator change + them. + """ + ext = self.save_mode_to_ext.get(save_mode_type) + if ext is None: + raise AssertionError(( + "Couldn't find file extension for TVPaint's save mode: > {} <" + ).format(save_mode)) + + frame_padding = 4 + frame_end_str_len = len(str(frame_end)) + if frame_end_str_len > frame_padding: + frame_padding = frame_end_str_len + + return "{{frame:0>{}}}".format(frame_padding) + ext + + def render( + self, save_mode, filename_template, output_dir, layers, + first_frame, last_frame, thumbnail_filename + ): + """ Export images from TVPaint. + + Args: + save_mode (str): Argument for `tv_savemode` george script function. + More about save mode in documentation. + filename_template (str): Filename template of an output. Template + should already contain extension. Template may contain only + keyword argument `{frame}` or index argument (for same value). + Extension in template must match `save_mode`. + layers (list): List of layers to be exported. + first_frame (int): Starting frame from which export will begin. + last_frame (int): On which frame export will end. + + Retruns: + dict: Mapping frame to output filepath. + """ + + # Add save mode arguments to function + save_mode = "tv_SaveMode {}".format(save_mode) + + # Map layers by position + layers_by_position = { + layer["position"]: layer + for layer in layers + } + + # Sort layer positions in reverse order + sorted_positions = list(reversed(sorted(layers_by_position.keys()))) + if not sorted_positions: + return + + # Create temporary layer + new_layer_id = lib.execute_george("tv_layercreate _tmp_layer") + + # Merge layers to temp layer + george_script_lines = [] + # Set duplicated layer as current + george_script_lines.append("tv_layerset {}".format(new_layer_id)) + for position in sorted_positions: + layer = layers_by_position[position] + george_script_lines.append( + "tv_layermerge {}".format(layer["layer_id"]) + ) + + lib.execute_george_through_file("\n".join(george_script_lines)) + + # Frames with keyframe + exposure_frames = lib.get_exposure_frames( + new_layer_id, first_frame, last_frame + ) + + # TODO what if there is not exposue frames? + # - this force to have first frame all the time + if first_frame not in exposure_frames: + exposure_frames.insert(0, first_frame) + + # Restart george script lines + george_script_lines = [] + george_script_lines.append(save_mode) + + all_output_files = {} + for frame in exposure_frames: + filename = filename_template.format(frame, frame=frame) + dst_path = "/".join([output_dir, filename]) + all_output_files[frame] = os.path.normpath(dst_path) + + # Go to frame + george_script_lines.append("tv_layerImage {}".format(frame)) + # Store image to output + george_script_lines.append("tv_saveimage \"{}\"".format(dst_path)) + + # Export thumbnail + if thumbnail_filename: + basename, ext = os.path.splitext(thumbnail_filename) + if not ext: + ext = ".png" + thumbnail_fullpath = "/".join([output_dir, basename + ext]) + all_output_files[thumbnail_filename] = thumbnail_fullpath + # Force save mode to png for thumbnail + george_script_lines.append("tv_SaveMode \"PNG\"") + # Go to frame + george_script_lines.append("tv_layerImage {}".format(first_frame)) + # Store image to output + george_script_lines.append( + "tv_saveimage \"{}\"".format(thumbnail_fullpath) + ) + + # Delete temporary layer + george_script_lines.append("tv_layerkill {}".format(new_layer_id)) + + lib.execute_george_through_file("\n".join(george_script_lines)) + + return all_output_files + + def fill_missing_frames( + self, filepaths_by_frame, first_frame, last_frame, filename_template + ): + """Fill not rendered frames with previous frame. + + Extractor is rendering only frames with keyframes (exposure frames) to + get output faster which means there may be gaps between frames. + This function fill the missing frames. + """ + output_dir = None + previous_frame_filepath = None + for frame in range(first_frame, last_frame + 1): + if frame in filepaths_by_frame: + previous_frame_filepath = filepaths_by_frame[frame] + continue + + elif previous_frame_filepath is None: + self.log.warning( + "No frames to fill. Seems like nothing was exported." + ) + break + + if output_dir is None: + output_dir = os.path.dirname(previous_frame_filepath) + + filename = filename_template.format(frame=frame) + space_filepath = os.path.normpath( + os.path.join(output_dir, filename) + ) + filepaths_by_frame[frame] = space_filepath + shutil.copy(previous_frame_filepath, space_filepath) diff --git a/pype/plugins/tvpaint/publish/validate_frame_range.py b/pype/plugins/tvpaint/publish/validate_frame_range.py new file mode 100644 index 0000000000..cb9a103fa2 --- /dev/null +++ b/pype/plugins/tvpaint/publish/validate_frame_range.py @@ -0,0 +1,76 @@ +import collections +import pyblish.api + + +class ValidateLayersGroup(pyblish.api.InstancePlugin): + """Validate group ids of renderPass layers. + + Validates that all layers are in same group as they were during creation. + """ + + label = "Validate Layers Group" + order = pyblish.api.ValidatorOrder + families = ["renderPass"] + + def process(self, instance): + # Prepare layers + layers_data = instance.context.data["layersData"] + layers_by_id = { + layer["layer_id"]: layer + for layer in layers_data + } + + # Expected group id for instance layers + group_id = instance.data["group_id"] + # Layers ids of an instance + layer_ids = instance.data["layer_ids"] + # Check if all layers from render pass are in right group + invalid_layers_by_group_id = collections.defaultdict(list) + for layer_id in layer_ids: + layer = layers_by_id.get(layer_id) + _group_id = layer["group_id"] + if _group_id != group_id: + invalid_layers_by_group_id[_group_id].append(layer) + + # Everything is OK and skip exception + if not invalid_layers_by_group_id: + return + + # Exception message preparations + groups_data = instance.context.data["groupsData"] + groups_by_id = { + group["group_id"]: group + for group in groups_data + } + correct_group = groups_by_id[group_id] + + per_group_msgs = [] + for _group_id, layers in invalid_layers_by_group_id.items(): + _group = groups_by_id[_group_id] + layers_msgs = [] + for layer in layers: + layers_msgs.append( + "\"{}\" (id: {})".format(layer["name"], layer["layer_id"]) + ) + per_group_msgs.append( + "Group \"{}\" (id: {}) < {} >".format( + _group["name"], + _group["group_id"], + ", ".join(layers_msgs) + ) + ) + + # Raise an error + raise AssertionError(( + # Short message + "Layers in wrong group." + # Description what's wrong + " Layers from render pass \"{}\" must be in group {} (id: {})." + # Detailed message + " Layers in wrong group: {}" + ).format( + instance.data["label"], + correct_group["name"], + correct_group["group_id"], + " | ".join(per_group_msgs) + )) diff --git a/pype/scripts/fusion_switch_shot.py b/pype/scripts/fusion_switch_shot.py index f936b7d8e0..5791220acd 100644 --- a/pype/scripts/fusion_switch_shot.py +++ b/pype/scripts/fusion_switch_shot.py @@ -191,7 +191,7 @@ def switch(asset_name, filepath=None, new=True): representations = [] for container in containers: try: - representation = pype.switch_item(container, + representation = fusion_lib.switch_item(container, asset_name=asset_name) representations.append(representation) except Exception as e: diff --git a/pype/scripts/otio_burnin.py b/pype/scripts/otio_burnin.py index ef09661f6e..8da1dd8616 100644 --- a/pype/scripts/otio_burnin.py +++ b/pype/scripts/otio_burnin.py @@ -213,9 +213,7 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): if frame_start is None: replacement_final = replacement_size = str(MISSING_KEY_VALUE) else: - replacement_final = "\\'{}\\'".format( - r'%%{eif\:n+%d\:d}' % frame_start - ) + replacement_final = "%{eif:n+" + str(frame_start) + ":d}" replacement_size = str(frame_end) final_text = final_text.replace( @@ -328,11 +326,13 @@ class ModifiedBurnins(ffmpeg_burnins.Burnins): _stdout, _stderr = proc.communicate() if _stdout: - print(_stdout.decode("utf-8")) + for line in _stdout.split(b"\r\n"): + print(line.decode("utf-8")) # This will probably never happen as ffmpeg use stdout if _stderr: - print(_stderr.decode("utf-8")) + for line in _stderr.split(b"\r\n"): + print(line.decode("utf-8")) if proc.returncode != 0: raise RuntimeError( @@ -578,7 +578,10 @@ def burnins_from_data( if __name__ == "__main__": print("* Burnin script started") - in_data = json.loads(sys.argv[-1]) + in_data_json_path = sys.argv[-1] + with open(in_data_json_path, "r") as file_stream: + in_data = json.load(file_stream) + burnins_from_data( in_data["input"], in_data["output"], diff --git a/pype/tests/README.md b/pype/tests/README.md new file mode 100644 index 0000000000..c05166767c --- /dev/null +++ b/pype/tests/README.md @@ -0,0 +1,4 @@ +Tests for Pype +-------------- +Trigger by: + `pype test --pype` \ No newline at end of file diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py new file mode 100644 index 0000000000..152be8d1eb --- /dev/null +++ b/pype/tests/test_lib_restructuralization.py @@ -0,0 +1,39 @@ +# Test for backward compatibility of restructure of lib.py into lib library +# Contains simple imports that should still work + + +def test_backward_compatibility(printer): + printer("Test if imports still work") + try: + from pype.lib import filter_pyblish_plugins + from pype.lib import execute_hook + from pype.lib import PypeHook + + from pype.lib import get_latest_version + from pype.lib import ApplicationLaunchFailed + from pype.lib import launch_application + from pype.lib import ApplicationAction + from pype.lib import get_avalon_database + from pype.lib import set_io_database + + from pype.lib import get_ffmpeg_tool_path + from pype.lib import get_last_version_from_path + from pype.lib import get_paths_from_environ + from pype.lib import get_version_from_path + from pype.lib import version_up + + from pype.lib import is_latest + from pype.lib import any_outdated + from pype.lib import get_asset + from pype.lib import get_hierarchy + from pype.lib import get_linked_assets + from pype.lib import get_latest_version + from pype.lib import ffprobe_streams + + from pype.hosts.fusion.lib import switch_item + + from pype.lib import source_hash + from pype.lib import _subprocess + + except ImportError as e: + raise diff --git a/pype/tools/pyblish_pype/app.css b/pype/tools/pyblish_pype/app.css index c51126e89f..33b6acbddb 100644 --- a/pype/tools/pyblish_pype/app.css +++ b/pype/tools/pyblish_pype/app.css @@ -222,10 +222,6 @@ QToolButton { background: #444; } -#Header #ArtistTab { - background-image: url("img/tab-home.png"); -} - #Header #TerminalTab { background-image: url("img/tab-terminal.png"); } diff --git a/pype/tools/pyblish_pype/delegate.py b/pype/tools/pyblish_pype/delegate.py index cb9123bf3a..bf3fbc1853 100644 --- a/pype/tools/pyblish_pype/delegate.py +++ b/pype/tools/pyblish_pype/delegate.py @@ -522,168 +522,6 @@ class PluginDelegate(QtWidgets.QStyledItemDelegate): return QtCore.QSize(option.rect.width(), 20) -class ArtistDelegate(QtWidgets.QStyledItemDelegate): - """Delegate used on Artist page""" - - def paint(self, painter, option, index): - """Paint checkbox and text - - _______________________________________________ - | | label | duration |arrow| - |toggle |_____________________| | to | - | | families | |persp| - |_______|_____________________|___________|_____| - - """ - - # Layout - spacing = 10 - - body_rect = QtCore.QRectF(option.rect).adjusted(2, 2, -8, -2) - content_rect = body_rect.adjusted(5, 5, -5, -5) - - perspective_rect = QtCore.QRectF(body_rect) - perspective_rect.setWidth(35) - perspective_rect.setHeight(35) - perspective_rect.translate( - content_rect.width() - (perspective_rect.width() / 2) + 10, - (content_rect.height() / 2) - (perspective_rect.height() / 2) - ) - - toggle_rect = QtCore.QRectF(body_rect) - toggle_rect.setWidth(7) - toggle_rect.adjust(1, 1, 0, -1) - - icon_rect = QtCore.QRectF(content_rect) - icon_rect.translate(toggle_rect.width() + spacing, 3) - icon_rect.setWidth(35) - icon_rect.setHeight(35) - - duration_rect = QtCore.QRectF(content_rect) - duration_rect.translate(content_rect.width() - 50, 0) - - # Colors - check_color = colors["idle"] - - publish_states = index.data(Roles.PublishFlagsRole) - if publish_states is None: - return - if publish_states & InstanceStates.InProgress: - check_color = colors["active"] - - elif publish_states & InstanceStates.HasError: - check_color = colors["error"] - - elif publish_states & InstanceStates.HasWarning: - check_color = colors["warning"] - - elif publish_states & InstanceStates.HasFinished: - check_color = colors["ok"] - - elif not index.data(Roles.IsEnabledRole): - check_color = colors["inactive"] - - perspective_icon = icons["angle-right"] - - if not index.data(QtCore.Qt.CheckStateRole): - font_color = colors["inactive"] - else: - font_color = colors["idle"] - - if ( - option.state - & ( - QtWidgets.QStyle.State_MouseOver - or QtWidgets.QStyle.State_Selected - ) - ): - perspective_color = colors["idle"] - else: - perspective_color = colors["inactive"] - # Maintan reference to state, so we can restore it once we're done - painter.save() - - # Draw background - painter.fillRect(body_rect, colors["hover"]) - - # Draw icon - icon = index.data(QtCore.Qt.DecorationRole) - - painter.setFont(fonts["largeAwesome"]) - painter.setPen(QtGui.QPen(font_color)) - painter.drawText(icon_rect, icon) - - # Draw label - painter.setFont(fonts["h3"]) - label_rect = QtCore.QRectF(content_rect) - label_x_offset = icon_rect.width() + spacing - label_rect.translate( - label_x_offset, - 0 - ) - metrics = painter.fontMetrics() - label_rect.setHeight(metrics.lineSpacing()) - label_rect.setWidth( - content_rect.width() - - label_x_offset - - perspective_rect.width() - ) - # Elide label - label = index.data(QtCore.Qt.DisplayRole) - label = metrics.elidedText( - label, QtCore.Qt.ElideRight, label_rect.width() - ) - painter.drawText(label_rect, label) - - # Draw families - painter.setFont(fonts["h5"]) - painter.setPen(QtGui.QPen(colors["inactive"])) - - families = ", ".join(index.data(Roles.FamiliesRole)) - families = painter.fontMetrics().elidedText( - families, QtCore.Qt.ElideRight, label_rect.width() - ) - - families_rect = QtCore.QRectF(label_rect) - families_rect.translate(0, label_rect.height() + spacing) - - painter.drawText(families_rect, families) - - painter.setFont(fonts["largeAwesome"]) - painter.setPen(QtGui.QPen(perspective_color)) - painter.drawText(perspective_rect, perspective_icon) - - # Draw checkbox - pen = QtGui.QPen(check_color, 1) - painter.setPen(pen) - - if index.data(Roles.IsOptionalRole): - painter.drawRect(toggle_rect) - - if index.data(QtCore.Qt.CheckStateRole): - painter.fillRect(toggle_rect, check_color) - - elif ( - index.data(QtCore.Qt.CheckStateRole) - ): - painter.fillRect(toggle_rect, check_color) - - if option.state & QtWidgets.QStyle.State_MouseOver: - painter.fillRect(body_rect, colors["hover"]) - - if option.state & QtWidgets.QStyle.State_Selected: - painter.fillRect(body_rect, colors["selected"]) - - painter.setPen(colors["outline"]) - painter.drawRect(body_rect) - - # Ok, we're done, tidy up. - painter.restore() - - def sizeHint(self, option, index): - return QtCore.QSize(option.rect.width(), 80) - - class TerminalItem(QtWidgets.QStyledItemDelegate): """Delegate used exclusively for the Terminal""" diff --git a/pype/tools/pyblish_pype/img/tab-home.png b/pype/tools/pyblish_pype/img/tab-home.png deleted file mode 100644 index 9133d06edc..0000000000 Binary files a/pype/tools/pyblish_pype/img/tab-home.png and /dev/null differ diff --git a/pype/tools/pyblish_pype/model.py b/pype/tools/pyblish_pype/model.py index 1482ff85b0..ec9689381e 100644 --- a/pype/tools/pyblish_pype/model.py +++ b/pype/tools/pyblish_pype/model.py @@ -717,15 +717,18 @@ class InstanceModel(QtGui.QStandardItemModel): def append(self, instance): new_item = InstanceItem(instance) - families = new_item.data(Roles.FamiliesRole) - group_item = self.group_items.get(families[0]) - if not group_item: - group_item = GroupItem(families[0]) - self.appendRow(group_item) - self.group_items[families[0]] = group_item - self.group_created.emit(group_item.index()) + if new_item.is_context: + self.appendRow(new_item) + else: + families = new_item.data(Roles.FamiliesRole) + group_item = self.group_items.get(families[0]) + if not group_item: + group_item = GroupItem(families[0]) + self.appendRow(group_item) + self.group_items[families[0]] = group_item + self.group_created.emit(group_item.index()) - group_item.appendRow(new_item) + group_item.appendRow(new_item) instance_id = instance.id self.instance_items[instance_id] = new_item @@ -842,162 +845,20 @@ class InstanceModel(QtGui.QStandardItemModel): ) -class ArtistProxy(QtCore.QAbstractProxyModel): +class InstanceSortProxy(QtCore.QSortFilterProxyModel): def __init__(self, *args, **kwargs): - self.mapping_from = [] - self.mapping_to = [] - super(ArtistProxy, self).__init__(*args, **kwargs) + super(InstanceSortProxy, self).__init__(*args, **kwargs) + # Do not care about lower/upper case + self.setSortCaseSensitivity(QtCore.Qt.CaseInsensitive) - def on_rows_inserted(self, parent_index, from_row, to_row): - if not parent_index.isValid(): - return - - parent_row = parent_index.row() - if parent_row >= len(self.mapping_from): - self.mapping_from.append(list()) - - new_from = None - new_to = None - for row_num in range(from_row, to_row + 1): - new_row = len(self.mapping_to) - new_to = new_row - if new_from is None: - new_from = new_row - - self.mapping_from[parent_row].insert(row_num, new_row) - self.mapping_to.insert(new_row, [parent_row, row_num]) - - self.rowsInserted.emit(self.parent(), new_from, new_to + 1) - - def _remove_rows(self, parent_row, from_row, to_row): - increment_num = self.mapping_from[parent_row][from_row] - - to_end_index = len(self.mapping_from[parent_row]) - 1 - for _idx in range(0, parent_row): - to_end_index += len(self.mapping_from[_idx]) - - removed_rows = 0 - _emit_last = None - for row_num in reversed(range(from_row, to_row + 1)): - row = self.mapping_from[parent_row].pop(row_num) - _emit_last = row - removed_rows += 1 - - _emit_first = int(increment_num) - mapping_from_len = len(self.mapping_from) - mapping_from_parent_len = len(self.mapping_from[parent_row]) - if parent_row < mapping_from_len: - for idx in range(from_row, mapping_from_parent_len): - self.mapping_from[parent_row][idx] = increment_num - increment_num += 1 - - if parent_row < mapping_from_len - 1: - for idx_i in range(parent_row + 1, mapping_from_len): - sub_values = self.mapping_from[idx_i] - if not sub_values: - continue - - for idx_j in range(0, len(sub_values)): - self.mapping_from[idx_i][idx_j] = increment_num - increment_num += 1 - - for idx in range(removed_rows): - self.mapping_to.pop(to_end_index - idx) - - return (_emit_first, _emit_last) - - def on_rows_removed(self, parent_index, from_row, to_row): - if parent_index.isValid(): - parent_row = parent_index.row() - _emit_first, _emit_last = self._remove_rows( - parent_row, from_row, to_row - ) - self.rowsRemoved.emit(self.parent(), _emit_first, _emit_last) - - else: - removed_rows = False - emit_first = None - emit_last = None - for row_num in reversed(range(from_row, to_row + 1)): - remaining_rows = self.mapping_from[row_num] - if remaining_rows: - removed_rows = True - _emit_first, _emit_last = self._remove_rows( - row_num, 0, len(remaining_rows) - 1 - ) - if emit_first is None: - emit_first = _emit_first - emit_last = _emit_last - - self.mapping_from.pop(row_num) - - diff = to_row - from_row + 1 - mapping_to_len = len(self.mapping_to) - if from_row < mapping_to_len: - for idx in range(from_row, mapping_to_len): - self.mapping_to[idx][0] -= diff - - if removed_rows: - self.rowsRemoved.emit(self.parent(), emit_first, emit_last) - - def on_reset(self): - self.modelReset.emit() - self.mapping_from = [] - self.mapping_to = [] - - def setSourceModel(self, source_model): - super(ArtistProxy, self).setSourceModel(source_model) - source_model.rowsInserted.connect(self.on_rows_inserted) - source_model.rowsRemoved.connect(self.on_rows_removed) - source_model.modelReset.connect(self.on_reset) - source_model.dataChanged.connect(self.on_data_changed) - - def on_data_changed(self, from_index, to_index, roles=None): - proxy_from_index = self.mapFromSource(from_index) - if from_index == to_index: - proxy_to_index = proxy_from_index - else: - proxy_to_index = self.mapFromSource(to_index) - - args = [proxy_from_index, proxy_to_index] - if Qt.__binding__ not in ("PyQt4", "PySide"): - args.append(roles or []) - self.dataChanged.emit(*args) - - def columnCount(self, parent=QtCore.QModelIndex()): - # This is not right for global proxy, but in this case it is enough - return self.sourceModel().columnCount() - - def rowCount(self, parent=QtCore.QModelIndex()): - if parent.isValid(): - return 0 - return len(self.mapping_to) - - def mapFromSource(self, index): - if not index.isValid(): - return QtCore.QModelIndex() - - parent_index = index.parent() - if not parent_index.isValid(): - return QtCore.QModelIndex() - - parent_idx = self.mapping_from[parent_index.row()] - my_row = parent_idx[index.row()] - return self.index(my_row, index.column()) - - def mapToSource(self, index): - if not index.isValid() or index.row() > len(self.mapping_to): - return self.sourceModel().index(index.row(), index.column()) - - parent_row, item_row = self.mapping_to[index.row()] - parent_index = self.sourceModel().index(parent_row, 0) - return self.sourceModel().index(item_row, 0, parent_index) - - def index(self, row, column, parent=QtCore.QModelIndex()): - return self.createIndex(row, column, QtCore.QModelIndex()) - - def parent(self, index=None): - return QtCore.QModelIndex() + def lessThan(self, x_index, y_index): + x_type = x_index.data(Roles.TypeRole) + y_type = y_index.data(Roles.TypeRole) + if x_type != y_type: + if x_type == GroupType: + return False + return True + return super(InstanceSortProxy, self).lessThan(x_index, y_index) class TerminalDetailItem(QtGui.QStandardItem): diff --git a/pype/tools/pyblish_pype/settings.py b/pype/tools/pyblish_pype/settings.py index fefdbea77f..5848cdf698 100644 --- a/pype/tools/pyblish_pype/settings.py +++ b/pype/tools/pyblish_pype/settings.py @@ -8,7 +8,7 @@ UseLabel = True # Customize which tab to start on. Possible choices are: "artist", "overview" # and "terminal". -InitialTab = "artist" +InitialTab = "overview" # Customize the window size. WindowSize = (430, 600) diff --git a/pype/tools/pyblish_pype/view.py b/pype/tools/pyblish_pype/view.py index b18738c9ab..7245393714 100644 --- a/pype/tools/pyblish_pype/view.py +++ b/pype/tools/pyblish_pype/view.py @@ -11,61 +11,6 @@ def _import_widgets(): from . import widgets -class ArtistView(QtWidgets.QListView): - # An item is requesting to be toggled, with optional forced-state - toggled = QtCore.Signal(QtCore.QModelIndex, object) - show_perspective = QtCore.Signal(QtCore.QModelIndex) - - def __init__(self, parent=None): - super(ArtistView, self).__init__(parent) - - self.horizontalScrollBar().hide() - self.viewport().setAttribute(QtCore.Qt.WA_Hover, True) - self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection) - self.setResizeMode(QtWidgets.QListView.Adjust) - self.setVerticalScrollMode(QtWidgets.QListView.ScrollPerPixel) - - def event(self, event): - if not event.type() == QtCore.QEvent.KeyPress: - return super(ArtistView, self).event(event) - - elif event.key() == QtCore.Qt.Key_Space: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, None) - - return True - - elif event.key() == QtCore.Qt.Key_Backspace: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, False) - - return True - - elif event.key() == QtCore.Qt.Key_Return: - for index in self.selectionModel().selectedIndexes(): - self.toggled.emit(index, True) - - return True - - return super(ArtistView, self).event(event) - - def focusOutEvent(self, event): - self.selectionModel().clear() - - def mouseReleaseEvent(self, event): - if event.button() == QtCore.Qt.LeftButton: - indexes = self.selectionModel().selectedIndexes() - if len(indexes) <= 1 and event.pos().x() < 20: - for index in indexes: - self.toggled.emit(index, None) - if len(indexes) == 1 and event.pos().x() > self.width() - 40: - for index in indexes: - self.show_perspective.emit(index) - - return super(ArtistView, self).mouseReleaseEvent(event) - - class OverviewView(QtWidgets.QTreeView): # An item is requesting to be toggled, with optional forced-state toggled = QtCore.Signal(QtCore.QModelIndex, object) @@ -160,6 +105,8 @@ class PluginView(OverviewView): class InstanceView(OverviewView): def __init__(self, *args, **kwargs): super(InstanceView, self).__init__(*args, **kwargs) + self.setSortingEnabled(True) + self.sortByColumn(0, QtCore.Qt.AscendingOrder) self.viewport().setMouseTracking(True) self._pressed_group_index = None self._pressed_expander = None diff --git a/pype/tools/pyblish_pype/window.py b/pype/tools/pyblish_pype/window.py index 2f663cc08a..df1d8687d5 100644 --- a/pype/tools/pyblish_pype/window.py +++ b/pype/tools/pyblish_pype/window.py @@ -97,7 +97,6 @@ class Window(QtWidgets.QDialog): header_widget = QtWidgets.QWidget(parent=main_widget) header_tab_widget = QtWidgets.QWidget(header_widget) - header_tab_artist = QtWidgets.QRadioButton(header_tab_widget) header_tab_overview = QtWidgets.QRadioButton(header_tab_widget) header_tab_terminal = QtWidgets.QRadioButton(header_tab_widget) header_spacer = QtWidgets.QWidget(header_tab_widget) @@ -125,7 +124,6 @@ class Window(QtWidgets.QDialog): layout_tab = QtWidgets.QHBoxLayout(header_tab_widget) layout_tab.setContentsMargins(0, 0, 0, 0) layout_tab.setSpacing(0) - layout_tab.addWidget(header_tab_artist, 0) layout_tab.addWidget(header_tab_overview, 0) layout_tab.addWidget(header_tab_terminal, 0) layout_tab.addWidget(button_suspend_logs_widget, 0) @@ -141,27 +139,6 @@ class Window(QtWidgets.QDialog): header_widget.setLayout(layout) - # Artist Page - instance_model = model.InstanceModel(controller) - - artist_page = QtWidgets.QWidget() - - artist_view = view.ArtistView() - artist_view.show_perspective.connect(self.toggle_perspective_widget) - artist_proxy = model.ArtistProxy() - artist_proxy.setSourceModel(instance_model) - artist_view.setModel(artist_proxy) - - artist_delegate = delegate.ArtistDelegate() - artist_view.setItemDelegate(artist_delegate) - - layout = QtWidgets.QVBoxLayout(artist_page) - layout.addWidget(artist_view) - layout.setContentsMargins(5, 5, 5, 5) - layout.setSpacing(0) - - artist_page.setLayout(layout) - # Overview Page # TODO add parent overview_page = QtWidgets.QWidget() @@ -172,8 +149,12 @@ class Window(QtWidgets.QDialog): overview_instance_delegate = delegate.InstanceDelegate( parent=overview_instance_view ) + instance_model = model.InstanceModel(controller) + instance_sort_proxy = model.InstanceSortProxy() + instance_sort_proxy.setSourceModel(instance_model) + overview_instance_view.setItemDelegate(overview_instance_delegate) - overview_instance_view.setModel(instance_model) + overview_instance_view.setModel(instance_sort_proxy) overview_plugin_view = view.PluginView( animated=settings.Animated, parent=overview_page @@ -223,7 +204,6 @@ class Window(QtWidgets.QDialog): body_widget = QtWidgets.QWidget(main_widget) layout = QtWidgets.QHBoxLayout(body_widget) layout.setContentsMargins(5, 5, 5, 1) - layout.addWidget(artist_page) layout.addWidget(overview_page) layout.addWidget(terminal_page) @@ -361,12 +341,10 @@ class Window(QtWidgets.QDialog): "Footer": footer_widget, # Pages - "Artist": artist_page, "Overview": overview_page, "Terminal": terminal_page, # Tabs - "ArtistTab": header_tab_artist, "OverviewTab": header_tab_overview, "TerminalTab": header_tab_terminal, @@ -399,7 +377,6 @@ class Window(QtWidgets.QDialog): pages_widget, header_widget, body_widget, - artist_page, comment_box, overview_page, terminal_page, @@ -415,9 +392,6 @@ class Window(QtWidgets.QDialog): _widget.setAttribute(QtCore.Qt.WA_StyledBackground) # Signals - header_tab_artist.toggled.connect( - lambda: self.on_tab_changed("artist") - ) header_tab_overview.toggled.connect( lambda: self.on_tab_changed("overview") ) @@ -450,7 +424,6 @@ class Window(QtWidgets.QDialog): QtCore.Qt.DirectConnection ) - artist_view.toggled.connect(self.on_instance_toggle) overview_instance_view.toggled.connect(self.on_instance_toggle) overview_plugin_view.toggled.connect(self.on_plugin_toggle) @@ -466,9 +439,7 @@ class Window(QtWidgets.QDialog): self.on_plugin_action_menu_requested ) - instance_model.group_created.connect( - overview_instance_view.expand - ) + instance_model.group_created.connect(self.on_instance_group_created) self.main_widget = main_widget @@ -490,9 +461,7 @@ class Window(QtWidgets.QDialog): self.plugin_model = plugin_model self.plugin_proxy = plugin_proxy self.instance_model = instance_model - - self.artist_proxy = artist_proxy - self.artist_view = artist_view + self.instance_sort_proxy = instance_sort_proxy self.presets_button = presets_button @@ -510,17 +479,15 @@ class Window(QtWidgets.QDialog): self.perspective_widget = perspective_widget self.tabs = { - "artist": header_tab_artist, "overview": header_tab_overview, "terminal": header_tab_terminal } self.pages = ( - ("artist", artist_page), ("overview", overview_page), ("terminal", terminal_page) ) - current_page = settings.InitialTab or "artist" + current_page = settings.InitialTab or "overview" self.comment_main_widget.setVisible( not current_page == "terminal" ) @@ -620,6 +587,10 @@ class Window(QtWidgets.QDialog): self.update_compatibility() + def on_instance_group_created(self, index): + _index = self.instance_sort_proxy.mapFromSource(index) + self.overview_instance_view.expand(_index) + def on_plugin_toggle(self, index, state=None): """An item is requesting to be toggled""" if not index.data(Roles.IsOptionalRole): @@ -1016,11 +987,14 @@ class Window(QtWidgets.QDialog): def on_passed_group(self, order): for group_item in self.instance_model.group_items.values(): - if self.overview_instance_view.isExpanded(group_item.index()): + group_index = self.instance_sort_proxy.mapFromSource( + group_item.index() + ) + if self.overview_instance_view.isExpanded(group_index): continue if group_item.publish_states & GroupStates.HasError: - self.overview_instance_view.expand(group_item.index()) + self.overview_instance_view.expand(group_index) for group_item in self.plugin_model.group_items.values(): # TODO check only plugins from the group @@ -1030,19 +1004,16 @@ class Window(QtWidgets.QDialog): if order != group_item.order: continue + group_index = self.plugin_proxy.mapFromSource(group_item.index()) if group_item.publish_states & GroupStates.HasError: - self.overview_plugin_view.expand( - self.plugin_proxy.mapFromSource(group_item.index()) - ) + self.overview_plugin_view.expand(group_index) continue group_item.setData( {GroupStates.HasFinished: True}, Roles.PublishFlagsRole ) - self.overview_plugin_view.collapse( - self.plugin_proxy.mapFromSource(group_item.index()) - ) + self.overview_plugin_view.collapse(group_index) def on_was_stopped(self): errored = self.controller.errored @@ -1122,11 +1093,6 @@ class Window(QtWidgets.QDialog): for instance_id in existing_ids: self.instance_model.remove(instance_id) - if result.get("error"): - # Toggle from artist to overview tab on error - if self.tabs["artist"].isChecked(): - self.tabs["overview"].toggle() - result["records"] = self.terminal_model.prepare_records( result, self._suspend_logs @@ -1274,7 +1240,6 @@ class Window(QtWidgets.QDialog): self.terminal_proxy.deleteLater() self.plugin_proxy.deleteLater() - self.artist_view.setModel(None) self.overview_instance_view.setModel(None) self.overview_plugin_view.setModel(None) self.terminal_view.setModel(None) diff --git a/pype/version.py b/pype/version.py index 8f57b15e91..4de2b4e35d 100644 --- a/pype/version.py +++ b/pype/version.py @@ -1 +1 @@ -__version__ = "2.13.4" +__version__ = "2.13.6"