From 3618e8f856859106714bc9c550af7ac8aac9f8c6 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:10:23 +0200 Subject: [PATCH 01/16] create formatting function for file sizes 'format_file_size' --- openpype/lib/path_tools.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 4f28be3302..f807917f5b 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -14,6 +14,27 @@ from .profiles_filtering import filter_profiles log = logging.getLogger(__name__) +def format_file_size(file_size, suffix=None): + """Returns formatted string with size in appropriate unit. + + Args: + file_size (int): Size of file in bytes. + suffix (str): Suffix for formatted size. Default is 'B' (as bytes). + + Returns: + str: Formatted size using proper unit and passed suffix (e.g. 7 MiB). + """ + + if suffix is None: + suffix = "B" + + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(file_size) < 1024.0: + return "%3.1f%s%s" % (file_size, unit, suffix) + file_size /= 1024.0 + return "%.1f%s%s" % (file_size, "Yi", suffix) + + def create_hard_link(src_path, dst_path): """Create hardlink of file. From 6398f021092d2de440218acec0d8a024aa55d75d Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:10:48 +0200 Subject: [PATCH 02/16] copied function to collect frames 'collect_frames' --- openpype/lib/path_tools.py | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index f807917f5b..45aa54d6cb 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -6,6 +6,8 @@ import logging import six import platform +import clique + from openpype.client import get_project from openpype.settings import get_project_settings @@ -71,6 +73,43 @@ def create_hard_link(src_path, dst_path): ) +def collect_frames(files): + """Returns dict of source path and its frame, if from sequence + + Uses clique as most precise solution, used when anatomy template that + created files is not known. + + Assumption is that frames are separated by '.', negative frames are not + allowed. + + Args: + files(list) or (set with single value): list of source paths + + Returns: + (dict): {'/asset/subset_v001.0001.png': '0001', ....} + """ + + patterns = [clique.PATTERNS["frames"]] + collections, remainder = clique.assemble( + files, minimum_items=1, patterns=patterns) + + sources_and_frames = {} + if collections: + for collection in collections: + src_head = collection.head + src_tail = collection.tail + + for index in collection.indexes: + src_frame = collection.format("{padding}") % index + src_file_name = "{}{}{}".format( + src_head, src_frame, src_tail) + sources_and_frames[src_file_name] = src_frame + else: + sources_and_frames[remainder.pop()] = None + + return sources_and_frames + + def _rreplace(s, a, b, n=1): """Replace a with b in string s from right side n times.""" return b.join(s.rsplit(a, n)) From c26119cc9f6a9fe4c330842f9dddbf7865a63425 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:13:08 +0200 Subject: [PATCH 03/16] use new functions in code --- openpype/lib/__init__.py | 4 ++++ .../publish/submit_aftereffects_deadline.py | 6 ++++-- .../validate_expected_and_rendered_files.py | 2 +- .../action_delete_old_versions.py | 17 +++++++---------- openpype/plugins/load/delete_old_versions.py | 11 ++--------- openpype/plugins/load/delivery.py | 11 +++++++---- 6 files changed, 25 insertions(+), 26 deletions(-) diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py index adb857a056..17aafc3e8b 100644 --- a/openpype/lib/__init__.py +++ b/openpype/lib/__init__.py @@ -192,6 +192,8 @@ from .plugin_tools import ( ) from .path_tools import ( + format_file_size, + collect_frames, create_hard_link, version_up, get_version_from_path, @@ -353,6 +355,8 @@ __all__ = [ "set_plugin_attributes_from_settings", "source_hash", + "format_file_size", + "collect_frames", "create_hard_link", "version_up", "get_version_from_path", diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py index c55f85c8da..1d68793d53 100644 --- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py +++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py @@ -3,8 +3,10 @@ import attr import getpass import pyblish.api -from openpype.lib import env_value_to_bool -from openpype.lib.delivery import collect_frames +from openpype.lib import ( + env_value_to_bool, + collect_frames, +) from openpype.pipeline import legacy_io from openpype_modules.deadline import abstract_submit_deadline from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py index c2426e0d78..f0a3ddd246 100644 --- a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py +++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py @@ -3,7 +3,7 @@ import requests import pyblish.api -from openpype.lib.delivery import collect_frames +from openpype.lib import collect_frames from openpype_modules.deadline.abstract_submit_deadline import requests_get diff --git a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py index 79d04a7854..c543dc8834 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py @@ -11,7 +11,11 @@ from openpype.client import ( get_versions, get_representations ) -from openpype.lib import StringTemplate, TemplateUnsolved +from openpype.lib import ( + StringTemplate, + TemplateUnsolved, + format_file_size, +) from openpype.pipeline import AvalonMongoDB, Anatomy from openpype_modules.ftrack.lib import BaseAction, statics_icon @@ -134,13 +138,6 @@ class DeleteOldVersions(BaseAction): "title": self.inteface_title } - def sizeof_fmt(self, num, suffix='B'): - for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Yi', suffix) - def launch(self, session, entities, event): values = event["data"].get("values") if not values: @@ -359,7 +356,7 @@ class DeleteOldVersions(BaseAction): dir_paths, file_paths_by_dir, delete=False ) - msg = "Total size of files: " + self.sizeof_fmt(size) + msg = "Total size of files: {}".format(format_file_size(size)) self.log.warning(msg) @@ -430,7 +427,7 @@ class DeleteOldVersions(BaseAction): "message": msg } - msg = "Total size of files deleted: " + self.sizeof_fmt(size) + msg = "Total size of files deleted: {}".format(format_file_size(size)) self.log.warning(msg) diff --git a/openpype/plugins/load/delete_old_versions.py b/openpype/plugins/load/delete_old_versions.py index 6e0b464cc1..ce6f204c64 100644 --- a/openpype/plugins/load/delete_old_versions.py +++ b/openpype/plugins/load/delete_old_versions.py @@ -10,7 +10,7 @@ from Qt import QtWidgets, QtCore from openpype.client import get_versions, get_representations from openpype import style from openpype.pipeline import load, AvalonMongoDB, Anatomy -from openpype.lib import StringTemplate +from openpype.lib import StringTemplate, format_file_size from openpype.modules import ModulesManager @@ -38,13 +38,6 @@ class DeleteOldVersions(load.SubsetLoaderPlugin): ) ] - def sizeof_fmt(self, num, suffix='B'): - for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Yi', suffix) - def delete_whole_dir_paths(self, dir_paths, delete=True): size = 0 @@ -456,7 +449,7 @@ class DeleteOldVersions(load.SubsetLoaderPlugin): size += self.main(project_name, data, remove_publish_folder) print("Progressing {}/{}".format(count + 1, len(contexts))) - msg = "Total size of files: " + self.sizeof_fmt(size) + msg = "Total size of files: {}".format(format_file_size(size)) self.log.info(msg) self.message(msg) diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index f6e1d4f06b..2a9f25e0fb 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -7,15 +7,17 @@ from openpype.client import get_representations from openpype.pipeline import load, Anatomy from openpype import resources, style +from openpype.lib import ( + format_file_size, + collect_frames, +) from openpype.lib.dateutils import get_datetime_data from openpype.lib.delivery import ( - sizeof_fmt, path_from_representation, get_format_dict, check_destination_path, process_single_file, process_sequence, - collect_frames ) @@ -263,8 +265,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): def _prepare_label(self): """Provides text with no of selected files and their size.""" - label = "{} files, size {}".format(self.files_selected, - sizeof_fmt(self.size_selected)) + label = "{} files, size {}".format( + self.files_selected, + format_file_size(self.size_selected)) return label def _get_selected_repres(self): From aeb30b3101c31f8965e80cf40287f5e0d4e4dfe9 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:13:21 +0200 Subject: [PATCH 04/16] marked functions in delivery as deprecated --- openpype/lib/delivery.py | 78 ++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index ffcfe9fa4d..5244187354 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -4,6 +4,8 @@ import shutil import glob import clique import collections +import functools +import warnings from .path_templates import ( StringTemplate, @@ -11,6 +13,52 @@ from .path_templates import ( ) +class DeliveryDeprecatedWarning(DeprecationWarning): + pass + + +def deprecated(new_destination): + """Mark functions as deprecated. + + It will result in a warning being emitted when the function is used. + """ + + func = None + if callable(new_destination): + func = new_destination + new_destination = None + + def _decorator(decorated_func): + if new_destination is None: + warning_message = ( + " Please check content of deprecated function to figure out" + " possible replacement." + ) + else: + warning_message = " Please replace your usage with '{}'.".format( + new_destination + ) + + @functools.wraps(decorated_func) + def wrapper(*args, **kwargs): + warnings.simplefilter("always", DeliveryDeprecatedWarning) + warnings.warn( + ( + "Call to deprecated function '{}'" + "\nFunction was moved or removed.{}" + ).format(decorated_func.__name__, warning_message), + category=DeliveryDeprecatedWarning, + stacklevel=4 + ) + return decorated_func(*args, **kwargs) + return wrapper + + if func is None: + return _decorator + return _decorator(func) + + +@deprecated("openpype.lib.path_tools.collect_frames") def collect_frames(files): """ Returns dict of source path and its frame, if from sequence @@ -26,34 +74,18 @@ def collect_frames(files): Returns: (dict): {'/asset/subset_v001.0001.png': '0001', ....} """ - patterns = [clique.PATTERNS["frames"]] - collections, remainder = clique.assemble(files, minimum_items=1, - patterns=patterns) - sources_and_frames = {} - if collections: - for collection in collections: - src_head = collection.head - src_tail = collection.tail + from .path_tools import collect_frames - for index in collection.indexes: - src_frame = collection.format("{padding}") % index - src_file_name = "{}{}{}".format(src_head, src_frame, - src_tail) - sources_and_frames[src_file_name] = src_frame - else: - sources_and_frames[remainder.pop()] = None - - return sources_and_frames + return collect_frames(files) -def sizeof_fmt(num, suffix='B'): +@deprecated("openpype.lib.path_tools.format_file_size") +def sizeof_fmt(num, suffix=None): """Returns formatted string with size in appropriate unit""" - for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: - if abs(num) < 1024.0: - return "%3.1f%s%s" % (num, unit, suffix) - num /= 1024.0 - return "%.1f%s%s" % (num, 'Yi', suffix) + + from .path_tools import format_file_size + return format_file_size(num, suffix) def path_from_representation(representation, anatomy): From d58ea894159cb1190fb5bcad5cdf4e949adf39f7 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:26:23 +0200 Subject: [PATCH 05/16] implemented 'get_representation_path_with_anatomy'. --- openpype/pipeline/load/__init__.py | 5 +++ openpype/pipeline/load/utils.py | 62 +++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/openpype/pipeline/load/__init__.py b/openpype/pipeline/load/__init__.py index b6bdd13d50..4fc8ad1d16 100644 --- a/openpype/pipeline/load/__init__.py +++ b/openpype/pipeline/load/__init__.py @@ -1,6 +1,8 @@ from .utils import ( HeroVersionType, + IncompatibleLoaderError, + InvalidRepresentationContext, get_repres_contexts, get_subset_contexts, @@ -20,6 +22,7 @@ from .utils import ( get_representation_path_from_context, get_representation_path, + get_representation_path_with_anatomy, is_compatible_loader, @@ -46,7 +49,9 @@ from .plugins import ( __all__ = ( # utils.py "HeroVersionType", + "IncompatibleLoaderError", + "InvalidRepresentationContext", "get_repres_contexts", "get_subset_contexts", diff --git a/openpype/pipeline/load/utils.py b/openpype/pipeline/load/utils.py index 99d6876d4b..d4a5c2be5a 100644 --- a/openpype/pipeline/load/utils.py +++ b/openpype/pipeline/load/utils.py @@ -23,10 +23,16 @@ from openpype.client import ( get_representation_by_name, get_representation_parents ) +from openpype.lib import ( + StringTemplate, + TemplateUnsolved, +) from openpype.pipeline import ( schema, legacy_io, Anatomy, + registered_root, + registered_host, ) log = logging.getLogger(__name__) @@ -61,6 +67,11 @@ class IncompatibleLoaderError(ValueError): pass +class InvalidRepresentationContext(ValueError): + """Representation path can't be received using representation document.""" + pass + + def get_repres_contexts(representation_ids, dbcon=None): """Return parenthood context for representation. @@ -515,6 +526,52 @@ def get_representation_path_from_context(context): return get_representation_path(representation, root) +def get_representation_path_with_anatomy(repre_doc, anatomy): + """Receive representation path using representation document and anatomy. + + Anatomy is used to replace 'root' key in representation file. Ideally + should be used instead of 'get_representation_path' which is based on + "current context". + + Future notes: + We want also be able store resources into representation and I can + imagine the result should also contain paths to possible resources. + + Args: + repre_doc (Dict[str, Any]): Representation document. + anatomy (Anatomy): Project anatomy object. + + Returns: + Union[None, TemplateResult]: None if path can't be received + + Raises: + InvalidRepresentationContext: When representation data are probably + invalid or not available. + """ + + try: + template = repre_doc["data"]["template"] + + except KeyError: + raise InvalidRepresentationContext(( + "Representation document does not" + " contain template in data ('data.template')" + )) + + try: + context = repre_doc["context"] + context["root"] = anatomy.roots + path = StringTemplate.format_strict_template(template, context) + + except TemplateUnsolved as exc: + raise InvalidRepresentationContext(( + "Couldn't resolve representation template with available data." + " Reason: {}".format(str(exc)) + )) + + return path.normalized() + + def get_representation_path(representation, root=None, dbcon=None): """Get filename from representation document @@ -533,14 +590,10 @@ def get_representation_path(representation, root=None, dbcon=None): """ - from openpype.lib import StringTemplate, TemplateUnsolved - if dbcon is None: dbcon = legacy_io if root is None: - from openpype.pipeline import registered_root - root = registered_root() def path_from_represenation(): @@ -736,7 +789,6 @@ def get_outdated_containers(host=None, project_name=None): """ if host is None: - from openpype.pipeline import registered_host host = registered_host() if project_name is None: From 315cf40d8baa47bb3a4f6864e49551bdfc6d196b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:29:43 +0200 Subject: [PATCH 06/16] fixed import in load utils --- openpype/pipeline/load/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/load/utils.py b/openpype/pipeline/load/utils.py index d4a5c2be5a..83b904e4a7 100644 --- a/openpype/pipeline/load/utils.py +++ b/openpype/pipeline/load/utils.py @@ -31,8 +31,6 @@ from openpype.pipeline import ( schema, legacy_io, Anatomy, - registered_root, - registered_host, ) log = logging.getLogger(__name__) @@ -594,6 +592,8 @@ def get_representation_path(representation, root=None, dbcon=None): dbcon = legacy_io if root is None: + from openpype.pipeline import registered_root + root = registered_root() def path_from_represenation(): @@ -789,6 +789,8 @@ def get_outdated_containers(host=None, project_name=None): """ if host is None: + from openpype.pipeline import registered_host + host = registered_host() if project_name is None: From f2a191861b9264383da3c0b63ed8f4feac629a1c Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:41:28 +0200 Subject: [PATCH 07/16] marked 'path_from_representation' as deprecated and replaced it's usage with 'get_representation_path_with_anatomy' --- openpype/lib/delivery.py | 29 +++++-------- .../event_handlers_user/action_delivery.py | 6 +-- openpype/pipeline/load/__init__.py | 1 + openpype/plugins/load/delete_old_versions.py | 41 +++++++++++-------- openpype/plugins/load/delivery.py | 8 ++-- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index 5244187354..ea757932c9 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -7,11 +7,6 @@ import collections import functools import warnings -from .path_templates import ( - StringTemplate, - TemplateUnsolved, -) - class DeliveryDeprecatedWarning(DeprecationWarning): pass @@ -88,24 +83,22 @@ def sizeof_fmt(num, suffix=None): return format_file_size(num, suffix) +@deprecated("openpype.pipeline.load.get_representation_path_with_anatomy") def path_from_representation(representation, anatomy): - try: - template = representation["data"]["template"] + """Get representation path using representation document and anatomy. - except KeyError: - return None + Args: + representation (Dict[str, Any]): Representation document. + anatomy (Anatomy): Project anatomy. - try: - context = representation["context"] - context["root"] = anatomy.roots - path = StringTemplate.format_strict_template(template, context) - return os.path.normpath(path) + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. + """ - except TemplateUnsolved: - # Template references unavailable data - return None + from openpype.pipeline.load import get_representation_path_with_anatomy - return path + return get_representation_path_with_anatomy(representation, anatomy) def copy_file(src_path, dst_path): diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index eec245070c..59a34b3f85 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -10,15 +10,15 @@ from openpype.client import ( get_versions, get_representations ) -from openpype.pipeline import Anatomy from openpype_modules.ftrack.lib import BaseAction, statics_icon from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY from openpype_modules.ftrack.lib.custom_attributes import ( query_custom_attributes ) from openpype.lib.dateutils import get_datetime_data +from openpype.pipeline import Anatomy +from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.lib.delivery import ( - path_from_representation, get_format_dict, check_destination_path, process_single_file, @@ -580,7 +580,7 @@ class Delivery(BaseAction): if frame: repre["context"]["frame"] = len(str(frame)) * "#" - repre_path = path_from_representation(repre, anatomy) + repre_path = get_representation_path_with_anatomy(repre, anatomy) # TODO add backup solution where root of path from component # is replaced with root args = ( diff --git a/openpype/pipeline/load/__init__.py b/openpype/pipeline/load/__init__.py index 4fc8ad1d16..bf38a0b3c8 100644 --- a/openpype/pipeline/load/__init__.py +++ b/openpype/pipeline/load/__init__.py @@ -71,6 +71,7 @@ __all__ = ( "get_representation_path_from_context", "get_representation_path", + "get_representation_path_with_anatomy", "is_compatible_loader", diff --git a/openpype/plugins/load/delete_old_versions.py b/openpype/plugins/load/delete_old_versions.py index ce6f204c64..8c8546d9c8 100644 --- a/openpype/plugins/load/delete_old_versions.py +++ b/openpype/plugins/load/delete_old_versions.py @@ -7,11 +7,15 @@ from pymongo import UpdateOne import qargparse from Qt import QtWidgets, QtCore -from openpype.client import get_versions, get_representations from openpype import style -from openpype.pipeline import load, AvalonMongoDB, Anatomy -from openpype.lib import StringTemplate, format_file_size +from openpype.client import get_versions, get_representations from openpype.modules import ModulesManager +from openpype.lib import StringTemplate, format_file_size +from openpype.pipeline import load, AvalonMongoDB, Anatomy +from openpype.pipeline.load import ( + get_representation_path_with_anatomy, + InvalidRepresentationContext, +) class DeleteOldVersions(load.SubsetLoaderPlugin): @@ -73,27 +77,28 @@ class DeleteOldVersions(load.SubsetLoaderPlugin): def path_from_representation(self, representation, anatomy): try: - template = representation["data"]["template"] - + context = representation["context"] except KeyError: return (None, None) + try: + path = get_representation_path_with_anatomy( + representation, anatomy + ) + except InvalidRepresentationContext: + return (None, None) + sequence_path = None - try: - context = representation["context"] - context["root"] = anatomy.roots - path = str(StringTemplate.format_template(template, context)) - if "frame" in context: - context["frame"] = self.sequence_splitter - sequence_path = os.path.normpath(str( - StringTemplate.format_template(template, context) - )) + if "frame" in context: + context["frame"] = self.sequence_splitter + sequence_path = get_representation_path_with_anatomy( + representation, anatomy + ) - except KeyError: - # Template references unavailable data - return (None, None) + if sequence_path: + sequence_path = sequence_path.normalized() - return (os.path.normpath(path), sequence_path) + return (path.normalized(), sequence_path) def delete_only_repre_files(self, dir_paths, file_paths, delete=True): size = 0 diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index 2a9f25e0fb..4651efd4a3 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -10,10 +10,10 @@ from openpype import resources, style from openpype.lib import ( format_file_size, collect_frames, + get_datetime_data, ) -from openpype.lib.dateutils import get_datetime_data +from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.lib.delivery import ( - path_from_representation, get_format_dict, check_destination_path, process_single_file, @@ -169,7 +169,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): if repre["name"] not in selected_repres: continue - repre_path = path_from_representation(repre, self.anatomy) + repre_path = get_representation_path_with_anatomy( + repre, self.anatomy + ) anatomy_data = copy.deepcopy(repre["context"]) new_report_items = check_destination_path(str(repre["_id"]), From ea241ca807837896ac1f8299ec0c6c6bbb1020ad Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:41:44 +0200 Subject: [PATCH 08/16] added some docstrings to deprecated functions --- openpype/lib/delivery.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index ea757932c9..e09188d3bb 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -55,19 +55,23 @@ def deprecated(new_destination): @deprecated("openpype.lib.path_tools.collect_frames") def collect_frames(files): - """ - Returns dict of source path and its frame, if from sequence + """Returns dict of source path and its frame, if from sequence - Uses clique as most precise solution, used when anatomy template that - created files is not known. + Uses clique as most precise solution, used when anatomy template that + created files is not known. - Assumption is that frames are separated by '.', negative frames are not - allowed. + Assumption is that frames are separated by '.', negative frames are not + allowed. - Args: - files(list) or (set with single value): list of source paths - Returns: - (dict): {'/asset/subset_v001.0001.png': '0001', ....} + Args: + files(list) or (set with single value): list of source paths + + Returns: + (dict): {'/asset/subset_v001.0001.png': '0001', ....} + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. """ from .path_tools import collect_frames @@ -77,7 +81,12 @@ def collect_frames(files): @deprecated("openpype.lib.path_tools.format_file_size") def sizeof_fmt(num, suffix=None): - """Returns formatted string with size in appropriate unit""" + """Returns formatted string with size in appropriate unit + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. + """ from .path_tools import format_file_size return format_file_size(num, suffix) From 14dc209ab0a42d799cfa37eebd08d090666b537f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 14:53:52 +0200 Subject: [PATCH 09/16] 'get_project_template_data' can access project name --- openpype/pipeline/template_data.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/openpype/pipeline/template_data.py b/openpype/pipeline/template_data.py index 824a25127c..bab46a627d 100644 --- a/openpype/pipeline/template_data.py +++ b/openpype/pipeline/template_data.py @@ -28,27 +28,37 @@ def get_general_template_data(system_settings=None): } -def get_project_template_data(project_doc): +def get_project_template_data(project_doc=None, project_name=None): """Extract data from project document that are used in templates. Project document must have 'name' and (at this moment) optional key 'data.code'. + One of 'project_name' or 'project_doc' must be passed. With prepared + project document is function much faster because don't have to query. + Output contains formatting keys: - 'project[name]' - Project name - 'project[code]' - Project code Args: project_doc (Dict[str, Any]): Queried project document. + project_name (str): Name of project. Returns: Dict[str, Dict[str, str]]: Template data based on project document. """ + if not project_name: + project_name = project_doc["name"] + + if not project_doc: + project_code = get_project(project_name, fields=["data.code"]) + project_code = project_doc.get("data", {}).get("code") return { "project": { - "name": project_doc["name"], + "name": project_name, "code": project_code } } From e2060b9d65e4b7224fff916b4f492de0d015e9bf Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:00:08 +0200 Subject: [PATCH 10/16] marked 'get_format_dict' as deprecated and moved it to pipeline delivery --- openpype/lib/delivery.py | 33 +++++++++---------- .../event_handlers_user/action_delivery.py | 4 ++- openpype/pipeline/delivery.py | 26 +++++++++++++++ openpype/plugins/load/delivery.py | 4 ++- 4 files changed, 47 insertions(+), 20 deletions(-) create mode 100644 openpype/pipeline/delivery.py diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index e09188d3bb..1e364c45d7 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -125,28 +125,25 @@ def copy_file(src_path, dst_path): shutil.copyfile(src_path, dst_path) +@deprecated("openpype.pipeline.delivery.get_format_dict") def get_format_dict(anatomy, location_path): """Returns replaced root values from user provider value. - Args: - anatomy (Anatomy) - location_path (str): user provided value - Returns: - (dict): prepared for formatting of a template + Args: + anatomy (Anatomy) + location_path (str): user provided value + + Returns: + (dict): prepared for formatting of a template + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. """ - format_dict = {} - if location_path: - location_path = location_path.replace("\\", "/") - root_names = anatomy.root_names_from_templates( - anatomy.templates["delivery"] - ) - if root_names is None: - format_dict["root"] = location_path - else: - format_dict["root"] = {} - for name in root_names: - format_dict["root"][name] = location_path - return format_dict + + from openpype.pipeline.delivery import get_format_dict + + return get_format_dict(anatomy, location_path) def check_destination_path(repre_id, diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index 59a34b3f85..08d6e53078 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -18,8 +18,10 @@ from openpype_modules.ftrack.lib.custom_attributes import ( from openpype.lib.dateutils import get_datetime_data from openpype.pipeline import Anatomy from openpype.pipeline.load import get_representation_path_with_anatomy -from openpype.lib.delivery import ( +from openpype.pipeline.delivery import ( get_format_dict, +) +from openpype.lib.delivery import ( check_destination_path, process_single_file, process_sequence diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py new file mode 100644 index 0000000000..03319f7ddc --- /dev/null +++ b/openpype/pipeline/delivery.py @@ -0,0 +1,26 @@ +"""Functions useful for delivery of published representations.""" + + +def get_format_dict(anatomy, location_path): + """Returns replaced root values from user provider value. + + Args: + anatomy (Anatomy): Project anatomy. + location_path (str): User provided value. + + Returns: + (dict): Prepared data for formatting of a template. + """ + + format_dict = {} + if not location_path: + return format_dict + + location_path = location_path.replace("\\", "/") + root_names = anatomy.root_names_from_templates( + anatomy.templates["delivery"] + ) + format_dict["root"] = {} + for name in root_names: + format_dict["root"][name] = location_path + return format_dict diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index 4651efd4a3..0ea62510a4 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -13,8 +13,10 @@ from openpype.lib import ( get_datetime_data, ) from openpype.pipeline.load import get_representation_path_with_anatomy -from openpype.lib.delivery import ( +from openpype.pipeline.delivery import ( get_format_dict, +) +from openpype.lib.delivery import ( check_destination_path, process_single_file, process_sequence, From fe566f4a4b1f5695e73731927601a814a330d8ae Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:00:57 +0200 Subject: [PATCH 11/16] copied 'copy_file' to 'pipeline.delivery' and renamed to '_copy_file' --- openpype/pipeline/delivery.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py index 03319f7ddc..5906892d59 100644 --- a/openpype/pipeline/delivery.py +++ b/openpype/pipeline/delivery.py @@ -1,4 +1,26 @@ """Functions useful for delivery of published representations.""" +import os +import shutil + +from openpype.lib import create_hard_link + + +def _copy_file(src_path, dst_path): + """Hardlink file if possible(to save space), copy if not. + + Because of using hardlinks should not be function used in other parts + of pipeline. + """ + + if os.path.exists(dst_path): + return + try: + create_hard_link( + src_path, + dst_path + ) + except OSError: + shutil.copyfile(src_path, dst_path) def get_format_dict(anatomy, location_path): From dc77d4a60908729f3b7ce343216b9af9852f1912 Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:04:03 +0200 Subject: [PATCH 12/16] marked 'check_destination_path' as deprecated and moved to pipeline.delivery --- openpype/lib/delivery.py | 65 +++++++------------ .../event_handlers_user/action_delivery.py | 2 +- openpype/pipeline/delivery.py | 61 +++++++++++++++++ openpype/plugins/load/delivery.py | 2 +- 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index 1e364c45d7..543c3d12e5 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -3,7 +3,6 @@ import os import shutil import glob import clique -import collections import functools import warnings @@ -146,56 +145,38 @@ def get_format_dict(anatomy, location_path): return get_format_dict(anatomy, location_path) +@deprecated("openpype.pipeline.delivery.check_destination_path") def check_destination_path(repre_id, anatomy, anatomy_data, datetime_data, template_name): """ Try to create destination path based on 'template_name'. - In the case that path cannot be filled, template contains unmatched - keys, provide error message to filter out repre later. + In the case that path cannot be filled, template contains unmatched + keys, provide error message to filter out repre later. - Args: - anatomy (Anatomy) - anatomy_data (dict): context to fill anatomy - datetime_data (dict): values with actual date - template_name (str): to pick correct delivery template - Returns: - (collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"} + Args: + anatomy (Anatomy) + anatomy_data (dict): context to fill anatomy + datetime_data (dict): values with actual date + template_name (str): to pick correct delivery template + + Returns: + (collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"} + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. """ - anatomy_data.update(datetime_data) - anatomy_filled = anatomy.format_all(anatomy_data) - dest_path = anatomy_filled["delivery"][template_name] - report_items = collections.defaultdict(list) - if not dest_path.solved: - msg = ( - "Missing keys in Representation's context" - " for anatomy template \"{}\"." - ).format(template_name) + from openpype.pipeline.delivery import check_destination_path - sub_msg = ( - "Representation: {}
" - ).format(repre_id) - - if dest_path.missing_keys: - keys = ", ".join(dest_path.missing_keys) - sub_msg += ( - "- Missing keys: \"{}\"
" - ).format(keys) - - if dest_path.invalid_types: - items = [] - for key, value in dest_path.invalid_types.items(): - items.append("\"{}\" {}".format(key, str(value))) - - keys = ", ".join(items) - sub_msg += ( - "- Invalid value DataType: \"{}\"
" - ).format(keys) - - report_items[msg].append(sub_msg) - - return report_items + return check_destination_path( + repre_id, + anatomy, + anatomy_data, + datetime_data, + template_name + ) def process_single_file( diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index 08d6e53078..8b314d8f1d 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -20,9 +20,9 @@ from openpype.pipeline import Anatomy from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.pipeline.delivery import ( get_format_dict, + check_destination_path, ) from openpype.lib.delivery import ( - check_destination_path, process_single_file, process_sequence ) diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py index 5906892d59..79667161a6 100644 --- a/openpype/pipeline/delivery.py +++ b/openpype/pipeline/delivery.py @@ -1,6 +1,7 @@ """Functions useful for delivery of published representations.""" import os import shutil +import collections from openpype.lib import create_hard_link @@ -46,3 +47,63 @@ def get_format_dict(anatomy, location_path): for name in root_names: format_dict["root"][name] = location_path return format_dict + + +def check_destination_path( + repre_id, + anatomy, + anatomy_data, + datetime_data, + template_name +): + """ Try to create destination path based on 'template_name'. + + In the case that path cannot be filled, template contains unmatched + keys, provide error message to filter out repre later. + + Args: + repre_id (str): Representation id. + anatomy (Anatomy): Project anatomy. + anatomy_data (dict): Template data to fill anatomy templates. + datetime_data (dict): Values with actual date. + template_name (str): Name of template which should be used from anatomy + templates. + Returns: + Dict[str, List[str]]: Report of happened errors. Key is message title + value is detailed information. + """ + + anatomy_data.update(datetime_data) + anatomy_filled = anatomy.format_all(anatomy_data) + dest_path = anatomy_filled["delivery"][template_name] + report_items = collections.defaultdict(list) + + if not dest_path.solved: + msg = ( + "Missing keys in Representation's context" + " for anatomy template \"{}\"." + ).format(template_name) + + sub_msg = ( + "Representation: {}
" + ).format(repre_id) + + if dest_path.missing_keys: + keys = ", ".join(dest_path.missing_keys) + sub_msg += ( + "- Missing keys: \"{}\"
" + ).format(keys) + + if dest_path.invalid_types: + items = [] + for key, value in dest_path.invalid_types.items(): + items.append("\"{}\" {}".format(key, str(value))) + + keys = ", ".join(items) + sub_msg += ( + "- Invalid value DataType: \"{}\"
" + ).format(keys) + + report_items[msg].append(sub_msg) + + return report_items diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index 0ea62510a4..1161636cb7 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -15,9 +15,9 @@ from openpype.lib import ( from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.pipeline.delivery import ( get_format_dict, + check_destination_path, ) from openpype.lib.delivery import ( - check_destination_path, process_single_file, process_sequence, ) From eaff50b23e29dcb142557e3068c3031e1f1e268a Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:07:45 +0200 Subject: [PATCH 13/16] Marked 'process_single_file' as deprecated and moved to pipeline delivery as 'deliver_single_file' --- openpype/lib/delivery.py | 61 +++++++------------ .../event_handlers_user/action_delivery.py | 4 +- openpype/pipeline/delivery.py | 59 ++++++++++++++++++ openpype/plugins/load/delivery.py | 6 +- 4 files changed, 87 insertions(+), 43 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index 543c3d12e5..455401d0fd 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -179,53 +179,38 @@ def check_destination_path(repre_id, ) +@deprecated("openpype.pipeline.delivery.deliver_single_file") def process_single_file( src_path, repre, anatomy, template_name, anatomy_data, format_dict, report_items, log ): """Copy single file to calculated path based on template - Args: - src_path(str): path of source representation file - _repre (dict): full repre, used only in process_sequence, here only - as to share same signature - anatomy (Anatomy) - template_name (string): user selected delivery template name - anatomy_data (dict): data from repre to fill anatomy with - format_dict (dict): root dictionary with names and values - report_items (collections.defaultdict): to return error messages - log (Logger): for log printing - Returns: - (collections.defaultdict , int) + Args: + src_path(str): path of source representation file + _repre (dict): full repre, used only in process_sequence, here only + as to share same signature + anatomy (Anatomy) + template_name (string): user selected delivery template name + anatomy_data (dict): data from repre to fill anatomy with + format_dict (dict): root dictionary with names and values + report_items (collections.defaultdict): to return error messages + log (Logger): for log printing + + Returns: + (collections.defaultdict , int) + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. """ - # Make sure path is valid for all platforms - src_path = os.path.normpath(src_path.replace("\\", "/")) - if not os.path.exists(src_path): - msg = "{} doesn't exist for {}".format(src_path, repre["_id"]) - report_items["Source file was not found"].append(msg) - return report_items, 0 + from openpype.pipeline.delivery import deliver_single_file - anatomy_filled = anatomy.format(anatomy_data) - if format_dict: - template_result = anatomy_filled["delivery"][template_name] - delivery_path = template_result.rootless.format(**format_dict) - else: - delivery_path = anatomy_filled["delivery"][template_name] - - # Backwards compatibility when extension contained `.` - delivery_path = delivery_path.replace("..", ".") - # Make sure path is valid for all platforms - delivery_path = os.path.normpath(delivery_path.replace("\\", "/")) - - delivery_folder = os.path.dirname(delivery_path) - if not os.path.exists(delivery_folder): - os.makedirs(delivery_folder) - - log.debug("Copying single: {} -> {}".format(src_path, delivery_path)) - copy_file(src_path, delivery_path) - - return report_items, 1 + return deliver_single_file( + src_path, repre, anatomy, template_name, anatomy_data, format_dict, + report_items, log + ) def process_sequence( diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index 8b314d8f1d..fe91670c3d 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -21,9 +21,9 @@ from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.pipeline.delivery import ( get_format_dict, check_destination_path, + deliver_single_file, ) from openpype.lib.delivery import ( - process_single_file, process_sequence ) @@ -596,7 +596,7 @@ class Delivery(BaseAction): self.log ) if not frame: - process_single_file(*args) + deliver_single_file(*args) else: process_sequence(*args) diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py index 79667161a6..7c5121aa53 100644 --- a/openpype/pipeline/delivery.py +++ b/openpype/pipeline/delivery.py @@ -1,6 +1,8 @@ """Functions useful for delivery of published representations.""" import os import shutil +import glob +import clique import collections from openpype.lib import create_hard_link @@ -107,3 +109,60 @@ def check_destination_path( report_items[msg].append(sub_msg) return report_items + + +def deliver_single_file( + src_path, + repre, + anatomy, + template_name, + anatomy_data, + format_dict, + report_items, + log +): + """Copy single file to calculated path based on template + + Args: + src_path(str): path of source representation file + repre (dict): full repre, used only in process_sequence, here only + as to share same signature + anatomy (Anatomy) + template_name (string): user selected delivery template name + anatomy_data (dict): data from repre to fill anatomy with + format_dict (dict): root dictionary with names and values + report_items (collections.defaultdict): to return error messages + log (logging.Logger): for log printing + + Returns: + (collections.defaultdict, int) + """ + + # Make sure path is valid for all platforms + src_path = os.path.normpath(src_path.replace("\\", "/")) + + if not os.path.exists(src_path): + msg = "{} doesn't exist for {}".format(src_path, repre["_id"]) + report_items["Source file was not found"].append(msg) + return report_items, 0 + + anatomy_filled = anatomy.format(anatomy_data) + if format_dict: + template_result = anatomy_filled["delivery"][template_name] + delivery_path = template_result.rootless.format(**format_dict) + else: + delivery_path = anatomy_filled["delivery"][template_name] + + # Backwards compatibility when extension contained `.` + delivery_path = delivery_path.replace("..", ".") + # Make sure path is valid for all platforms + delivery_path = os.path.normpath(delivery_path.replace("\\", "/")) + + delivery_folder = os.path.dirname(delivery_path) + if not os.path.exists(delivery_folder): + os.makedirs(delivery_folder) + + log.debug("Copying single: {} -> {}".format(src_path, delivery_path)) + _copy_file(src_path, delivery_path) + + return report_items, 1 diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index 1161636cb7..a028ac0a87 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -16,9 +16,9 @@ from openpype.pipeline.load import get_representation_path_with_anatomy from openpype.pipeline.delivery import ( get_format_dict, check_destination_path, + deliver_single_file, ) from openpype.lib.delivery import ( - process_single_file, process_sequence, ) @@ -208,7 +208,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): args[0] = src_path if frame: anatomy_data["frame"] = frame - new_report_items, uploaded = process_single_file(*args) + new_report_items, uploaded = deliver_single_file(*args) report_items.update(new_report_items) self._update_progress(uploaded) else: # fallback for Pype2 and representations without files @@ -217,7 +217,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): repre["context"]["frame"] = len(str(frame)) * "#" if not frame: - new_report_items, uploaded = process_single_file(*args) + new_report_items, uploaded = deliver_single_file(*args) else: new_report_items, uploaded = process_sequence(*args) report_items.update(new_report_items) From d3a7637d1561a411a40a7b74494d692e292f5c4b Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:10:44 +0200 Subject: [PATCH 14/16] Marked 'process_sequence' as deprecated and moved to pipeline delivery as 'deliver_sequence' --- openpype/lib/delivery.py | 144 +++--------------- .../event_handlers_user/action_delivery.py | 6 +- openpype/pipeline/delivery.py | 144 +++++++++++++++++- openpype/plugins/load/delivery.py | 6 +- 4 files changed, 172 insertions(+), 128 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index 455401d0fd..d44a4edb3f 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -213,6 +213,7 @@ def process_single_file( ) +@deprecated("openpype.pipeline.delivery.deliver_sequence") def process_sequence( src_path, repre, anatomy, template_name, anatomy_data, format_dict, report_items, log @@ -220,128 +221,33 @@ def process_sequence( """ For Pype2(mainly - works in 3 too) where representation might not contain files. - Uses listing physical files (not 'files' on repre as a)might not be - present, b)might not be reliable for representation and copying them. + Uses listing physical files (not 'files' on repre as a)might not be + present, b)might not be reliable for representation and copying them. - TODO Should be refactored when files are sufficient to drive all - representations. + TODO Should be refactored when files are sufficient to drive all + representations. - Args: - src_path(str): path of source representation file - repre (dict): full representation - anatomy (Anatomy) - template_name (string): user selected delivery template name - anatomy_data (dict): data from repre to fill anatomy with - format_dict (dict): root dictionary with names and values - report_items (collections.defaultdict): to return error messages - log (Logger): for log printing - Returns: - (collections.defaultdict , int) + Args: + src_path(str): path of source representation file + repre (dict): full representation + anatomy (Anatomy) + template_name (string): user selected delivery template name + anatomy_data (dict): data from repre to fill anatomy with + format_dict (dict): root dictionary with names and values + report_items (collections.defaultdict): to return error messages + log (Logger): for log printing + + Returns: + (collections.defaultdict , int) + + Deprecated: + Function was moved to different location and will be removed + after 3.16.* release. """ - src_path = os.path.normpath(src_path.replace("\\", "/")) - def hash_path_exist(myPath): - res = myPath.replace('#', '*') - glob_search_results = glob.glob(res) - if len(glob_search_results) > 0: - return True - return False + from openpype.pipeline.delivery import deliver_sequence - if not hash_path_exist(src_path): - msg = "{} doesn't exist for {}".format(src_path, - repre["_id"]) - report_items["Source file was not found"].append(msg) - return report_items, 0 - - delivery_templates = anatomy.templates.get("delivery") or {} - delivery_template = delivery_templates.get(template_name) - if delivery_template is None: - msg = ( - "Delivery template \"{}\" in anatomy of project \"{}\"" - " was not found" - ).format(template_name, anatomy.project_name) - report_items[""].append(msg) - return report_items, 0 - - # Check if 'frame' key is available in template which is required - # for sequence delivery - if "{frame" not in delivery_template: - msg = ( - "Delivery template \"{}\" in anatomy of project \"{}\"" - "does not contain '{{frame}}' key to fill. Delivery of sequence" - " can't be processed." - ).format(template_name, anatomy.project_name) - report_items[""].append(msg) - return report_items, 0 - - dir_path, file_name = os.path.split(str(src_path)) - - context = repre["context"] - ext = context.get("ext", context.get("representation")) - - if not ext: - msg = "Source extension not found, cannot find collection" - report_items[msg].append(src_path) - log.warning("{} <{}>".format(msg, context)) - return report_items, 0 - - ext = "." + ext - # context.representation could be .psd - ext = ext.replace("..", ".") - - src_collections, remainder = clique.assemble(os.listdir(dir_path)) - src_collection = None - for col in src_collections: - if col.tail != ext: - continue - - src_collection = col - break - - if src_collection is None: - msg = "Source collection of files was not found" - report_items[msg].append(src_path) - log.warning("{} <{}>".format(msg, src_path)) - return report_items, 0 - - frame_indicator = "@####@" - - anatomy_data["frame"] = frame_indicator - anatomy_filled = anatomy.format(anatomy_data) - - if format_dict: - template_result = anatomy_filled["delivery"][template_name] - delivery_path = template_result.rootless.format(**format_dict) - else: - delivery_path = anatomy_filled["delivery"][template_name] - - delivery_path = os.path.normpath(delivery_path.replace("\\", "/")) - delivery_folder = os.path.dirname(delivery_path) - dst_head, dst_tail = delivery_path.split(frame_indicator) - dst_padding = src_collection.padding - dst_collection = clique.Collection( - head=dst_head, - tail=dst_tail, - padding=dst_padding + return deliver_sequence( + src_path, repre, anatomy, template_name, anatomy_data, format_dict, + report_items, log ) - - if not os.path.exists(delivery_folder): - os.makedirs(delivery_folder) - - src_head = src_collection.head - src_tail = src_collection.tail - uploaded = 0 - for index in src_collection.indexes: - src_padding = src_collection.format("{padding}") % index - src_file_name = "{}{}{}".format(src_head, src_padding, src_tail) - src = os.path.normpath( - os.path.join(dir_path, src_file_name) - ) - - dst_padding = dst_collection.format("{padding}") % index - dst = "{}{}{}".format(dst_head, dst_padding, dst_tail) - log.debug("Copying single: {} -> {}".format(src, dst)) - copy_file(src, dst) - uploaded += 1 - - return report_items, uploaded diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py index fe91670c3d..a400c8f5f0 100644 --- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py +++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py @@ -22,9 +22,7 @@ from openpype.pipeline.delivery import ( get_format_dict, check_destination_path, deliver_single_file, -) -from openpype.lib.delivery import ( - process_sequence + deliver_sequence, ) @@ -598,7 +596,7 @@ class Delivery(BaseAction): if not frame: deliver_single_file(*args) else: - process_sequence(*args) + deliver_sequence(*args) return self.report(report_items) diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py index 7c5121aa53..8cf9a43aac 100644 --- a/openpype/pipeline/delivery.py +++ b/openpype/pipeline/delivery.py @@ -125,7 +125,7 @@ def deliver_single_file( Args: src_path(str): path of source representation file - repre (dict): full repre, used only in process_sequence, here only + repre (dict): full repre, used only in deliver_sequence, here only as to share same signature anatomy (Anatomy) template_name (string): user selected delivery template name @@ -166,3 +166,145 @@ def deliver_single_file( _copy_file(src_path, delivery_path) return report_items, 1 + + +def deliver_sequence( + src_path, + repre, + anatomy, + template_name, + anatomy_data, + format_dict, + report_items, + log +): + """ For Pype2(mainly - works in 3 too) where representation might not + contain files. + + Uses listing physical files (not 'files' on repre as a)might not be + present, b)might not be reliable for representation and copying them. + + TODO Should be refactored when files are sufficient to drive all + representations. + + Args: + src_path(str): path of source representation file + repre (dict): full representation + anatomy (Anatomy) + template_name (string): user selected delivery template name + anatomy_data (dict): data from repre to fill anatomy with + format_dict (dict): root dictionary with names and values + report_items (collections.defaultdict): to return error messages + log (logging.Logger): for log printing + + Returns: + (collections.defaultdict, int) + """ + + src_path = os.path.normpath(src_path.replace("\\", "/")) + + def hash_path_exist(myPath): + res = myPath.replace('#', '*') + glob_search_results = glob.glob(res) + if len(glob_search_results) > 0: + return True + return False + + if not hash_path_exist(src_path): + msg = "{} doesn't exist for {}".format( + src_path, repre["_id"]) + report_items["Source file was not found"].append(msg) + return report_items, 0 + + delivery_templates = anatomy.templates.get("delivery") or {} + delivery_template = delivery_templates.get(template_name) + if delivery_template is None: + msg = ( + "Delivery template \"{}\" in anatomy of project \"{}\"" + " was not found" + ).format(template_name, anatomy.project_name) + report_items[""].append(msg) + return report_items, 0 + + # Check if 'frame' key is available in template which is required + # for sequence delivery + if "{frame" not in delivery_template: + msg = ( + "Delivery template \"{}\" in anatomy of project \"{}\"" + "does not contain '{{frame}}' key to fill. Delivery of sequence" + " can't be processed." + ).format(template_name, anatomy.project_name) + report_items[""].append(msg) + return report_items, 0 + + dir_path, file_name = os.path.split(str(src_path)) + + context = repre["context"] + ext = context.get("ext", context.get("representation")) + + if not ext: + msg = "Source extension not found, cannot find collection" + report_items[msg].append(src_path) + log.warning("{} <{}>".format(msg, context)) + return report_items, 0 + + ext = "." + ext + # context.representation could be .psd + ext = ext.replace("..", ".") + + src_collections, remainder = clique.assemble(os.listdir(dir_path)) + src_collection = None + for col in src_collections: + if col.tail != ext: + continue + + src_collection = col + break + + if src_collection is None: + msg = "Source collection of files was not found" + report_items[msg].append(src_path) + log.warning("{} <{}>".format(msg, src_path)) + return report_items, 0 + + frame_indicator = "@####@" + + anatomy_data["frame"] = frame_indicator + anatomy_filled = anatomy.format(anatomy_data) + + if format_dict: + template_result = anatomy_filled["delivery"][template_name] + delivery_path = template_result.rootless.format(**format_dict) + else: + delivery_path = anatomy_filled["delivery"][template_name] + + delivery_path = os.path.normpath(delivery_path.replace("\\", "/")) + delivery_folder = os.path.dirname(delivery_path) + dst_head, dst_tail = delivery_path.split(frame_indicator) + dst_padding = src_collection.padding + dst_collection = clique.Collection( + head=dst_head, + tail=dst_tail, + padding=dst_padding + ) + + if not os.path.exists(delivery_folder): + os.makedirs(delivery_folder) + + src_head = src_collection.head + src_tail = src_collection.tail + uploaded = 0 + for index in src_collection.indexes: + src_padding = src_collection.format("{padding}") % index + src_file_name = "{}{}{}".format(src_head, src_padding, src_tail) + src = os.path.normpath( + os.path.join(dir_path, src_file_name) + ) + + dst_padding = dst_collection.format("{padding}") % index + dst = "{}{}{}".format(dst_head, dst_padding, dst_tail) + log.debug("Copying single: {} -> {}".format(src, dst)) + _copy_file(src, dst) + uploaded += 1 + + return report_items, uploaded diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py index a028ac0a87..89c24f2402 100644 --- a/openpype/plugins/load/delivery.py +++ b/openpype/plugins/load/delivery.py @@ -17,9 +17,7 @@ from openpype.pipeline.delivery import ( get_format_dict, check_destination_path, deliver_single_file, -) -from openpype.lib.delivery import ( - process_sequence, + deliver_sequence, ) @@ -219,7 +217,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog): if not frame: new_report_items, uploaded = deliver_single_file(*args) else: - new_report_items, uploaded = process_sequence(*args) + new_report_items, uploaded = deliver_sequence(*args) report_items.update(new_report_items) self._update_progress(uploaded) From 19c7d2b8a150d4d052db9a9b4a813bd2922b272f Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 15:20:21 +0200 Subject: [PATCH 15/16] marked 'copy_file' as deprecated --- openpype/lib/delivery.py | 3 +-- openpype/lib/path_tools.py | 11 ++++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py index d44a4edb3f..efb542de75 100644 --- a/openpype/lib/delivery.py +++ b/openpype/lib/delivery.py @@ -1,8 +1,6 @@ """Functions useful for delivery action or loader""" import os import shutil -import glob -import clique import functools import warnings @@ -109,6 +107,7 @@ def path_from_representation(representation, anatomy): return get_representation_path_with_anatomy(representation, anatomy) +@deprecated def copy_file(src_path, dst_path): """Hardlink file if possible(to save space), copy if not""" from openpype.lib import create_hard_link # safer importing diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 45aa54d6cb..1835c71644 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -179,12 +179,12 @@ def get_version_from_path(file): """Find version number in file path string. Args: - file (string): file path + file (str): file path Returns: - v: version number in string ('001') - + str: version number in string ('001') """ + pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE) try: return pattern.findall(file)[-1] @@ -200,16 +200,17 @@ def get_last_version_from_path(path_dir, filter): """Find last version of given directory content. Args: - path_dir (string): directory path + path_dir (str): directory path filter (list): list of strings used as file name filter Returns: - string: file name with last version + str: file name with last version Example: last_version_file = get_last_version_from_path( "/project/shots/shot01/work", ["shot01", "compositing", "nk"]) """ + assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory" assert isinstance(filter, list) and ( len(filter) != 0), "`filter` argument needs to be list and not empty" From cd09f23b968ee0162441c388172ec0027e825a5e Mon Sep 17 00:00:00 2001 From: Jakub Trllo Date: Mon, 29 Aug 2022 16:44:51 +0200 Subject: [PATCH 16/16] removed unused import --- openpype/plugins/load/delete_old_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/load/delete_old_versions.py b/openpype/plugins/load/delete_old_versions.py index 8c8546d9c8..b7ac015268 100644 --- a/openpype/plugins/load/delete_old_versions.py +++ b/openpype/plugins/load/delete_old_versions.py @@ -10,7 +10,7 @@ from Qt import QtWidgets, QtCore from openpype import style from openpype.client import get_versions, get_representations from openpype.modules import ModulesManager -from openpype.lib import StringTemplate, format_file_size +from openpype.lib import format_file_size from openpype.pipeline import load, AvalonMongoDB, Anatomy from openpype.pipeline.load import ( get_representation_path_with_anatomy,