diff --git a/openpype/lib/__init__.py b/openpype/lib/__init__.py
index adb857a056..17aafc3e8b 100644
--- a/openpype/lib/__init__.py
+++ b/openpype/lib/__init__.py
@@ -192,6 +192,8 @@ from .plugin_tools import (
)
from .path_tools import (
+ format_file_size,
+ collect_frames,
create_hard_link,
version_up,
get_version_from_path,
@@ -353,6 +355,8 @@ __all__ = [
"set_plugin_attributes_from_settings",
"source_hash",
+ "format_file_size",
+ "collect_frames",
"create_hard_link",
"version_up",
"get_version_from_path",
diff --git a/openpype/lib/delivery.py b/openpype/lib/delivery.py
index ffcfe9fa4d..efb542de75 100644
--- a/openpype/lib/delivery.py
+++ b/openpype/lib/delivery.py
@@ -1,81 +1,113 @@
"""Functions useful for delivery action or loader"""
import os
import shutil
-import glob
-import clique
-import collections
-
-from .path_templates import (
- StringTemplate,
- TemplateUnsolved,
-)
+import functools
+import warnings
+class DeliveryDeprecatedWarning(DeprecationWarning):
+ pass
+
+
+def deprecated(new_destination):
+ """Mark functions as deprecated.
+
+ It will result in a warning being emitted when the function is used.
+ """
+
+ func = None
+ if callable(new_destination):
+ func = new_destination
+ new_destination = None
+
+ def _decorator(decorated_func):
+ if new_destination is None:
+ warning_message = (
+ " Please check content of deprecated function to figure out"
+ " possible replacement."
+ )
+ else:
+ warning_message = " Please replace your usage with '{}'.".format(
+ new_destination
+ )
+
+ @functools.wraps(decorated_func)
+ def wrapper(*args, **kwargs):
+ warnings.simplefilter("always", DeliveryDeprecatedWarning)
+ warnings.warn(
+ (
+ "Call to deprecated function '{}'"
+ "\nFunction was moved or removed.{}"
+ ).format(decorated_func.__name__, warning_message),
+ category=DeliveryDeprecatedWarning,
+ stacklevel=4
+ )
+ return decorated_func(*args, **kwargs)
+ return wrapper
+
+ if func is None:
+ return _decorator
+ return _decorator(func)
+
+
+@deprecated("openpype.lib.path_tools.collect_frames")
def collect_frames(files):
+ """Returns dict of source path and its frame, if from sequence
+
+ Uses clique as most precise solution, used when anatomy template that
+ created files is not known.
+
+ Assumption is that frames are separated by '.', negative frames are not
+ allowed.
+
+ Args:
+ files(list) or (set with single value): list of source paths
+
+ Returns:
+ (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- Returns dict of source path and its frame, if from sequence
- Uses clique as most precise solution, used when anatomy template that
- created files is not known.
+ from .path_tools import collect_frames
- Assumption is that frames are separated by '.', negative frames are not
- allowed.
+ return collect_frames(files)
- Args:
- files(list) or (set with single value): list of source paths
- Returns:
- (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+
+@deprecated("openpype.lib.path_tools.format_file_size")
+def sizeof_fmt(num, suffix=None):
+ """Returns formatted string with size in appropriate unit
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- patterns = [clique.PATTERNS["frames"]]
- collections, remainder = clique.assemble(files, minimum_items=1,
- patterns=patterns)
- sources_and_frames = {}
- if collections:
- for collection in collections:
- src_head = collection.head
- src_tail = collection.tail
-
- for index in collection.indexes:
- src_frame = collection.format("{padding}") % index
- src_file_name = "{}{}{}".format(src_head, src_frame,
- src_tail)
- sources_and_frames[src_file_name] = src_frame
- else:
- sources_and_frames[remainder.pop()] = None
-
- return sources_and_frames
-
-
-def sizeof_fmt(num, suffix='B'):
- """Returns formatted string with size in appropriate unit"""
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Yi', suffix)
+ from .path_tools import format_file_size
+ return format_file_size(num, suffix)
+@deprecated("openpype.pipeline.load.get_representation_path_with_anatomy")
def path_from_representation(representation, anatomy):
- try:
- template = representation["data"]["template"]
+ """Get representation path using representation document and anatomy.
- except KeyError:
- return None
+ Args:
+ representation (Dict[str, Any]): Representation document.
+ anatomy (Anatomy): Project anatomy.
- try:
- context = representation["context"]
- context["root"] = anatomy.roots
- path = StringTemplate.format_strict_template(template, context)
- return os.path.normpath(path)
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
+ """
- except TemplateUnsolved:
- # Template references unavailable data
- return None
+ from openpype.pipeline.load import get_representation_path_with_anatomy
- return path
+ return get_representation_path_with_anatomy(representation, anatomy)
+@deprecated
def copy_file(src_path, dst_path):
"""Hardlink file if possible(to save space), copy if not"""
from openpype.lib import create_hard_link # safer importing
@@ -91,131 +123,96 @@ def copy_file(src_path, dst_path):
shutil.copyfile(src_path, dst_path)
+@deprecated("openpype.pipeline.delivery.get_format_dict")
def get_format_dict(anatomy, location_path):
"""Returns replaced root values from user provider value.
- Args:
- anatomy (Anatomy)
- location_path (str): user provided value
- Returns:
- (dict): prepared for formatting of a template
+ Args:
+ anatomy (Anatomy)
+ location_path (str): user provided value
+
+ Returns:
+ (dict): prepared for formatting of a template
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- format_dict = {}
- if location_path:
- location_path = location_path.replace("\\", "/")
- root_names = anatomy.root_names_from_templates(
- anatomy.templates["delivery"]
- )
- if root_names is None:
- format_dict["root"] = location_path
- else:
- format_dict["root"] = {}
- for name in root_names:
- format_dict["root"][name] = location_path
- return format_dict
+
+ from openpype.pipeline.delivery import get_format_dict
+
+ return get_format_dict(anatomy, location_path)
+@deprecated("openpype.pipeline.delivery.check_destination_path")
def check_destination_path(repre_id,
anatomy, anatomy_data,
datetime_data, template_name):
""" Try to create destination path based on 'template_name'.
- In the case that path cannot be filled, template contains unmatched
- keys, provide error message to filter out repre later.
+ In the case that path cannot be filled, template contains unmatched
+ keys, provide error message to filter out repre later.
- Args:
- anatomy (Anatomy)
- anatomy_data (dict): context to fill anatomy
- datetime_data (dict): values with actual date
- template_name (str): to pick correct delivery template
- Returns:
- (collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"}
+ Args:
+ anatomy (Anatomy)
+ anatomy_data (dict): context to fill anatomy
+ datetime_data (dict): values with actual date
+ template_name (str): to pick correct delivery template
+
+ Returns:
+ (collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"}
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- anatomy_data.update(datetime_data)
- anatomy_filled = anatomy.format_all(anatomy_data)
- dest_path = anatomy_filled["delivery"][template_name]
- report_items = collections.defaultdict(list)
- if not dest_path.solved:
- msg = (
- "Missing keys in Representation's context"
- " for anatomy template \"{}\"."
- ).format(template_name)
+ from openpype.pipeline.delivery import check_destination_path
- sub_msg = (
- "Representation: {}
"
- ).format(repre_id)
-
- if dest_path.missing_keys:
- keys = ", ".join(dest_path.missing_keys)
- sub_msg += (
- "- Missing keys: \"{}\"
"
- ).format(keys)
-
- if dest_path.invalid_types:
- items = []
- for key, value in dest_path.invalid_types.items():
- items.append("\"{}\" {}".format(key, str(value)))
-
- keys = ", ".join(items)
- sub_msg += (
- "- Invalid value DataType: \"{}\"
"
- ).format(keys)
-
- report_items[msg].append(sub_msg)
-
- return report_items
+ return check_destination_path(
+ repre_id,
+ anatomy,
+ anatomy_data,
+ datetime_data,
+ template_name
+ )
+@deprecated("openpype.pipeline.delivery.deliver_single_file")
def process_single_file(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
):
"""Copy single file to calculated path based on template
- Args:
- src_path(str): path of source representation file
- _repre (dict): full repre, used only in process_sequence, here only
- as to share same signature
- anatomy (Anatomy)
- template_name (string): user selected delivery template name
- anatomy_data (dict): data from repre to fill anatomy with
- format_dict (dict): root dictionary with names and values
- report_items (collections.defaultdict): to return error messages
- log (Logger): for log printing
- Returns:
- (collections.defaultdict , int)
+ Args:
+ src_path(str): path of source representation file
+ _repre (dict): full repre, used only in process_sequence, here only
+ as to share same signature
+ anatomy (Anatomy)
+ template_name (string): user selected delivery template name
+ anatomy_data (dict): data from repre to fill anatomy with
+ format_dict (dict): root dictionary with names and values
+ report_items (collections.defaultdict): to return error messages
+ log (Logger): for log printing
+
+ Returns:
+ (collections.defaultdict , int)
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- # Make sure path is valid for all platforms
- src_path = os.path.normpath(src_path.replace("\\", "/"))
- if not os.path.exists(src_path):
- msg = "{} doesn't exist for {}".format(src_path, repre["_id"])
- report_items["Source file was not found"].append(msg)
- return report_items, 0
+ from openpype.pipeline.delivery import deliver_single_file
- anatomy_filled = anatomy.format(anatomy_data)
- if format_dict:
- template_result = anatomy_filled["delivery"][template_name]
- delivery_path = template_result.rootless.format(**format_dict)
- else:
- delivery_path = anatomy_filled["delivery"][template_name]
-
- # Backwards compatibility when extension contained `.`
- delivery_path = delivery_path.replace("..", ".")
- # Make sure path is valid for all platforms
- delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
-
- delivery_folder = os.path.dirname(delivery_path)
- if not os.path.exists(delivery_folder):
- os.makedirs(delivery_folder)
-
- log.debug("Copying single: {} -> {}".format(src_path, delivery_path))
- copy_file(src_path, delivery_path)
-
- return report_items, 1
+ return deliver_single_file(
+ src_path, repre, anatomy, template_name, anatomy_data, format_dict,
+ report_items, log
+ )
+@deprecated("openpype.pipeline.delivery.deliver_sequence")
def process_sequence(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
@@ -223,128 +220,33 @@ def process_sequence(
""" For Pype2(mainly - works in 3 too) where representation might not
contain files.
- Uses listing physical files (not 'files' on repre as a)might not be
- present, b)might not be reliable for representation and copying them.
+ Uses listing physical files (not 'files' on repre as a)might not be
+ present, b)might not be reliable for representation and copying them.
- TODO Should be refactored when files are sufficient to drive all
- representations.
+ TODO Should be refactored when files are sufficient to drive all
+ representations.
- Args:
- src_path(str): path of source representation file
- repre (dict): full representation
- anatomy (Anatomy)
- template_name (string): user selected delivery template name
- anatomy_data (dict): data from repre to fill anatomy with
- format_dict (dict): root dictionary with names and values
- report_items (collections.defaultdict): to return error messages
- log (Logger): for log printing
- Returns:
- (collections.defaultdict , int)
+ Args:
+ src_path(str): path of source representation file
+ repre (dict): full representation
+ anatomy (Anatomy)
+ template_name (string): user selected delivery template name
+ anatomy_data (dict): data from repre to fill anatomy with
+ format_dict (dict): root dictionary with names and values
+ report_items (collections.defaultdict): to return error messages
+ log (Logger): for log printing
+
+ Returns:
+ (collections.defaultdict , int)
+
+ Deprecated:
+ Function was moved to different location and will be removed
+ after 3.16.* release.
"""
- src_path = os.path.normpath(src_path.replace("\\", "/"))
- def hash_path_exist(myPath):
- res = myPath.replace('#', '*')
- glob_search_results = glob.glob(res)
- if len(glob_search_results) > 0:
- return True
- return False
+ from openpype.pipeline.delivery import deliver_sequence
- if not hash_path_exist(src_path):
- msg = "{} doesn't exist for {}".format(src_path,
- repre["_id"])
- report_items["Source file was not found"].append(msg)
- return report_items, 0
-
- delivery_templates = anatomy.templates.get("delivery") or {}
- delivery_template = delivery_templates.get(template_name)
- if delivery_template is None:
- msg = (
- "Delivery template \"{}\" in anatomy of project \"{}\""
- " was not found"
- ).format(template_name, anatomy.project_name)
- report_items[""].append(msg)
- return report_items, 0
-
- # Check if 'frame' key is available in template which is required
- # for sequence delivery
- if "{frame" not in delivery_template:
- msg = (
- "Delivery template \"{}\" in anatomy of project \"{}\""
- "does not contain '{{frame}}' key to fill. Delivery of sequence"
- " can't be processed."
- ).format(template_name, anatomy.project_name)
- report_items[""].append(msg)
- return report_items, 0
-
- dir_path, file_name = os.path.split(str(src_path))
-
- context = repre["context"]
- ext = context.get("ext", context.get("representation"))
-
- if not ext:
- msg = "Source extension not found, cannot find collection"
- report_items[msg].append(src_path)
- log.warning("{} <{}>".format(msg, context))
- return report_items, 0
-
- ext = "." + ext
- # context.representation could be .psd
- ext = ext.replace("..", ".")
-
- src_collections, remainder = clique.assemble(os.listdir(dir_path))
- src_collection = None
- for col in src_collections:
- if col.tail != ext:
- continue
-
- src_collection = col
- break
-
- if src_collection is None:
- msg = "Source collection of files was not found"
- report_items[msg].append(src_path)
- log.warning("{} <{}>".format(msg, src_path))
- return report_items, 0
-
- frame_indicator = "@####@"
-
- anatomy_data["frame"] = frame_indicator
- anatomy_filled = anatomy.format(anatomy_data)
-
- if format_dict:
- template_result = anatomy_filled["delivery"][template_name]
- delivery_path = template_result.rootless.format(**format_dict)
- else:
- delivery_path = anatomy_filled["delivery"][template_name]
-
- delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
- delivery_folder = os.path.dirname(delivery_path)
- dst_head, dst_tail = delivery_path.split(frame_indicator)
- dst_padding = src_collection.padding
- dst_collection = clique.Collection(
- head=dst_head,
- tail=dst_tail,
- padding=dst_padding
+ return deliver_sequence(
+ src_path, repre, anatomy, template_name, anatomy_data, format_dict,
+ report_items, log
)
-
- if not os.path.exists(delivery_folder):
- os.makedirs(delivery_folder)
-
- src_head = src_collection.head
- src_tail = src_collection.tail
- uploaded = 0
- for index in src_collection.indexes:
- src_padding = src_collection.format("{padding}") % index
- src_file_name = "{}{}{}".format(src_head, src_padding, src_tail)
- src = os.path.normpath(
- os.path.join(dir_path, src_file_name)
- )
-
- dst_padding = dst_collection.format("{padding}") % index
- dst = "{}{}{}".format(dst_head, dst_padding, dst_tail)
- log.debug("Copying single: {} -> {}".format(src, dst))
- copy_file(src, dst)
- uploaded += 1
-
- return report_items, uploaded
diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py
index 4f28be3302..1835c71644 100644
--- a/openpype/lib/path_tools.py
+++ b/openpype/lib/path_tools.py
@@ -6,6 +6,8 @@ import logging
import six
import platform
+import clique
+
from openpype.client import get_project
from openpype.settings import get_project_settings
@@ -14,6 +16,27 @@ from .profiles_filtering import filter_profiles
log = logging.getLogger(__name__)
+def format_file_size(file_size, suffix=None):
+ """Returns formatted string with size in appropriate unit.
+
+ Args:
+ file_size (int): Size of file in bytes.
+ suffix (str): Suffix for formatted size. Default is 'B' (as bytes).
+
+ Returns:
+ str: Formatted size using proper unit and passed suffix (e.g. 7 MiB).
+ """
+
+ if suffix is None:
+ suffix = "B"
+
+ for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
+ if abs(file_size) < 1024.0:
+ return "%3.1f%s%s" % (file_size, unit, suffix)
+ file_size /= 1024.0
+ return "%.1f%s%s" % (file_size, "Yi", suffix)
+
+
def create_hard_link(src_path, dst_path):
"""Create hardlink of file.
@@ -50,6 +73,43 @@ def create_hard_link(src_path, dst_path):
)
+def collect_frames(files):
+ """Returns dict of source path and its frame, if from sequence
+
+ Uses clique as most precise solution, used when anatomy template that
+ created files is not known.
+
+ Assumption is that frames are separated by '.', negative frames are not
+ allowed.
+
+ Args:
+ files(list) or (set with single value): list of source paths
+
+ Returns:
+ (dict): {'/asset/subset_v001.0001.png': '0001', ....}
+ """
+
+ patterns = [clique.PATTERNS["frames"]]
+ collections, remainder = clique.assemble(
+ files, minimum_items=1, patterns=patterns)
+
+ sources_and_frames = {}
+ if collections:
+ for collection in collections:
+ src_head = collection.head
+ src_tail = collection.tail
+
+ for index in collection.indexes:
+ src_frame = collection.format("{padding}") % index
+ src_file_name = "{}{}{}".format(
+ src_head, src_frame, src_tail)
+ sources_and_frames[src_file_name] = src_frame
+ else:
+ sources_and_frames[remainder.pop()] = None
+
+ return sources_and_frames
+
+
def _rreplace(s, a, b, n=1):
"""Replace a with b in string s from right side n times."""
return b.join(s.rsplit(a, n))
@@ -119,12 +179,12 @@ def get_version_from_path(file):
"""Find version number in file path string.
Args:
- file (string): file path
+ file (str): file path
Returns:
- v: version number in string ('001')
-
+ str: version number in string ('001')
"""
+
pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE)
try:
return pattern.findall(file)[-1]
@@ -140,16 +200,17 @@ def get_last_version_from_path(path_dir, filter):
"""Find last version of given directory content.
Args:
- path_dir (string): directory path
+ path_dir (str): directory path
filter (list): list of strings used as file name filter
Returns:
- string: file name with last version
+ str: file name with last version
Example:
last_version_file = get_last_version_from_path(
"/project/shots/shot01/work", ["shot01", "compositing", "nk"])
"""
+
assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory"
assert isinstance(filter, list) and (
len(filter) != 0), "`filter` argument needs to be list and not empty"
diff --git a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
index c55f85c8da..1d68793d53 100644
--- a/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
+++ b/openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
@@ -3,8 +3,10 @@ import attr
import getpass
import pyblish.api
-from openpype.lib import env_value_to_bool
-from openpype.lib.delivery import collect_frames
+from openpype.lib import (
+ env_value_to_bool,
+ collect_frames,
+)
from openpype.pipeline import legacy_io
from openpype_modules.deadline import abstract_submit_deadline
from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo
diff --git a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
index c2426e0d78..f0a3ddd246 100644
--- a/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
+++ b/openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
@@ -3,7 +3,7 @@ import requests
import pyblish.api
-from openpype.lib.delivery import collect_frames
+from openpype.lib import collect_frames
from openpype_modules.deadline.abstract_submit_deadline import requests_get
diff --git a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
index 79d04a7854..c543dc8834 100644
--- a/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
@@ -11,7 +11,11 @@ from openpype.client import (
get_versions,
get_representations
)
-from openpype.lib import StringTemplate, TemplateUnsolved
+from openpype.lib import (
+ StringTemplate,
+ TemplateUnsolved,
+ format_file_size,
+)
from openpype.pipeline import AvalonMongoDB, Anatomy
from openpype_modules.ftrack.lib import BaseAction, statics_icon
@@ -134,13 +138,6 @@ class DeleteOldVersions(BaseAction):
"title": self.inteface_title
}
- def sizeof_fmt(self, num, suffix='B'):
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Yi', suffix)
-
def launch(self, session, entities, event):
values = event["data"].get("values")
if not values:
@@ -359,7 +356,7 @@ class DeleteOldVersions(BaseAction):
dir_paths, file_paths_by_dir, delete=False
)
- msg = "Total size of files: " + self.sizeof_fmt(size)
+ msg = "Total size of files: {}".format(format_file_size(size))
self.log.warning(msg)
@@ -430,7 +427,7 @@ class DeleteOldVersions(BaseAction):
"message": msg
}
- msg = "Total size of files deleted: " + self.sizeof_fmt(size)
+ msg = "Total size of files deleted: {}".format(format_file_size(size))
self.log.warning(msg)
diff --git a/openpype/modules/ftrack/event_handlers_user/action_delivery.py b/openpype/modules/ftrack/event_handlers_user/action_delivery.py
index eec245070c..a400c8f5f0 100644
--- a/openpype/modules/ftrack/event_handlers_user/action_delivery.py
+++ b/openpype/modules/ftrack/event_handlers_user/action_delivery.py
@@ -10,19 +10,19 @@ from openpype.client import (
get_versions,
get_representations
)
-from openpype.pipeline import Anatomy
from openpype_modules.ftrack.lib import BaseAction, statics_icon
from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
from openpype_modules.ftrack.lib.custom_attributes import (
query_custom_attributes
)
from openpype.lib.dateutils import get_datetime_data
-from openpype.lib.delivery import (
- path_from_representation,
+from openpype.pipeline import Anatomy
+from openpype.pipeline.load import get_representation_path_with_anatomy
+from openpype.pipeline.delivery import (
get_format_dict,
check_destination_path,
- process_single_file,
- process_sequence
+ deliver_single_file,
+ deliver_sequence,
)
@@ -580,7 +580,7 @@ class Delivery(BaseAction):
if frame:
repre["context"]["frame"] = len(str(frame)) * "#"
- repre_path = path_from_representation(repre, anatomy)
+ repre_path = get_representation_path_with_anatomy(repre, anatomy)
# TODO add backup solution where root of path from component
# is replaced with root
args = (
@@ -594,9 +594,9 @@ class Delivery(BaseAction):
self.log
)
if not frame:
- process_single_file(*args)
+ deliver_single_file(*args)
else:
- process_sequence(*args)
+ deliver_sequence(*args)
return self.report(report_items)
diff --git a/openpype/pipeline/delivery.py b/openpype/pipeline/delivery.py
new file mode 100644
index 0000000000..8cf9a43aac
--- /dev/null
+++ b/openpype/pipeline/delivery.py
@@ -0,0 +1,310 @@
+"""Functions useful for delivery of published representations."""
+import os
+import shutil
+import glob
+import clique
+import collections
+
+from openpype.lib import create_hard_link
+
+
+def _copy_file(src_path, dst_path):
+ """Hardlink file if possible(to save space), copy if not.
+
+ Because of using hardlinks should not be function used in other parts
+ of pipeline.
+ """
+
+ if os.path.exists(dst_path):
+ return
+ try:
+ create_hard_link(
+ src_path,
+ dst_path
+ )
+ except OSError:
+ shutil.copyfile(src_path, dst_path)
+
+
+def get_format_dict(anatomy, location_path):
+ """Returns replaced root values from user provider value.
+
+ Args:
+ anatomy (Anatomy): Project anatomy.
+ location_path (str): User provided value.
+
+ Returns:
+ (dict): Prepared data for formatting of a template.
+ """
+
+ format_dict = {}
+ if not location_path:
+ return format_dict
+
+ location_path = location_path.replace("\\", "/")
+ root_names = anatomy.root_names_from_templates(
+ anatomy.templates["delivery"]
+ )
+ format_dict["root"] = {}
+ for name in root_names:
+ format_dict["root"][name] = location_path
+ return format_dict
+
+
+def check_destination_path(
+ repre_id,
+ anatomy,
+ anatomy_data,
+ datetime_data,
+ template_name
+):
+ """ Try to create destination path based on 'template_name'.
+
+ In the case that path cannot be filled, template contains unmatched
+ keys, provide error message to filter out repre later.
+
+ Args:
+ repre_id (str): Representation id.
+ anatomy (Anatomy): Project anatomy.
+ anatomy_data (dict): Template data to fill anatomy templates.
+ datetime_data (dict): Values with actual date.
+ template_name (str): Name of template which should be used from anatomy
+ templates.
+ Returns:
+ Dict[str, List[str]]: Report of happened errors. Key is message title
+ value is detailed information.
+ """
+
+ anatomy_data.update(datetime_data)
+ anatomy_filled = anatomy.format_all(anatomy_data)
+ dest_path = anatomy_filled["delivery"][template_name]
+ report_items = collections.defaultdict(list)
+
+ if not dest_path.solved:
+ msg = (
+ "Missing keys in Representation's context"
+ " for anatomy template \"{}\"."
+ ).format(template_name)
+
+ sub_msg = (
+ "Representation: {}
"
+ ).format(repre_id)
+
+ if dest_path.missing_keys:
+ keys = ", ".join(dest_path.missing_keys)
+ sub_msg += (
+ "- Missing keys: \"{}\"
"
+ ).format(keys)
+
+ if dest_path.invalid_types:
+ items = []
+ for key, value in dest_path.invalid_types.items():
+ items.append("\"{}\" {}".format(key, str(value)))
+
+ keys = ", ".join(items)
+ sub_msg += (
+ "- Invalid value DataType: \"{}\"
"
+ ).format(keys)
+
+ report_items[msg].append(sub_msg)
+
+ return report_items
+
+
+def deliver_single_file(
+ src_path,
+ repre,
+ anatomy,
+ template_name,
+ anatomy_data,
+ format_dict,
+ report_items,
+ log
+):
+ """Copy single file to calculated path based on template
+
+ Args:
+ src_path(str): path of source representation file
+ repre (dict): full repre, used only in deliver_sequence, here only
+ as to share same signature
+ anatomy (Anatomy)
+ template_name (string): user selected delivery template name
+ anatomy_data (dict): data from repre to fill anatomy with
+ format_dict (dict): root dictionary with names and values
+ report_items (collections.defaultdict): to return error messages
+ log (logging.Logger): for log printing
+
+ Returns:
+ (collections.defaultdict, int)
+ """
+
+ # Make sure path is valid for all platforms
+ src_path = os.path.normpath(src_path.replace("\\", "/"))
+
+ if not os.path.exists(src_path):
+ msg = "{} doesn't exist for {}".format(src_path, repre["_id"])
+ report_items["Source file was not found"].append(msg)
+ return report_items, 0
+
+ anatomy_filled = anatomy.format(anatomy_data)
+ if format_dict:
+ template_result = anatomy_filled["delivery"][template_name]
+ delivery_path = template_result.rootless.format(**format_dict)
+ else:
+ delivery_path = anatomy_filled["delivery"][template_name]
+
+ # Backwards compatibility when extension contained `.`
+ delivery_path = delivery_path.replace("..", ".")
+ # Make sure path is valid for all platforms
+ delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
+
+ delivery_folder = os.path.dirname(delivery_path)
+ if not os.path.exists(delivery_folder):
+ os.makedirs(delivery_folder)
+
+ log.debug("Copying single: {} -> {}".format(src_path, delivery_path))
+ _copy_file(src_path, delivery_path)
+
+ return report_items, 1
+
+
+def deliver_sequence(
+ src_path,
+ repre,
+ anatomy,
+ template_name,
+ anatomy_data,
+ format_dict,
+ report_items,
+ log
+):
+ """ For Pype2(mainly - works in 3 too) where representation might not
+ contain files.
+
+ Uses listing physical files (not 'files' on repre as a)might not be
+ present, b)might not be reliable for representation and copying them.
+
+ TODO Should be refactored when files are sufficient to drive all
+ representations.
+
+ Args:
+ src_path(str): path of source representation file
+ repre (dict): full representation
+ anatomy (Anatomy)
+ template_name (string): user selected delivery template name
+ anatomy_data (dict): data from repre to fill anatomy with
+ format_dict (dict): root dictionary with names and values
+ report_items (collections.defaultdict): to return error messages
+ log (logging.Logger): for log printing
+
+ Returns:
+ (collections.defaultdict, int)
+ """
+
+ src_path = os.path.normpath(src_path.replace("\\", "/"))
+
+ def hash_path_exist(myPath):
+ res = myPath.replace('#', '*')
+ glob_search_results = glob.glob(res)
+ if len(glob_search_results) > 0:
+ return True
+ return False
+
+ if not hash_path_exist(src_path):
+ msg = "{} doesn't exist for {}".format(
+ src_path, repre["_id"])
+ report_items["Source file was not found"].append(msg)
+ return report_items, 0
+
+ delivery_templates = anatomy.templates.get("delivery") or {}
+ delivery_template = delivery_templates.get(template_name)
+ if delivery_template is None:
+ msg = (
+ "Delivery template \"{}\" in anatomy of project \"{}\""
+ " was not found"
+ ).format(template_name, anatomy.project_name)
+ report_items[""].append(msg)
+ return report_items, 0
+
+ # Check if 'frame' key is available in template which is required
+ # for sequence delivery
+ if "{frame" not in delivery_template:
+ msg = (
+ "Delivery template \"{}\" in anatomy of project \"{}\""
+ "does not contain '{{frame}}' key to fill. Delivery of sequence"
+ " can't be processed."
+ ).format(template_name, anatomy.project_name)
+ report_items[""].append(msg)
+ return report_items, 0
+
+ dir_path, file_name = os.path.split(str(src_path))
+
+ context = repre["context"]
+ ext = context.get("ext", context.get("representation"))
+
+ if not ext:
+ msg = "Source extension not found, cannot find collection"
+ report_items[msg].append(src_path)
+ log.warning("{} <{}>".format(msg, context))
+ return report_items, 0
+
+ ext = "." + ext
+ # context.representation could be .psd
+ ext = ext.replace("..", ".")
+
+ src_collections, remainder = clique.assemble(os.listdir(dir_path))
+ src_collection = None
+ for col in src_collections:
+ if col.tail != ext:
+ continue
+
+ src_collection = col
+ break
+
+ if src_collection is None:
+ msg = "Source collection of files was not found"
+ report_items[msg].append(src_path)
+ log.warning("{} <{}>".format(msg, src_path))
+ return report_items, 0
+
+ frame_indicator = "@####@"
+
+ anatomy_data["frame"] = frame_indicator
+ anatomy_filled = anatomy.format(anatomy_data)
+
+ if format_dict:
+ template_result = anatomy_filled["delivery"][template_name]
+ delivery_path = template_result.rootless.format(**format_dict)
+ else:
+ delivery_path = anatomy_filled["delivery"][template_name]
+
+ delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
+ delivery_folder = os.path.dirname(delivery_path)
+ dst_head, dst_tail = delivery_path.split(frame_indicator)
+ dst_padding = src_collection.padding
+ dst_collection = clique.Collection(
+ head=dst_head,
+ tail=dst_tail,
+ padding=dst_padding
+ )
+
+ if not os.path.exists(delivery_folder):
+ os.makedirs(delivery_folder)
+
+ src_head = src_collection.head
+ src_tail = src_collection.tail
+ uploaded = 0
+ for index in src_collection.indexes:
+ src_padding = src_collection.format("{padding}") % index
+ src_file_name = "{}{}{}".format(src_head, src_padding, src_tail)
+ src = os.path.normpath(
+ os.path.join(dir_path, src_file_name)
+ )
+
+ dst_padding = dst_collection.format("{padding}") % index
+ dst = "{}{}{}".format(dst_head, dst_padding, dst_tail)
+ log.debug("Copying single: {} -> {}".format(src, dst))
+ _copy_file(src, dst)
+ uploaded += 1
+
+ return report_items, uploaded
diff --git a/openpype/pipeline/load/__init__.py b/openpype/pipeline/load/__init__.py
index b6bdd13d50..bf38a0b3c8 100644
--- a/openpype/pipeline/load/__init__.py
+++ b/openpype/pipeline/load/__init__.py
@@ -1,6 +1,8 @@
from .utils import (
HeroVersionType,
+
IncompatibleLoaderError,
+ InvalidRepresentationContext,
get_repres_contexts,
get_subset_contexts,
@@ -20,6 +22,7 @@ from .utils import (
get_representation_path_from_context,
get_representation_path,
+ get_representation_path_with_anatomy,
is_compatible_loader,
@@ -46,7 +49,9 @@ from .plugins import (
__all__ = (
# utils.py
"HeroVersionType",
+
"IncompatibleLoaderError",
+ "InvalidRepresentationContext",
"get_repres_contexts",
"get_subset_contexts",
@@ -66,6 +71,7 @@ __all__ = (
"get_representation_path_from_context",
"get_representation_path",
+ "get_representation_path_with_anatomy",
"is_compatible_loader",
diff --git a/openpype/pipeline/load/utils.py b/openpype/pipeline/load/utils.py
index 99d6876d4b..83b904e4a7 100644
--- a/openpype/pipeline/load/utils.py
+++ b/openpype/pipeline/load/utils.py
@@ -23,6 +23,10 @@ from openpype.client import (
get_representation_by_name,
get_representation_parents
)
+from openpype.lib import (
+ StringTemplate,
+ TemplateUnsolved,
+)
from openpype.pipeline import (
schema,
legacy_io,
@@ -61,6 +65,11 @@ class IncompatibleLoaderError(ValueError):
pass
+class InvalidRepresentationContext(ValueError):
+ """Representation path can't be received using representation document."""
+ pass
+
+
def get_repres_contexts(representation_ids, dbcon=None):
"""Return parenthood context for representation.
@@ -515,6 +524,52 @@ def get_representation_path_from_context(context):
return get_representation_path(representation, root)
+def get_representation_path_with_anatomy(repre_doc, anatomy):
+ """Receive representation path using representation document and anatomy.
+
+ Anatomy is used to replace 'root' key in representation file. Ideally
+ should be used instead of 'get_representation_path' which is based on
+ "current context".
+
+ Future notes:
+ We want also be able store resources into representation and I can
+ imagine the result should also contain paths to possible resources.
+
+ Args:
+ repre_doc (Dict[str, Any]): Representation document.
+ anatomy (Anatomy): Project anatomy object.
+
+ Returns:
+ Union[None, TemplateResult]: None if path can't be received
+
+ Raises:
+ InvalidRepresentationContext: When representation data are probably
+ invalid or not available.
+ """
+
+ try:
+ template = repre_doc["data"]["template"]
+
+ except KeyError:
+ raise InvalidRepresentationContext((
+ "Representation document does not"
+ " contain template in data ('data.template')"
+ ))
+
+ try:
+ context = repre_doc["context"]
+ context["root"] = anatomy.roots
+ path = StringTemplate.format_strict_template(template, context)
+
+ except TemplateUnsolved as exc:
+ raise InvalidRepresentationContext((
+ "Couldn't resolve representation template with available data."
+ " Reason: {}".format(str(exc))
+ ))
+
+ return path.normalized()
+
+
def get_representation_path(representation, root=None, dbcon=None):
"""Get filename from representation document
@@ -533,8 +588,6 @@ def get_representation_path(representation, root=None, dbcon=None):
"""
- from openpype.lib import StringTemplate, TemplateUnsolved
-
if dbcon is None:
dbcon = legacy_io
@@ -737,6 +790,7 @@ def get_outdated_containers(host=None, project_name=None):
if host is None:
from openpype.pipeline import registered_host
+
host = registered_host()
if project_name is None:
diff --git a/openpype/pipeline/template_data.py b/openpype/pipeline/template_data.py
index 824a25127c..bab46a627d 100644
--- a/openpype/pipeline/template_data.py
+++ b/openpype/pipeline/template_data.py
@@ -28,27 +28,37 @@ def get_general_template_data(system_settings=None):
}
-def get_project_template_data(project_doc):
+def get_project_template_data(project_doc=None, project_name=None):
"""Extract data from project document that are used in templates.
Project document must have 'name' and (at this moment) optional
key 'data.code'.
+ One of 'project_name' or 'project_doc' must be passed. With prepared
+ project document is function much faster because don't have to query.
+
Output contains formatting keys:
- 'project[name]' - Project name
- 'project[code]' - Project code
Args:
project_doc (Dict[str, Any]): Queried project document.
+ project_name (str): Name of project.
Returns:
Dict[str, Dict[str, str]]: Template data based on project document.
"""
+ if not project_name:
+ project_name = project_doc["name"]
+
+ if not project_doc:
+ project_code = get_project(project_name, fields=["data.code"])
+
project_code = project_doc.get("data", {}).get("code")
return {
"project": {
- "name": project_doc["name"],
+ "name": project_name,
"code": project_code
}
}
diff --git a/openpype/plugins/load/delete_old_versions.py b/openpype/plugins/load/delete_old_versions.py
index 6e0b464cc1..b7ac015268 100644
--- a/openpype/plugins/load/delete_old_versions.py
+++ b/openpype/plugins/load/delete_old_versions.py
@@ -7,11 +7,15 @@ from pymongo import UpdateOne
import qargparse
from Qt import QtWidgets, QtCore
-from openpype.client import get_versions, get_representations
from openpype import style
-from openpype.pipeline import load, AvalonMongoDB, Anatomy
-from openpype.lib import StringTemplate
+from openpype.client import get_versions, get_representations
from openpype.modules import ModulesManager
+from openpype.lib import format_file_size
+from openpype.pipeline import load, AvalonMongoDB, Anatomy
+from openpype.pipeline.load import (
+ get_representation_path_with_anatomy,
+ InvalidRepresentationContext,
+)
class DeleteOldVersions(load.SubsetLoaderPlugin):
@@ -38,13 +42,6 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
)
]
- def sizeof_fmt(self, num, suffix='B'):
- for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Yi', suffix)
-
def delete_whole_dir_paths(self, dir_paths, delete=True):
size = 0
@@ -80,27 +77,28 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
def path_from_representation(self, representation, anatomy):
try:
- template = representation["data"]["template"]
-
+ context = representation["context"]
except KeyError:
return (None, None)
+ try:
+ path = get_representation_path_with_anatomy(
+ representation, anatomy
+ )
+ except InvalidRepresentationContext:
+ return (None, None)
+
sequence_path = None
- try:
- context = representation["context"]
- context["root"] = anatomy.roots
- path = str(StringTemplate.format_template(template, context))
- if "frame" in context:
- context["frame"] = self.sequence_splitter
- sequence_path = os.path.normpath(str(
- StringTemplate.format_template(template, context)
- ))
+ if "frame" in context:
+ context["frame"] = self.sequence_splitter
+ sequence_path = get_representation_path_with_anatomy(
+ representation, anatomy
+ )
- except KeyError:
- # Template references unavailable data
- return (None, None)
+ if sequence_path:
+ sequence_path = sequence_path.normalized()
- return (os.path.normpath(path), sequence_path)
+ return (path.normalized(), sequence_path)
def delete_only_repre_files(self, dir_paths, file_paths, delete=True):
size = 0
@@ -456,7 +454,7 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
size += self.main(project_name, data, remove_publish_folder)
print("Progressing {}/{}".format(count + 1, len(contexts)))
- msg = "Total size of files: " + self.sizeof_fmt(size)
+ msg = "Total size of files: {}".format(format_file_size(size))
self.log.info(msg)
self.message(msg)
diff --git a/openpype/plugins/load/delivery.py b/openpype/plugins/load/delivery.py
index f6e1d4f06b..89c24f2402 100644
--- a/openpype/plugins/load/delivery.py
+++ b/openpype/plugins/load/delivery.py
@@ -7,15 +7,17 @@ from openpype.client import get_representations
from openpype.pipeline import load, Anatomy
from openpype import resources, style
-from openpype.lib.dateutils import get_datetime_data
-from openpype.lib.delivery import (
- sizeof_fmt,
- path_from_representation,
+from openpype.lib import (
+ format_file_size,
+ collect_frames,
+ get_datetime_data,
+)
+from openpype.pipeline.load import get_representation_path_with_anatomy
+from openpype.pipeline.delivery import (
get_format_dict,
check_destination_path,
- process_single_file,
- process_sequence,
- collect_frames
+ deliver_single_file,
+ deliver_sequence,
)
@@ -167,7 +169,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
if repre["name"] not in selected_repres:
continue
- repre_path = path_from_representation(repre, self.anatomy)
+ repre_path = get_representation_path_with_anatomy(
+ repre, self.anatomy
+ )
anatomy_data = copy.deepcopy(repre["context"])
new_report_items = check_destination_path(str(repre["_id"]),
@@ -202,7 +206,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
args[0] = src_path
if frame:
anatomy_data["frame"] = frame
- new_report_items, uploaded = process_single_file(*args)
+ new_report_items, uploaded = deliver_single_file(*args)
report_items.update(new_report_items)
self._update_progress(uploaded)
else: # fallback for Pype2 and representations without files
@@ -211,9 +215,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
repre["context"]["frame"] = len(str(frame)) * "#"
if not frame:
- new_report_items, uploaded = process_single_file(*args)
+ new_report_items, uploaded = deliver_single_file(*args)
else:
- new_report_items, uploaded = process_sequence(*args)
+ new_report_items, uploaded = deliver_sequence(*args)
report_items.update(new_report_items)
self._update_progress(uploaded)
@@ -263,8 +267,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
def _prepare_label(self):
"""Provides text with no of selected files and their size."""
- label = "{} files, size {}".format(self.files_selected,
- sizeof_fmt(self.size_selected))
+ label = "{} files, size {}".format(
+ self.files_selected,
+ format_file_size(self.size_selected))
return label
def _get_selected_repres(self):