Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3751 from pypeclub/feature/OP-3844_Move-delivery-logic-to-pipeline

Browse source 
General: Move delivery logic to pipeline
This commit is contained in:
Jakub Trllo 2022-08-29 16:45:02 +02:00 • committed by GitHub
commit e13179a167
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 691 additions and 342 deletions
Download patch file Download diff file Expand all files Collapse all files

4
openpype/lib/__init__.py
View file

@ -192,6 +192,8 @@ from .plugin_tools import (
)
from .path_tools import (
format_file_size,
collect_frames,
create_hard_link,
version_up,
get_version_from_path,
@ -353,6 +355,8 @@ __all__ = [
"set_plugin_attributes_from_settings",
"source_hash",
"format_file_size",
"collect_frames",
"create_hard_link",
"version_up",
"get_version_from_path",

448
openpype/lib/delivery.py
View file

@ -1,81 +1,113 @@
"""Functions useful for delivery action or loader"""
import os
import shutil
import glob
import clique
import collections
from .path_templates import (
StringTemplate,
TemplateUnsolved,
)
import functools
import warnings
class DeliveryDeprecatedWarning(DeprecationWarning):
pass
def deprecated(new_destination):
"""Mark functions as deprecated.
It will result in a warning being emitted when the function is used.
"""
func = None
if callable(new_destination):
func = new_destination
new_destination = None
def _decorator(decorated_func):
if new_destination is None:
warning_message = (
" Please check content of deprecated function to figure out"
" possible replacement."
)
else:
warning_message = " Please replace your usage with '{}'.".format(
new_destination
)
@functools.wraps(decorated_func)
def wrapper(*args, **kwargs):
warnings.simplefilter("always", DeliveryDeprecatedWarning)
warnings.warn(
(
"Call to deprecated function '{}'"
"\nFunction was moved or removed.{}"
).format(decorated_func.__name__, warning_message),
category=DeliveryDeprecatedWarning,
stacklevel=4
)
return decorated_func(*args, **kwargs)
return wrapper
if func is None:
return _decorator
return _decorator(func)
@deprecated("openpype.lib.path_tools.collect_frames")
def collect_frames(files):
"""Returns dict of source path and its frame, if from sequence
Uses clique as most precise solution, used when anatomy template that
created files is not known.
Assumption is that frames are separated by '.', negative frames are not
allowed.
Args:
files(list) or (set with single value): list of source paths
Returns:
(dict): {'/asset/subset_v001.0001.png': '0001', ....}
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
Returns dict of source path and its frame, if from sequence
Uses clique as most precise solution, used when anatomy template that
created files is not known.
from .path_tools import collect_frames
Assumption is that frames are separated by '.', negative frames are not
allowed.
return collect_frames(files)
Args:
files(list) or (set with single value): list of source paths
Returns:
(dict): {'/asset/subset_v001.0001.png': '0001', ....}
@deprecated("openpype.lib.path_tools.format_file_size")
def sizeof_fmt(num, suffix=None):
"""Returns formatted string with size in appropriate unit
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
patterns = [clique.PATTERNS["frames"]]
collections, remainder = clique.assemble(files, minimum_items=1,
patterns=patterns)
sources_and_frames = {}
if collections:
for collection in collections:
src_head = collection.head
src_tail = collection.tail
for index in collection.indexes:
src_frame = collection.format("{padding}") % index
src_file_name = "{}{}{}".format(src_head, src_frame,
src_tail)
sources_and_frames[src_file_name] = src_frame
else:
sources_and_frames[remainder.pop()] = None
return sources_and_frames
def sizeof_fmt(num, suffix='B'):
"""Returns formatted string with size in appropriate unit"""
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
from .path_tools import format_file_size
return format_file_size(num, suffix)
@deprecated("openpype.pipeline.load.get_representation_path_with_anatomy")
def path_from_representation(representation, anatomy):
try:
template = representation["data"]["template"]
"""Get representation path using representation document and anatomy.
except KeyError:
return None
Args:
representation (Dict[str, Any]): Representation document.
anatomy (Anatomy): Project anatomy.
try:
context = representation["context"]
context["root"] = anatomy.roots
path = StringTemplate.format_strict_template(template, context)
return os.path.normpath(path)
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
except TemplateUnsolved:
# Template references unavailable data
return None
from openpype.pipeline.load import get_representation_path_with_anatomy
return path
return get_representation_path_with_anatomy(representation, anatomy)
@deprecated
def copy_file(src_path, dst_path):
"""Hardlink file if possible(to save space), copy if not"""
from openpype.lib import create_hard_link # safer importing
@ -91,131 +123,96 @@ def copy_file(src_path, dst_path):
shutil.copyfile(src_path, dst_path)
@deprecated("openpype.pipeline.delivery.get_format_dict")
def get_format_dict(anatomy, location_path):
"""Returns replaced root values from user provider value.
Args:
anatomy (Anatomy)
location_path (str): user provided value
Returns:
(dict): prepared for formatting of a template
Args:
anatomy (Anatomy)
location_path (str): user provided value
Returns:
(dict): prepared for formatting of a template
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
format_dict = {}
if location_path:
location_path = location_path.replace("\\", "/")
root_names = anatomy.root_names_from_templates(
anatomy.templates["delivery"]
)
if root_names is None:
format_dict["root"] = location_path
else:
format_dict["root"] = {}
for name in root_names:
format_dict["root"][name] = location_path
return format_dict
from openpype.pipeline.delivery import get_format_dict
return get_format_dict(anatomy, location_path)
@deprecated("openpype.pipeline.delivery.check_destination_path")
def check_destination_path(repre_id,
anatomy, anatomy_data,
datetime_data, template_name):
""" Try to create destination path based on 'template_name'.
In the case that path cannot be filled, template contains unmatched
keys, provide error message to filter out repre later.
In the case that path cannot be filled, template contains unmatched
keys, provide error message to filter out repre later.
Args:
anatomy (Anatomy)
anatomy_data (dict): context to fill anatomy
datetime_data (dict): values with actual date
template_name (str): to pick correct delivery template
Returns:
(collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"}
Args:
anatomy (Anatomy)
anatomy_data (dict): context to fill anatomy
datetime_data (dict): values with actual date
template_name (str): to pick correct delivery template
Returns:
(collections.defauldict): {"TYPE_OF_ERROR":"ERROR_DETAIL"}
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
anatomy_data.update(datetime_data)
anatomy_filled = anatomy.format_all(anatomy_data)
dest_path = anatomy_filled["delivery"][template_name]
report_items = collections.defaultdict(list)
if not dest_path.solved:
msg = (
"Missing keys in Representation's context"
" for anatomy template \"{}\"."
).format(template_name)
from openpype.pipeline.delivery import check_destination_path
sub_msg = (
"Representation: {}<br>"
).format(repre_id)
if dest_path.missing_keys:
keys = ", ".join(dest_path.missing_keys)
sub_msg += (
"- Missing keys: \"{}\"<br>"
).format(keys)
if dest_path.invalid_types:
items = []
for key, value in dest_path.invalid_types.items():
items.append("\"{}\" {}".format(key, str(value)))
keys = ", ".join(items)
sub_msg += (
"- Invalid value DataType: \"{}\"<br>"
).format(keys)
report_items[msg].append(sub_msg)
return report_items
return check_destination_path(
repre_id,
anatomy,
anatomy_data,
datetime_data,
template_name
)
@deprecated("openpype.pipeline.delivery.deliver_single_file")
def process_single_file(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
):
"""Copy single file to calculated path based on template
Args:
src_path(str): path of source representation file
_repre (dict): full repre, used only in process_sequence, here only
as to share same signature
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (Logger): for log printing
Returns:
(collections.defaultdict , int)
Args:
src_path(str): path of source representation file
_repre (dict): full repre, used only in process_sequence, here only
as to share same signature
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (Logger): for log printing
Returns:
(collections.defaultdict , int)
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
# Make sure path is valid for all platforms
src_path = os.path.normpath(src_path.replace("\\", "/"))
if not os.path.exists(src_path):
msg = "{} doesn't exist for {}".format(src_path, repre["_id"])
report_items["Source file was not found"].append(msg)
return report_items, 0
from openpype.pipeline.delivery import deliver_single_file
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][template_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][template_name]
# Backwards compatibility when extension contained `.`
delivery_path = delivery_path.replace("..", ".")
# Make sure path is valid for all platforms
delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
delivery_folder = os.path.dirname(delivery_path)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
log.debug("Copying single: {} -> {}".format(src_path, delivery_path))
copy_file(src_path, delivery_path)
return report_items, 1
return deliver_single_file(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
)
@deprecated("openpype.pipeline.delivery.deliver_sequence")
def process_sequence(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
@ -223,128 +220,33 @@ def process_sequence(
""" For Pype2(mainly - works in 3 too) where representation might not
contain files.
Uses listing physical files (not 'files' on repre as a)might not be
present, b)might not be reliable for representation and copying them.
Uses listing physical files (not 'files' on repre as a)might not be
present, b)might not be reliable for representation and copying them.
TODO Should be refactored when files are sufficient to drive all
representations.
TODO Should be refactored when files are sufficient to drive all
representations.
Args:
src_path(str): path of source representation file
repre (dict): full representation
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (Logger): for log printing
Returns:
(collections.defaultdict , int)
Args:
src_path(str): path of source representation file
repre (dict): full representation
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (Logger): for log printing
Returns:
(collections.defaultdict , int)
Deprecated:
Function was moved to different location and will be removed
after 3.16.* release.
"""
src_path = os.path.normpath(src_path.replace("\\", "/"))
def hash_path_exist(myPath):
res = myPath.replace('#', '*')
glob_search_results = glob.glob(res)
if len(glob_search_results) > 0:
return True
return False
from openpype.pipeline.delivery import deliver_sequence
if not hash_path_exist(src_path):
msg = "{} doesn't exist for {}".format(src_path,
repre["_id"])
report_items["Source file was not found"].append(msg)
return report_items, 0
delivery_templates = anatomy.templates.get("delivery") or {}
delivery_template = delivery_templates.get(template_name)
if delivery_template is None:
msg = (
"Delivery template \"{}\" in anatomy of project \"{}\""
" was not found"
).format(template_name, anatomy.project_name)
report_items[""].append(msg)
return report_items, 0
# Check if 'frame' key is available in template which is required
# for sequence delivery
if "{frame" not in delivery_template:
msg = (
"Delivery template \"{}\" in anatomy of project \"{}\""
"does not contain '{{frame}}' key to fill. Delivery of sequence"
" can't be processed."
).format(template_name, anatomy.project_name)
report_items[""].append(msg)
return report_items, 0
dir_path, file_name = os.path.split(str(src_path))
context = repre["context"]
ext = context.get("ext", context.get("representation"))
if not ext:
msg = "Source extension not found, cannot find collection"
report_items[msg].append(src_path)
log.warning("{} <{}>".format(msg, context))
return report_items, 0
ext = "." + ext
# context.representation could be .psd
ext = ext.replace("..", ".")
src_collections, remainder = clique.assemble(os.listdir(dir_path))
src_collection = None
for col in src_collections:
if col.tail != ext:
continue
src_collection = col
break
if src_collection is None:
msg = "Source collection of files was not found"
report_items[msg].append(src_path)
log.warning("{} <{}>".format(msg, src_path))
return report_items, 0
frame_indicator = "@####@"
anatomy_data["frame"] = frame_indicator
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][template_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][template_name]
delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
delivery_folder = os.path.dirname(delivery_path)
dst_head, dst_tail = delivery_path.split(frame_indicator)
dst_padding = src_collection.padding
dst_collection = clique.Collection(
head=dst_head,
tail=dst_tail,
padding=dst_padding
return deliver_sequence(
src_path, repre, anatomy, template_name, anatomy_data, format_dict,
report_items, log
)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
src_head = src_collection.head
src_tail = src_collection.tail
uploaded = 0
for index in src_collection.indexes:
src_padding = src_collection.format("{padding}") % index
src_file_name = "{}{}{}".format(src_head, src_padding, src_tail)
src = os.path.normpath(
os.path.join(dir_path, src_file_name)
)
dst_padding = dst_collection.format("{padding}") % index
dst = "{}{}{}".format(dst_head, dst_padding, dst_tail)
log.debug("Copying single: {} -> {}".format(src, dst))
copy_file(src, dst)
uploaded += 1
return report_items, uploaded

71
openpype/lib/path_tools.py
View file

@ -6,6 +6,8 @@ import logging
import six
import platform
import clique
from openpype.client import get_project
from openpype.settings import get_project_settings
@ -14,6 +16,27 @@ from .profiles_filtering import filter_profiles
log = logging.getLogger(__name__)
def format_file_size(file_size, suffix=None):
"""Returns formatted string with size in appropriate unit.
Args:
file_size (int): Size of file in bytes.
suffix (str): Suffix for formatted size. Default is 'B' (as bytes).
Returns:
str: Formatted size using proper unit and passed suffix (e.g. 7 MiB).
"""
if suffix is None:
suffix = "B"
for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
if abs(file_size) < 1024.0:
return "%3.1f%s%s" % (file_size, unit, suffix)
file_size /= 1024.0
return "%.1f%s%s" % (file_size, "Yi", suffix)
def create_hard_link(src_path, dst_path):
"""Create hardlink of file.
@ -50,6 +73,43 @@ def create_hard_link(src_path, dst_path):
)
def collect_frames(files):
"""Returns dict of source path and its frame, if from sequence
Uses clique as most precise solution, used when anatomy template that
created files is not known.
Assumption is that frames are separated by '.', negative frames are not
allowed.
Args:
files(list) or (set with single value): list of source paths
Returns:
(dict): {'/asset/subset_v001.0001.png': '0001', ....}
"""
patterns = [clique.PATTERNS["frames"]]
collections, remainder = clique.assemble(
files, minimum_items=1, patterns=patterns)
sources_and_frames = {}
if collections:
for collection in collections:
src_head = collection.head
src_tail = collection.tail
for index in collection.indexes:
src_frame = collection.format("{padding}") % index
src_file_name = "{}{}{}".format(
src_head, src_frame, src_tail)
sources_and_frames[src_file_name] = src_frame
else:
sources_and_frames[remainder.pop()] = None
return sources_and_frames
def _rreplace(s, a, b, n=1):
"""Replace a with b in string s from right side n times."""
return b.join(s.rsplit(a, n))
@ -119,12 +179,12 @@ def get_version_from_path(file):
"""Find version number in file path string.
Args:
file (string): file path
file (str): file path
Returns:
v: version number in string ('001')
str: version number in string ('001')
"""
pattern = re.compile(r"[\._]v([0-9]+)", re.IGNORECASE)
try:
return pattern.findall(file)[-1]
@ -140,16 +200,17 @@ def get_last_version_from_path(path_dir, filter):
"""Find last version of given directory content.
Args:
path_dir (string): directory path
path_dir (str): directory path
filter (list): list of strings used as file name filter
Returns:
string: file name with last version
str: file name with last version
Example:
last_version_file = get_last_version_from_path(
"/project/shots/shot01/work", ["shot01", "compositing", "nk"])
"""
assert os.path.isdir(path_dir), "`path_dir` argument needs to be directory"
assert isinstance(filter, list) and (
len(filter) != 0), "`filter` argument needs to be list and not empty"

6
openpype/modules/deadline/plugins/publish/submit_aftereffects_deadline.py
View file

@ -3,8 +3,10 @@ import attr
import getpass
import pyblish.api
from openpype.lib import env_value_to_bool
from openpype.lib.delivery import collect_frames
from openpype.lib import (
env_value_to_bool,
collect_frames,
)
from openpype.pipeline import legacy_io
from openpype_modules.deadline import abstract_submit_deadline
from openpype_modules.deadline.abstract_submit_deadline import DeadlineJobInfo

2
openpype/modules/deadline/plugins/publish/validate_expected_and_rendered_files.py
View file

@ -3,7 +3,7 @@ import requests
import pyblish.api
from openpype.lib.delivery import collect_frames
from openpype.lib import collect_frames
from openpype_modules.deadline.abstract_submit_deadline import requests_get

17
openpype/modules/ftrack/event_handlers_user/action_delete_old_versions.py
View file

@ -11,7 +11,11 @@ from openpype.client import (
get_versions,
get_representations
)
from openpype.lib import StringTemplate, TemplateUnsolved
from openpype.lib import (
StringTemplate,
TemplateUnsolved,
format_file_size,
)
from openpype.pipeline import AvalonMongoDB, Anatomy
from openpype_modules.ftrack.lib import BaseAction, statics_icon
@ -134,13 +138,6 @@ class DeleteOldVersions(BaseAction):
"title": self.inteface_title
}
def sizeof_fmt(self, num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def launch(self, session, entities, event):
values = event["data"].get("values")
if not values:
@ -359,7 +356,7 @@ class DeleteOldVersions(BaseAction):
dir_paths, file_paths_by_dir, delete=False
)
msg = "Total size of files: " + self.sizeof_fmt(size)
msg = "Total size of files: {}".format(format_file_size(size))
self.log.warning(msg)
@ -430,7 +427,7 @@ class DeleteOldVersions(BaseAction):
"message": msg
}
msg = "Total size of files deleted: " + self.sizeof_fmt(size)
msg = "Total size of files deleted: {}".format(format_file_size(size))
self.log.warning(msg)

16
openpype/modules/ftrack/event_handlers_user/action_delivery.py
View file

@ -10,19 +10,19 @@ from openpype.client import (
get_versions,
get_representations
)
from openpype.pipeline import Anatomy
from openpype_modules.ftrack.lib import BaseAction, statics_icon
from openpype_modules.ftrack.lib.avalon_sync import CUST_ATTR_ID_KEY
from openpype_modules.ftrack.lib.custom_attributes import (
query_custom_attributes
)
from openpype.lib.dateutils import get_datetime_data
from openpype.lib.delivery import (
path_from_representation,
from openpype.pipeline import Anatomy
from openpype.pipeline.load import get_representation_path_with_anatomy
from openpype.pipeline.delivery import (
get_format_dict,
check_destination_path,
process_single_file,
process_sequence
deliver_single_file,
deliver_sequence,
)
@ -580,7 +580,7 @@ class Delivery(BaseAction):
if frame:
repre["context"]["frame"] = len(str(frame)) * "#"
repre_path = path_from_representation(repre, anatomy)
repre_path = get_representation_path_with_anatomy(repre, anatomy)
# TODO add backup solution where root of path from component
# is replaced with root
args = (
@ -594,9 +594,9 @@ class Delivery(BaseAction):
self.log
)
if not frame:
process_single_file(*args)
deliver_single_file(*args)
else:
process_sequence(*args)
deliver_sequence(*args)
return self.report(report_items)

310
openpype/pipeline/delivery.py Normal file
View file

@ -0,0 +1,310 @@
"""Functions useful for delivery of published representations."""
import os
import shutil
import glob
import clique
import collections
from openpype.lib import create_hard_link
def _copy_file(src_path, dst_path):
"""Hardlink file if possible(to save space), copy if not.
Because of using hardlinks should not be function used in other parts
of pipeline.
"""
if os.path.exists(dst_path):
return
try:
create_hard_link(
src_path,
dst_path
)
except OSError:
shutil.copyfile(src_path, dst_path)
def get_format_dict(anatomy, location_path):
"""Returns replaced root values from user provider value.
Args:
anatomy (Anatomy): Project anatomy.
location_path (str): User provided value.
Returns:
(dict): Prepared data for formatting of a template.
"""
format_dict = {}
if not location_path:
return format_dict
location_path = location_path.replace("\\", "/")
root_names = anatomy.root_names_from_templates(
anatomy.templates["delivery"]
)
format_dict["root"] = {}
for name in root_names:
format_dict["root"][name] = location_path
return format_dict
def check_destination_path(
repre_id,
anatomy,
anatomy_data,
datetime_data,
template_name
):
""" Try to create destination path based on 'template_name'.
In the case that path cannot be filled, template contains unmatched
keys, provide error message to filter out repre later.
Args:
repre_id (str): Representation id.
anatomy (Anatomy): Project anatomy.
anatomy_data (dict): Template data to fill anatomy templates.
datetime_data (dict): Values with actual date.
template_name (str): Name of template which should be used from anatomy
templates.
Returns:
Dict[str, List[str]]: Report of happened errors. Key is message title
value is detailed information.
"""
anatomy_data.update(datetime_data)
anatomy_filled = anatomy.format_all(anatomy_data)
dest_path = anatomy_filled["delivery"][template_name]
report_items = collections.defaultdict(list)
if not dest_path.solved:
msg = (
"Missing keys in Representation's context"
" for anatomy template \"{}\"."
).format(template_name)
sub_msg = (
"Representation: {}<br>"
).format(repre_id)
if dest_path.missing_keys:
keys = ", ".join(dest_path.missing_keys)
sub_msg += (
"- Missing keys: \"{}\"<br>"
).format(keys)
if dest_path.invalid_types:
items = []
for key, value in dest_path.invalid_types.items():
items.append("\"{}\" {}".format(key, str(value)))
keys = ", ".join(items)
sub_msg += (
"- Invalid value DataType: \"{}\"<br>"
).format(keys)
report_items[msg].append(sub_msg)
return report_items
def deliver_single_file(
src_path,
repre,
anatomy,
template_name,
anatomy_data,
format_dict,
report_items,
log
):
"""Copy single file to calculated path based on template
Args:
src_path(str): path of source representation file
repre (dict): full repre, used only in deliver_sequence, here only
as to share same signature
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (logging.Logger): for log printing
Returns:
(collections.defaultdict, int)
"""
# Make sure path is valid for all platforms
src_path = os.path.normpath(src_path.replace("\\", "/"))
if not os.path.exists(src_path):
msg = "{} doesn't exist for {}".format(src_path, repre["_id"])
report_items["Source file was not found"].append(msg)
return report_items, 0
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][template_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][template_name]
# Backwards compatibility when extension contained `.`
delivery_path = delivery_path.replace("..", ".")
# Make sure path is valid for all platforms
delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
delivery_folder = os.path.dirname(delivery_path)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
log.debug("Copying single: {} -> {}".format(src_path, delivery_path))
_copy_file(src_path, delivery_path)
return report_items, 1
def deliver_sequence(
src_path,
repre,
anatomy,
template_name,
anatomy_data,
format_dict,
report_items,
log
):
""" For Pype2(mainly - works in 3 too) where representation might not
contain files.
Uses listing physical files (not 'files' on repre as a)might not be
present, b)might not be reliable for representation and copying them.
TODO Should be refactored when files are sufficient to drive all
representations.
Args:
src_path(str): path of source representation file
repre (dict): full representation
anatomy (Anatomy)
template_name (string): user selected delivery template name
anatomy_data (dict): data from repre to fill anatomy with
format_dict (dict): root dictionary with names and values
report_items (collections.defaultdict): to return error messages
log (logging.Logger): for log printing
Returns:
(collections.defaultdict, int)
"""
src_path = os.path.normpath(src_path.replace("\\", "/"))
def hash_path_exist(myPath):
res = myPath.replace('#', '*')
glob_search_results = glob.glob(res)
if len(glob_search_results) > 0:
return True
return False
if not hash_path_exist(src_path):
msg = "{} doesn't exist for {}".format(
src_path, repre["_id"])
report_items["Source file was not found"].append(msg)
return report_items, 0
delivery_templates = anatomy.templates.get("delivery") or {}
delivery_template = delivery_templates.get(template_name)
if delivery_template is None:
msg = (
"Delivery template \"{}\" in anatomy of project \"{}\""
" was not found"
).format(template_name, anatomy.project_name)
report_items[""].append(msg)
return report_items, 0
# Check if 'frame' key is available in template which is required
# for sequence delivery
if "{frame" not in delivery_template:
msg = (
"Delivery template \"{}\" in anatomy of project \"{}\""
"does not contain '{{frame}}' key to fill. Delivery of sequence"
" can't be processed."
).format(template_name, anatomy.project_name)
report_items[""].append(msg)
return report_items, 0
dir_path, file_name = os.path.split(str(src_path))
context = repre["context"]
ext = context.get("ext", context.get("representation"))
if not ext:
msg = "Source extension not found, cannot find collection"
report_items[msg].append(src_path)
log.warning("{} <{}>".format(msg, context))
return report_items, 0
ext = "." + ext
# context.representation could be .psd
ext = ext.replace("..", ".")
src_collections, remainder = clique.assemble(os.listdir(dir_path))
src_collection = None
for col in src_collections:
if col.tail != ext:
continue
src_collection = col
break
if src_collection is None:
msg = "Source collection of files was not found"
report_items[msg].append(src_path)
log.warning("{} <{}>".format(msg, src_path))
return report_items, 0
frame_indicator = "@####@"
anatomy_data["frame"] = frame_indicator
anatomy_filled = anatomy.format(anatomy_data)
if format_dict:
template_result = anatomy_filled["delivery"][template_name]
delivery_path = template_result.rootless.format(**format_dict)
else:
delivery_path = anatomy_filled["delivery"][template_name]
delivery_path = os.path.normpath(delivery_path.replace("\\", "/"))
delivery_folder = os.path.dirname(delivery_path)
dst_head, dst_tail = delivery_path.split(frame_indicator)
dst_padding = src_collection.padding
dst_collection = clique.Collection(
head=dst_head,
tail=dst_tail,
padding=dst_padding
)
if not os.path.exists(delivery_folder):
os.makedirs(delivery_folder)
src_head = src_collection.head
src_tail = src_collection.tail
uploaded = 0
for index in src_collection.indexes:
src_padding = src_collection.format("{padding}") % index
src_file_name = "{}{}{}".format(src_head, src_padding, src_tail)
src = os.path.normpath(
os.path.join(dir_path, src_file_name)
)
dst_padding = dst_collection.format("{padding}") % index
dst = "{}{}{}".format(dst_head, dst_padding, dst_tail)
log.debug("Copying single: {} -> {}".format(src, dst))
_copy_file(src, dst)
uploaded += 1
return report_items, uploaded

6
openpype/pipeline/load/__init__.py
View file

@ -1,6 +1,8 @@
from .utils import (
HeroVersionType,
IncompatibleLoaderError,
InvalidRepresentationContext,
get_repres_contexts,
get_subset_contexts,
@ -20,6 +22,7 @@ from .utils import (
get_representation_path_from_context,
get_representation_path,
get_representation_path_with_anatomy,
is_compatible_loader,
@ -46,7 +49,9 @@ from .plugins import (
__all__ = (
# utils.py
"HeroVersionType",
"IncompatibleLoaderError",
"InvalidRepresentationContext",
"get_repres_contexts",
"get_subset_contexts",
@ -66,6 +71,7 @@ __all__ = (
"get_representation_path_from_context",
"get_representation_path",
"get_representation_path_with_anatomy",
"is_compatible_loader",

58
openpype/pipeline/load/utils.py
View file

@ -23,6 +23,10 @@ from openpype.client import (
get_representation_by_name,
get_representation_parents
)
from openpype.lib import (
StringTemplate,
TemplateUnsolved,
)
from openpype.pipeline import (
schema,
legacy_io,
@ -61,6 +65,11 @@ class IncompatibleLoaderError(ValueError):
pass
class InvalidRepresentationContext(ValueError):
"""Representation path can't be received using representation document."""
pass
def get_repres_contexts(representation_ids, dbcon=None):
"""Return parenthood context for representation.
@ -515,6 +524,52 @@ def get_representation_path_from_context(context):
return get_representation_path(representation, root)
def get_representation_path_with_anatomy(repre_doc, anatomy):
"""Receive representation path using representation document and anatomy.
Anatomy is used to replace 'root' key in representation file. Ideally
should be used instead of 'get_representation_path' which is based on
"current context".
Future notes:
We want also be able store resources into representation and I can
imagine the result should also contain paths to possible resources.
Args:
repre_doc (Dict[str, Any]): Representation document.
anatomy (Anatomy): Project anatomy object.
Returns:
Union[None, TemplateResult]: None if path can't be received
Raises:
InvalidRepresentationContext: When representation data are probably
invalid or not available.
"""
try:
template = repre_doc["data"]["template"]
except KeyError:
raise InvalidRepresentationContext((
"Representation document does not"
" contain template in data ('data.template')"
))
try:
context = repre_doc["context"]
context["root"] = anatomy.roots
path = StringTemplate.format_strict_template(template, context)
except TemplateUnsolved as exc:
raise InvalidRepresentationContext((
"Couldn't resolve representation template with available data."
" Reason: {}".format(str(exc))
))
return path.normalized()
def get_representation_path(representation, root=None, dbcon=None):
"""Get filename from representation document
@ -533,8 +588,6 @@ def get_representation_path(representation, root=None, dbcon=None):
"""
from openpype.lib import StringTemplate, TemplateUnsolved
if dbcon is None:
dbcon = legacy_io
@ -737,6 +790,7 @@ def get_outdated_containers(host=None, project_name=None):
if host is None:
from openpype.pipeline import registered_host
host = registered_host()
if project_name is None:

14
openpype/pipeline/template_data.py
View file

@ -28,27 +28,37 @@ def get_general_template_data(system_settings=None):
}
def get_project_template_data(project_doc):
def get_project_template_data(project_doc=None, project_name=None):
"""Extract data from project document that are used in templates.
Project document must have 'name' and (at this moment) optional
key 'data.code'.
One of 'project_name' or 'project_doc' must be passed. With prepared
project document is function much faster because don't have to query.
Output contains formatting keys:
- 'project[name]' - Project name
- 'project[code]' - Project code
Args:
project_doc (Dict[str, Any]): Queried project document.
project_name (str): Name of project.
Returns:
Dict[str, Dict[str, str]]: Template data based on project document.
"""
if not project_name:
project_name = project_doc["name"]
if not project_doc:
project_code = get_project(project_name, fields=["data.code"])
project_code = project_doc.get("data", {}).get("code")
return {
"project": {
"name": project_doc["name"],
"name": project_name,
"code": project_code
}
}

50
openpype/plugins/load/delete_old_versions.py
View file

@ -7,11 +7,15 @@ from pymongo import UpdateOne
import qargparse
from Qt import QtWidgets, QtCore
from openpype.client import get_versions, get_representations
from openpype import style
from openpype.pipeline import load, AvalonMongoDB, Anatomy
from openpype.lib import StringTemplate
from openpype.client import get_versions, get_representations
from openpype.modules import ModulesManager
from openpype.lib import format_file_size
from openpype.pipeline import load, AvalonMongoDB, Anatomy
from openpype.pipeline.load import (
get_representation_path_with_anatomy,
InvalidRepresentationContext,
)
class DeleteOldVersions(load.SubsetLoaderPlugin):
@ -38,13 +42,6 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
)
]
def sizeof_fmt(self, num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def delete_whole_dir_paths(self, dir_paths, delete=True):
size = 0
@ -80,27 +77,28 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
def path_from_representation(self, representation, anatomy):
try:
template = representation["data"]["template"]
context = representation["context"]
except KeyError:
return (None, None)
try:
path = get_representation_path_with_anatomy(
representation, anatomy
)
except InvalidRepresentationContext:
return (None, None)
sequence_path = None
try:
context = representation["context"]
context["root"] = anatomy.roots
path = str(StringTemplate.format_template(template, context))
if "frame" in context:
context["frame"] = self.sequence_splitter
sequence_path = os.path.normpath(str(
StringTemplate.format_template(template, context)
))
if "frame" in context:
context["frame"] = self.sequence_splitter
sequence_path = get_representation_path_with_anatomy(
representation, anatomy
)
except KeyError:
# Template references unavailable data
return (None, None)
if sequence_path:
sequence_path = sequence_path.normalized()
return (os.path.normpath(path), sequence_path)
return (path.normalized(), sequence_path)
def delete_only_repre_files(self, dir_paths, file_paths, delete=True):
size = 0
@ -456,7 +454,7 @@ class DeleteOldVersions(load.SubsetLoaderPlugin):
size += self.main(project_name, data, remove_publish_folder)
print("Progressing {}/{}".format(count + 1, len(contexts)))
msg = "Total size of files: " + self.sizeof_fmt(size)
msg = "Total size of files: {}".format(format_file_size(size))
self.log.info(msg)
self.message(msg)

31
openpype/plugins/load/delivery.py
View file

@ -7,15 +7,17 @@ from openpype.client import get_representations
from openpype.pipeline import load, Anatomy
from openpype import resources, style
from openpype.lib.dateutils import get_datetime_data
from openpype.lib.delivery import (
sizeof_fmt,
path_from_representation,
from openpype.lib import (
format_file_size,
collect_frames,
get_datetime_data,
)
from openpype.pipeline.load import get_representation_path_with_anatomy
from openpype.pipeline.delivery import (
get_format_dict,
check_destination_path,
process_single_file,
process_sequence,
collect_frames
deliver_single_file,
deliver_sequence,
)
@ -167,7 +169,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
if repre["name"] not in selected_repres:
continue
repre_path = path_from_representation(repre, self.anatomy)
repre_path = get_representation_path_with_anatomy(
repre, self.anatomy
)
anatomy_data = copy.deepcopy(repre["context"])
new_report_items = check_destination_path(str(repre["_id"]),
@ -202,7 +206,7 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
args[0] = src_path
if frame:
anatomy_data["frame"] = frame
new_report_items, uploaded = process_single_file(*args)
new_report_items, uploaded = deliver_single_file(*args)
report_items.update(new_report_items)
self._update_progress(uploaded)
else: # fallback for Pype2 and representations without files
@ -211,9 +215,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
repre["context"]["frame"] = len(str(frame)) * "#"
if not frame:
new_report_items, uploaded = process_single_file(*args)
new_report_items, uploaded = deliver_single_file(*args)
else:
new_report_items, uploaded = process_sequence(*args)
new_report_items, uploaded = deliver_sequence(*args)
report_items.update(new_report_items)
self._update_progress(uploaded)
@ -263,8 +267,9 @@ class DeliveryOptionsDialog(QtWidgets.QDialog):
def _prepare_label(self):
"""Provides text with no of selected files and their size."""
label = "{} files, size {}".format(self.files_selected,
sizeof_fmt(self.size_selected))
label = "{} files, size {}".format(
self.files_selected,
format_file_size(self.size_selected))
return label
def _get_selected_repres(self):