Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 12:54:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into bugfix/1032-filedef-doesnt-work-properly-in-houdini

Browse source
This commit is contained in:
Jakub Trllo 2024-12-10 14:39:45 +01:00 • committed by GitHub
commit cabd483274
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 1419 additions and 757 deletions
Download patch file Download diff file Expand all files Collapse all files

35
.github/workflows/assign_pr_to_project.yml vendored
View file

@ -3,25 +3,46 @@ on:
workflow_dispatch:
inputs:
pr_number:
type: number
type: string
description: "Run workflow for this PR number"
required: true
project_id:
type: number
type: string
description: "Github Project Number"
required: true
default: 16
default: "16"
pull_request:
types:
- opened
env:
GH_TOKEN: ${{ github.token }}
jobs:
get-pr-repo:
runs-on: ubuntu-latest
outputs:
pr_repo_name: ${{ steps.get-repo-name.outputs.repo_name || github.event.pull_request.head.repo.full_name }}
# INFO `github.event.pull_request.head.repo.full_name` is not available on manual triggered (dispatched) runs
steps:
- name: Get PR repo name
if: ${{ github.event_name == 'workflow_dispatch' }}
id: get-repo-name
run: |
repo_name=$(gh pr view ${{ inputs.pr_number }} --json headRepository,headRepositoryOwner --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name')
echo "repo_name=$repo_name" >> $GITHUB_OUTPUT
auto-assign-pr:
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
needs:
- get-pr-repo
if: ${{ needs.get-pr-repo.outputs.pr_repo_name == github.repository }}
uses: ynput/ops-repo-automation/.github/workflows/pr_to_project.yml@main
with:
repo: "${{ github.repository }}"
project_id: "${{ inputs.project_id }}"
pull_request_number: "${{ github.event.pull_request.number || inputs.pr_number }}"
project_id: ${{ inputs.project_id != '' && fromJSON(inputs.project_id) || 16 }}
pull_request_number: ${{ github.event.pull_request.number || fromJSON(inputs.pr_number) }}
secrets:
token: ${{ secrets.YNPUT_BOT_TOKEN }}
# INFO fallback to default `github.token` is required for PRs from forks
# INFO organization secrets won't be available to forks
token: ${{ secrets.YNPUT_BOT_TOKEN || github.token}}

351
client/ayon_core/lib/path_templates.py
View file

@ -1,9 +1,15 @@
import os
import re
import copy
import numbers
import warnings
from string import Formatter
import typing
from typing import List, Dict, Any, Set
if typing.TYPE_CHECKING:
from typing import Union
KEY_PATTERN = re.compile(r"(\{.*?[^{0]*\})")
KEY_PADDING_PATTERN = re.compile(r"([^:]+)\S+[><]\S+")
SUB_DICT_PATTERN = re.compile(r"([^\[\]]+)")
OPTIONAL_PATTERN = re.compile(r"(<.*?[^{0]*>)[^0-9]*?")
@ -18,9 +24,7 @@ class TemplateUnsolved(Exception):
def __init__(self, template, missing_keys, invalid_types):
invalid_type_items = []
for _key, _type in invalid_types.items():
invalid_type_items.append(
"\"{0}\" {1}".format(_key, str(_type))
)
invalid_type_items.append(f"\"{_key}\" {str(_type)}")
invalid_types_msg = ""
if invalid_type_items:
@ -33,31 +37,32 @@ class TemplateUnsolved(Exception):
missing_keys_msg = self.missing_keys_msg.format(
", ".join(missing_keys)
)
super(TemplateUnsolved, self).__init__(
super().__init__(
self.msg.format(template, missing_keys_msg, invalid_types_msg)
)
class StringTemplate:
"""String that can be formatted."""
def __init__(self, template):
def __init__(self, template: str):
if not isinstance(template, str):
raise TypeError("<{}> argument must be a string, not {}.".format(
self.__class__.__name__, str(type(template))
))
raise TypeError(
f"<{self.__class__.__name__}> argument must be a string,"
f" not {str(type(template))}."
)
self._template = template
self._template: str = template
parts = []
last_end_idx = 0
for item in KEY_PATTERN.finditer(template):
start, end = item.span()
if start > last_end_idx:
parts.append(template[last_end_idx:start])
parts.append(FormattingPart(template[start:end]))
last_end_idx = end
formatter = Formatter()
if last_end_idx < len(template):
parts.append(template[last_end_idx:len(template)])
for item in formatter.parse(template):
literal_text, field_name, format_spec, conversion = item
if literal_text:
parts.append(literal_text)
if field_name:
parts.append(
FormattingPart(field_name, format_spec, conversion)
)
new_parts = []
for part in parts:
@ -77,15 +82,17 @@ class StringTemplate:
if substr:
new_parts.append(substr)
self._parts = self.find_optional_parts(new_parts)
self._parts: List["Union[str, OptionalPart, FormattingPart]"] = (
self.find_optional_parts(new_parts)
)
def __str__(self):
def __str__(self) -> str:
return self.template
def __repr__(self):
return "<{}> {}".format(self.__class__.__name__, self.template)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}> {self.template}"
def __contains__(self, other):
def __contains__(self, other: str) -> bool:
return other in self.template
def replace(self, *args, **kwargs):
@ -93,10 +100,10 @@ class StringTemplate:
return self
@property
def template(self):
def template(self) -> str:
return self._template
def format(self, data):
def format(self, data: Dict[str, Any]) -> "TemplateResult":
""" Figure out with whole formatting.
Separate advanced keys (*Like '{project[name]}') from string which must
@ -108,6 +115,7 @@ class StringTemplate:
Returns:
TemplateResult: Filled or partially filled template containing all
data needed or missing for filling template.
"""
result = TemplatePartResult()
for part in self._parts:
@ -135,23 +143,29 @@ class StringTemplate:
invalid_types
)
def format_strict(self, *args, **kwargs):
result = self.format(*args, **kwargs)
def format_strict(self, data: Dict[str, Any]) -> "TemplateResult":
result = self.format(data)
result.validate()
return result
@classmethod
def format_template(cls, template, data):
def format_template(
cls, template: str, data: Dict[str, Any]
) -> "TemplateResult":
objected_template = cls(template)
return objected_template.format(data)
@classmethod
def format_strict_template(cls, template, data):
def format_strict_template(
cls, template: str, data: Dict[str, Any]
) -> "TemplateResult":
objected_template = cls(template)
return objected_template.format_strict(data)
@staticmethod
def find_optional_parts(parts):
def find_optional_parts(
parts: List["Union[str, FormattingPart]"]
) -> List["Union[str, OptionalPart, FormattingPart]"]:
new_parts = []
tmp_parts = {}
counted_symb = -1
@ -216,11 +230,11 @@ class TemplateResult(str):
of number.
"""
used_values = None
solved = None
template = None
missing_keys = None
invalid_types = None
used_values: Dict[str, Any] = None
solved: bool = None
template: str = None
missing_keys: List[str] = None
invalid_types: Dict[str, Any] = None
def __new__(
cls, filled_template, template, solved,
@ -248,7 +262,7 @@ class TemplateResult(str):
self.invalid_types
)
def copy(self):
def copy(self) -> "TemplateResult":
cls = self.__class__
return cls(
str(self),
@ -259,7 +273,7 @@ class TemplateResult(str):
self.invalid_types
)
def normalized(self):
def normalized(self) -> "TemplateResult":
"""Convert to normalized path."""
cls = self.__class__
@ -275,27 +289,28 @@ class TemplateResult(str):
class TemplatePartResult:
"""Result to store result of template parts."""
def __init__(self, optional=False):
def __init__(self, optional: bool = False):
# Missing keys or invalid value types of required keys
self._missing_keys = set()
self._invalid_types = {}
self._missing_keys: Set[str] = set()
self._invalid_types: Dict[str, Any] = {}
# Missing keys or invalid value types of optional keys
self._missing_optional_keys = set()
self._invalid_optional_types = {}
self._missing_optional_keys: Set[str] = set()
self._invalid_optional_types: Dict[str, Any] = {}
# Used values stored by key with origin type
# - key without any padding or key modifiers
# - value from filling data
# Example: {"version": 1}
self._used_values = {}
self._used_values: Dict[str, Any] = {}
# Used values stored by key with all modifirs
# - value is already formatted string
# Example: {"version:0>3": "001"}
self._realy_used_values = {}
self._really_used_values: Dict[str, Any] = {}
# Concatenated string output after formatting
self._output = ""
self._output: str = ""
# Is this result from optional part
self._optional = True
# TODO find out why we don't use 'optional' from args
self._optional: bool = True
def add_output(self, other):
if isinstance(other, str):
@ -313,7 +328,7 @@ class TemplatePartResult:
if other.optional and not other.solved:
return
self._used_values.update(other.used_values)
self._realy_used_values.update(other.realy_used_values)
self._really_used_values.update(other.really_used_values)
else:
raise TypeError("Cannot add data from \"{}\" to \"{}\"".format(
@ -321,7 +336,7 @@ class TemplatePartResult:
)
@property
def solved(self):
def solved(self) -> bool:
if self.optional:
if (
len(self.missing_optional_keys) > 0
@ -334,45 +349,53 @@ class TemplatePartResult:
)
@property
def optional(self):
def optional(self) -> bool:
return self._optional
@property
def output(self):
def output(self) -> str:
return self._output
@property
def missing_keys(self):
def missing_keys(self) -> Set[str]:
return self._missing_keys
@property
def missing_optional_keys(self):
def missing_optional_keys(self) -> Set[str]:
return self._missing_optional_keys
@property
def invalid_types(self):
def invalid_types(self) -> Dict[str, Any]:
return self._invalid_types
@property
def invalid_optional_types(self):
def invalid_optional_types(self) -> Dict[str, Any]:
return self._invalid_optional_types
@property
def realy_used_values(self):
return self._realy_used_values
def really_used_values(self) -> Dict[str, Any]:
return self._really_used_values
@property
def used_values(self):
def realy_used_values(self) -> Dict[str, Any]:
warnings.warn(
"Property 'realy_used_values' is deprecated."
" Use 'really_used_values' instead.",
DeprecationWarning
)
return self._really_used_values
@property
def used_values(self) -> Dict[str, Any]:
return self._used_values
@staticmethod
def split_keys_to_subdicts(values):
def split_keys_to_subdicts(values: Dict[str, Any]) -> Dict[str, Any]:
output = {}
formatter = Formatter()
for key, value in values.items():
key_padding = list(KEY_PADDING_PATTERN.findall(key))
if key_padding:
key = key_padding[0]
key_subdict = list(SUB_DICT_PATTERN.findall(key))
_, field_name, _, _ = next(formatter.parse(f"{{{key}}}"))
key_subdict = list(SUB_DICT_PATTERN.findall(field_name))
data = output
last_key = key_subdict.pop(-1)
for subkey in key_subdict:
@ -382,7 +405,7 @@ class TemplatePartResult:
data[last_key] = value
return output
def get_clean_used_values(self):
def get_clean_used_values(self) -> Dict[str, Any]:
new_used_values = {}
for key, value in self.used_values.items():
if isinstance(value, FormatObject):
@ -391,19 +414,27 @@ class TemplatePartResult:
return self.split_keys_to_subdicts(new_used_values)
def add_realy_used_value(self, key, value):
self._realy_used_values[key] = value
def add_really_used_value(self, key: str, value: Any):
self._really_used_values[key] = value
def add_used_value(self, key, value):
def add_realy_used_value(self, key: str, value: Any):
warnings.warn(
"Method 'add_realy_used_value' is deprecated."
" Use 'add_really_used_value' instead.",
DeprecationWarning
)
self.add_really_used_value(key, value)
def add_used_value(self, key: str, value: Any):
self._used_values[key] = value
def add_missing_key(self, key):
def add_missing_key(self, key: str):
if self._optional:
self._missing_optional_keys.add(key)
else:
self._missing_keys.add(key)
def add_invalid_type(self, key, value):
def add_invalid_type(self, key: str, value: Any):
if self._optional:
self._invalid_optional_types[key] = type(value)
else:
@ -421,10 +452,10 @@ class FormatObject:
def __format__(self, *args, **kwargs):
return self.value.__format__(*args, **kwargs)
def __str__(self):
def __str__(self) -> str:
return str(self.value)
def __repr__(self):
def __repr__(self) -> str:
return self.__str__()
@ -434,23 +465,44 @@ class FormattingPart:
Containt only single key to format e.g. "{project[name]}".
Args:
template(str): String containing the formatting key.
field_name (str): Name of key.
format_spec (str): Format specification.
conversion (Union[str, None]): Conversion type.
"""
def __init__(self, template):
self._template = template
def __init__(
self,
field_name: str,
format_spec: str,
conversion: "Union[str, None]",
):
format_spec_v = ""
if format_spec:
format_spec_v = f":{format_spec}"
conversion_v = ""
if conversion:
conversion_v = f"!{conversion}"
self._field_name: str = field_name
self._format_spec: str = format_spec_v
self._conversion: str = conversion_v
template_base = f"{field_name}{format_spec_v}{conversion_v}"
self._template_base: str = template_base
self._template: str = f"{{{template_base}}}"
@property
def template(self):
def template(self) -> str:
return self._template
def __repr__(self):
def __repr__(self) -> str:
return "<Format:{}>".format(self._template)
def __str__(self):
def __str__(self) -> str:
return self._template
@staticmethod
def validate_value_type(value):
def validate_value_type(value: Any) -> bool:
"""Check if value can be used for formatting of single key."""
if isinstance(value, (numbers.Number, FormatObject)):
return True
@ -461,7 +513,7 @@ class FormattingPart:
return False
@staticmethod
def validate_key_is_matched(key):
def validate_key_is_matched(key: str) -> bool:
"""Validate that opening has closing at correct place.
Future-proof, only square brackets are currently used in keys.
@ -488,16 +540,29 @@ class FormattingPart:
return False
return not queue
def format(self, data, result):
@staticmethod
def keys_to_template_base(keys: List[str]):
if not keys:
return None
# Create copy of keys
keys = list(keys)
template_base = keys.pop(0)
joined_keys = "".join([f"[{key}]" for key in keys])
return f"{template_base}{joined_keys}"
def format(
self, data: Dict[str, Any], result: TemplatePartResult
) -> TemplatePartResult:
"""Format the formattings string.
Args:
data(dict): Data that should be used for formatting.
result(TemplatePartResult): Object where result is stored.
"""
key = self.template[1:-1]
if key in result.realy_used_values:
result.add_output(result.realy_used_values[key])
key = self._template_base
if key in result.really_used_values:
result.add_output(result.really_used_values[key])
return result
# ensure key is properly formed [({})] properly closed.
@ -507,17 +572,38 @@ class FormattingPart:
return result
# check if key expects subdictionary keys (e.g. project[name])
existence_check = key
key_padding = list(KEY_PADDING_PATTERN.findall(existence_check))
if key_padding:
existence_check = key_padding[0]
key_subdict = list(SUB_DICT_PATTERN.findall(existence_check))
key_subdict = list(SUB_DICT_PATTERN.findall(self._field_name))
value = data
missing_key = False
invalid_type = False
used_keys = []
keys_to_value = None
used_value = None
for sub_key in key_subdict:
if isinstance(value, list):
if not sub_key.lstrip("-").isdigit():
invalid_type = True
break
sub_key = int(sub_key)
if sub_key < 0:
sub_key = len(value) + sub_key
invalid = 0 > sub_key < len(data)
if invalid:
used_keys.append(sub_key)
missing_key = True
break
used_keys.append(sub_key)
if keys_to_value is None:
keys_to_value = list(used_keys)
keys_to_value.pop(-1)
used_value = copy.deepcopy(value)
value = value[sub_key]
continue
if (
value is None
or (hasattr(value, "items") and sub_key not in value)
@ -533,45 +619,57 @@ class FormattingPart:
used_keys.append(sub_key)
value = value.get(sub_key)
if missing_key or invalid_type:
if len(used_keys) == 0:
invalid_key = key_subdict[0]
else:
invalid_key = used_keys[0]
for idx, sub_key in enumerate(used_keys):
if idx == 0:
continue
invalid_key += "[{0}]".format(sub_key)
field_name = key_subdict[0]
if used_keys:
field_name = self.keys_to_template_base(used_keys)
if missing_key or invalid_type:
if missing_key:
result.add_missing_key(invalid_key)
result.add_missing_key(field_name)
elif invalid_type:
result.add_invalid_type(invalid_key, value)
result.add_invalid_type(field_name, value)
result.add_output(self.template)
return result
if self.validate_value_type(value):
fill_data = {}
first_value = True
for used_key in reversed(used_keys):
if first_value:
first_value = False
fill_data[used_key] = value
else:
_fill_data = {used_key: fill_data}
fill_data = _fill_data
formatted_value = self.template.format(**fill_data)
result.add_realy_used_value(key, formatted_value)
result.add_used_value(existence_check, formatted_value)
result.add_output(formatted_value)
if not self.validate_value_type(value):
result.add_invalid_type(key, value)
result.add_output(self.template)
return result
result.add_invalid_type(key, value)
result.add_output(self.template)
fill_data = root_fill_data = {}
parent_fill_data = None
parent_key = None
fill_value = data
value_filled = False
for used_key in used_keys:
if isinstance(fill_value, list):
parent_fill_data[parent_key] = fill_value
value_filled = True
break
fill_value = fill_value[used_key]
parent_fill_data = fill_data
fill_data = parent_fill_data.setdefault(used_key, {})
parent_key = used_key
if not value_filled:
parent_fill_data[used_keys[-1]] = value
template = f"{{{field_name}{self._format_spec}{self._conversion}}}"
formatted_value = template.format(**root_fill_data)
used_key = key
if keys_to_value is not None:
used_key = self.keys_to_template_base(keys_to_value)
if used_value is None:
if isinstance(value, numbers.Number):
used_value = value
else:
used_value = formatted_value
result.add_really_used_value(self._field_name, used_value)
result.add_used_value(used_key, used_value)
result.add_output(formatted_value)
return result
@ -585,20 +683,27 @@ class OptionalPart:
'FormattingPart'.
"""
def __init__(self, parts):
self._parts = parts
def __init__(
self,
parts: List["Union[str, OptionalPart, FormattingPart]"]
):
self._parts: List["Union[str, OptionalPart, FormattingPart]"] = parts
@property
def parts(self):
def parts(self) -> List["Union[str, OptionalPart, FormattingPart]"]:
return self._parts
def __str__(self):
def __str__(self) -> str:
return "<{}>".format("".join([str(p) for p in self._parts]))
def __repr__(self):
def __repr__(self) -> str:
return "<Optional:{}>".format("".join([str(p) for p in self._parts]))
def format(self, data, result):
def format(
self,
data: Dict[str, Any],
result: TemplatePartResult,
) -> TemplatePartResult:
new_result = TemplatePartResult(True)
for part in self._parts:
if isinstance(part, str):

10
client/ayon_core/pipeline/__init__.py
View file

@ -7,6 +7,10 @@ from .constants import (
from .anatomy import Anatomy
from .tempdir import get_temp_dir
from .staging_dir import get_staging_dir_info
from .create import (
BaseCreator,
Creator,
@ -117,6 +121,12 @@ __all__ = (
# --- Anatomy ---
"Anatomy",
# --- Temp dir ---
"get_temp_dir",
# --- Staging dir ---
"get_staging_dir_info",
# --- Create ---
"BaseCreator",
"Creator",

93
client/ayon_core/pipeline/create/creator_plugins.py
View file

@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
import os
import copy
import collections
from typing import TYPE_CHECKING, Optional, Dict, Any
@ -6,7 +7,7 @@ from typing import TYPE_CHECKING, Optional, Dict, Any
from abc import ABC, abstractmethod
from ayon_core.settings import get_project_settings
from ayon_core.lib import Logger
from ayon_core.lib import Logger, get_version_from_path
from ayon_core.pipeline.plugin_discover import (
discover,
register_plugin,
@ -14,6 +15,7 @@ from ayon_core.pipeline.plugin_discover import (
deregister_plugin,
deregister_plugin_path
)
from ayon_core.pipeline import get_staging_dir_info
from .constants import DEFAULT_VARIANT_VALUE
from .product_name import get_product_name
@ -831,6 +833,95 @@ class Creator(BaseCreator):
"""
return self.pre_create_attr_defs
def get_staging_dir(self, instance):
"""Return the staging dir and persistence from instance.
Args:
instance (CreatedInstance): Instance for which should be staging
dir gathered.
Returns:
Optional[namedtuple]: Staging dir path and persistence or None
"""
create_ctx = self.create_context
product_name = instance.get("productName")
product_type = instance.get("productType")
folder_path = instance.get("folderPath")
# this can only work if product name and folder path are available
if not product_name or not folder_path:
return None
publish_settings = self.project_settings["core"]["publish"]
follow_workfile_version = (
publish_settings
["CollectAnatomyInstanceData"]
["follow_workfile_version"]
)
# Gather version number provided from the instance.
version = instance.get("version")
# If follow workfile, gather version from workfile path.
if version is None and follow_workfile_version:
current_workfile = self.create_context.get_current_workfile_path()
workfile_version = get_version_from_path(current_workfile)
version = int(workfile_version)
# Fill-up version with next version available.
elif version is None:
versions = self.get_next_versions_for_instances(
[instance]
)
version, = tuple(versions.values())
template_data = {"version": version}
staging_dir_info = get_staging_dir_info(
create_ctx.get_current_project_entity(),
create_ctx.get_folder_entity(folder_path),
create_ctx.get_task_entity(folder_path, instance.get("task")),
product_type,
product_name,
create_ctx.host_name,
anatomy=create_ctx.get_current_project_anatomy(),
project_settings=create_ctx.get_current_project_settings(),
always_return_path=False,
logger=self.log,
template_data=template_data,
)
return staging_dir_info or None
def apply_staging_dir(self, instance):
"""Apply staging dir with persistence to instance's transient data.
Method is called on instance creation and on instance update.
Args:
instance (CreatedInstance): Instance for which should be staging
dir applied.
Returns:
Optional[str]: Staging dir path or None if not applied.
"""
staging_dir_info = self.get_staging_dir(instance)
if staging_dir_info is None:
return None
# path might be already created by get_staging_dir_info
staging_dir_path = staging_dir_info.directory
os.makedirs(staging_dir_path, exist_ok=True)
instance.transient_data.update({
"stagingDir": staging_dir_path,
"stagingDir_persistent": staging_dir_info.persistent,
})
self.log.info(f"Applied staging dir to instance: {staging_dir_path}")
return staging_dir_path
def _pre_create_attr_defs_changed(self):
"""Called when pre-create attribute definitions change.

65
client/ayon_core/pipeline/farm/pyblish_functions.py
View file

@ -7,7 +7,7 @@ from copy import deepcopy
import attr
import ayon_api
import clique
from ayon_core.lib import Logger, collect_frames
from ayon_core.lib import Logger
from ayon_core.pipeline import (
get_current_project_name,
get_representation_path,
@ -384,9 +384,9 @@ def prepare_representations(
frame_end = frames_to_render[-1]
if skeleton_data.get("slate"):
frame_start -= 1
frames_to_render.insert(0, frame_start)
files = _get_real_files_to_rendered(collection, frames_to_render)
files = _get_real_files_to_render(collection, frames_to_render)
# explicitly disable review by user
preview = preview and not do_not_add_review
rep = {
@ -435,13 +435,10 @@ def prepare_representations(
" This may cause issues on farm."
).format(staging))
files = _get_real_files_to_rendered(
[os.path.basename(remainder)], frames_to_render)
rep = {
"name": ext,
"ext": ext,
"files": files[0],
"files": os.path.basename(remainder),
"stagingDir": staging,
}
@ -495,34 +492,42 @@ def _get_real_frames_to_render(frames):
return frames_to_render
def _get_real_files_to_rendered(collection, frames_to_render):
"""Use expected files based on real frames_to_render.
def _get_real_files_to_render(collection, frames_to_render):
"""Filter files with frames that should be really rendered.
'expected_files' are collected from DCC based on timeline setting. This is
being calculated differently in each DCC. Filtering here is on single place
But artists might explicitly set frames they want to render in Publisher UI
This range would override and filter previously prepared expected files
from DCC.
Artists might explicitly set frames they want to render via Publisher UI.
This uses this value to filter out files
Args:
frames_to_render (list): of str '1001'
"""
files = [os.path.basename(f) for f in list(collection)]
file_name, extracted_frame = list(collect_frames(files).items())[0]
collection (clique.Collection): absolute paths
frames_to_render (list[int]): of int 1001
Returns:
(list[str])
if not extracted_frame:
return files
Example:
--------
found_frame_pattern_length = len(extracted_frame)
normalized_frames_to_render = {
str(frame_to_render).zfill(found_frame_pattern_length)
for frame_to_render in frames_to_render
}
return [
file_name
for file_name in files
if any(
frame in file_name
for frame in normalized_frames_to_render
)
expectedFiles = [
"foo_v01.0001.exr",
"foo_v01.0002.exr",
]
frames_to_render = 1
>>
["foo_v01.0001.exr"] - only explicitly requested frame returned
"""
included_frames = set(collection.indexes).intersection(frames_to_render)
real_collection = clique.Collection(
collection.head,
collection.tail,
collection.padding,
indexes=included_frames
)
real_full_paths = list(real_collection)
return [os.path.basename(file_url) for file_url in real_full_paths]
def create_instances_for_aov(instance, skeleton, aov_filter,

14
client/ayon_core/pipeline/load/utils.py
View file

@ -465,7 +465,9 @@ def update_container(container, version=-1):
from ayon_core.pipeline import get_current_project_name
# Compute the different version from 'representation'
project_name = get_current_project_name()
project_name = container.get("project_name")
if project_name is None:
project_name = get_current_project_name()
repre_id = container["representation"]
if not _is_valid_representation_id(repre_id):
raise ValueError(
@ -542,9 +544,6 @@ def update_container(container, version=-1):
)
)
path = get_representation_path(new_representation)
if not path or not os.path.exists(path):
raise ValueError("Path {} doesn't exist".format(path))
project_entity = ayon_api.get_project(project_name)
context = {
"project": project_entity,
@ -553,6 +552,9 @@ def update_container(container, version=-1):
"version": new_version,
"representation": new_representation,
}
path = get_representation_path_from_context(context)
if not path or not os.path.exists(path):
raise ValueError("Path {} doesn't exist".format(path))
return Loader().update(container, context)
@ -588,7 +590,9 @@ def switch_container(container, representation, loader_plugin=None):
)
# Get the new representation to switch to
project_name = get_current_project_name()
project_name = container.get("project_name")
if project_name is None:
project_name = get_current_project_name()
context = get_representation_context(
project_name, representation["id"]

1
client/ayon_core/pipeline/publish/constants.py
View file

@ -8,6 +8,5 @@ ValidateMeshOrder = pyblish.api.ValidatorOrder + 0.3
DEFAULT_PUBLISH_TEMPLATE = "default"
DEFAULT_HERO_PUBLISH_TEMPLATE = "default"
TRANSIENT_DIR_TEMPLATE = "default"
FARM_JOB_ENV_DATA_KEY: str = "farmJobEnv"

179
client/ayon_core/pipeline/publish/lib.py
View file

@ -2,7 +2,7 @@ import os
import sys
import inspect
import copy
import tempfile
import warnings
import xml.etree.ElementTree
from typing import Optional, Union, List
@ -18,15 +18,11 @@ from ayon_core.lib import (
)
from ayon_core.settings import get_project_settings
from ayon_core.addon import AddonsManager
from ayon_core.pipeline import (
tempdir,
Anatomy
)
from ayon_core.pipeline import get_staging_dir_info
from ayon_core.pipeline.plugin_discover import DiscoverResult
from .constants import (
DEFAULT_PUBLISH_TEMPLATE,
DEFAULT_HERO_PUBLISH_TEMPLATE,
TRANSIENT_DIR_TEMPLATE
)
@ -581,58 +577,6 @@ def context_plugin_should_run(plugin, context):
return False
def get_instance_staging_dir(instance):
"""Unified way how staging dir is stored and created on instances.
First check if 'stagingDir' is already set in instance data.
In case there already is new tempdir will not be created.
It also supports `AYON_TMPDIR`, so studio can define own temp
shared repository per project or even per more granular context.
Template formatting is supported also with optional keys. Folder is
created in case it doesn't exists.
Available anatomy formatting keys:
- root[work | <root name key>]
- project[name | code]
Note:
Staging dir does not have to be necessarily in tempdir so be careful
about its usage.
Args:
instance (pyblish.lib.Instance): Instance for which we want to get
staging dir.
Returns:
str: Path to staging dir of instance.
"""
staging_dir = instance.data.get('stagingDir')
if staging_dir:
return staging_dir
anatomy = instance.context.data.get("anatomy")
# get customized tempdir path from `AYON_TMPDIR` env var
custom_temp_dir = tempdir.create_custom_tempdir(
anatomy.project_name, anatomy)
if custom_temp_dir:
staging_dir = os.path.normpath(
tempfile.mkdtemp(
prefix="pyblish_tmp_",
dir=custom_temp_dir
)
)
else:
staging_dir = os.path.normpath(
tempfile.mkdtemp(prefix="pyblish_tmp_")
)
instance.data['stagingDir'] = staging_dir
return staging_dir
def get_publish_repre_path(instance, repre, only_published=False):
"""Get representation path that can be used for integration.
@ -685,6 +629,8 @@ def get_publish_repre_path(instance, repre, only_published=False):
return None
# deprecated: backward compatibility only (2024-09-12)
# TODO: remove in the future
def get_custom_staging_dir_info(
project_name,
host_name,
@ -694,67 +640,86 @@ def get_custom_staging_dir_info(
product_name,
project_settings=None,
anatomy=None,
log=None
log=None,
):
"""Checks profiles if context should use special custom dir as staging.
from ayon_core.pipeline.staging_dir import get_staging_dir_config
warnings.warn(
(
"Function 'get_custom_staging_dir_info' in"
" 'ayon_core.pipeline.publish' is deprecated. Please use"
" 'get_custom_staging_dir_info'"
" in 'ayon_core.pipeline.stagingdir'."
),
DeprecationWarning,
)
tr_data = get_staging_dir_config(
project_name,
task_type,
task_name,
product_type,
product_name,
host_name,
project_settings=project_settings,
anatomy=anatomy,
log=log,
)
Args:
project_name (str)
host_name (str)
product_type (str)
task_name (str)
task_type (str)
product_name (str)
project_settings(Dict[str, Any]): Prepared project settings.
anatomy (Dict[str, Any])
log (Logger) (optional)
if not tr_data:
return None, None
return tr_data["template"], tr_data["persistence"]
def get_instance_staging_dir(instance):
"""Unified way how staging dir is stored and created on instances.
First check if 'stagingDir' is already set in instance data.
In case there already is new tempdir will not be created.
Returns:
(tuple)
Raises:
ValueError - if misconfigured template should be used
str: Path to staging dir
"""
settings = project_settings or get_project_settings(project_name)
custom_staging_dir_profiles = (settings["core"]
["tools"]
["publish"]
["custom_staging_dir_profiles"])
if not custom_staging_dir_profiles:
return None, None
staging_dir = instance.data.get("stagingDir")
if not log:
log = Logger.get_logger("get_custom_staging_dir_info")
if staging_dir:
return staging_dir
filtering_criteria = {
"hosts": host_name,
"families": product_type,
"task_names": task_name,
"task_types": task_type,
"subsets": product_name
}
profile = filter_profiles(custom_staging_dir_profiles,
filtering_criteria,
logger=log)
anatomy_data = instance.data["anatomyData"]
template_data = copy.deepcopy(anatomy_data)
if not profile or not profile["active"]:
return None, None
# context data based variables
context = instance.context
if not anatomy:
anatomy = Anatomy(project_name)
# add current file as workfile name into formatting data
current_file = context.data.get("currentFile")
if current_file:
workfile = os.path.basename(current_file)
workfile_name, _ = os.path.splitext(workfile)
template_data["workfile_name"] = workfile_name
template_name = profile["template_name"] or TRANSIENT_DIR_TEMPLATE
custom_staging_dir = anatomy.get_template_item(
"staging", template_name, "directory", default=None
staging_dir_info = get_staging_dir_info(
context.data["projectEntity"],
instance.data.get("folderEntity"),
instance.data.get("taskEntity"),
instance.data["productType"],
instance.data["productName"],
context.data["hostName"],
anatomy=context.data["anatomy"],
project_settings=context.data["project_settings"],
template_data=template_data,
always_return_path=True,
)
if custom_staging_dir is None:
raise ValueError((
"Anatomy of project \"{}\" does not have set"
" \"{}\" template key!"
).format(project_name, template_name))
is_persistent = profile["custom_staging_dir_persistent"]
return custom_staging_dir.template, is_persistent
staging_dir_path = staging_dir_info.directory
# path might be already created by get_staging_dir_info
os.makedirs(staging_dir_path, exist_ok=True)
instance.data.update({
"stagingDir": staging_dir_path,
"stagingDir_persistent": staging_dir_info.persistent,
})
return staging_dir_path
def get_published_workfile_instance(context):

222
client/ayon_core/pipeline/staging_dir.py Normal file
View file

@ -0,0 +1,222 @@
from dataclasses import dataclass
from ayon_core.lib import Logger, filter_profiles
from ayon_core.settings import get_project_settings
from .template_data import get_template_data
from .anatomy import Anatomy
from .tempdir import get_temp_dir
@dataclass
class StagingDir:
directory: str
persistent: bool
def get_staging_dir_config(
project_name,
task_type,
task_name,
product_type,
product_name,
host_name,
project_settings=None,
anatomy=None,
log=None,
):
"""Get matching staging dir profile.
Args:
host_name (str): Name of host.
project_name (str): Name of project.
task_type (Optional[str]): Type of task.
task_name (Optional[str]): Name of task.
product_type (str): Type of product.
product_name (str): Name of product.
project_settings(Dict[str, Any]): Prepared project settings.
anatomy (Dict[str, Any])
log (Optional[logging.Logger])
Returns:
Dict or None: Data with directory template and is_persistent or None
Raises:
KeyError - if misconfigured template should be used
"""
settings = project_settings or get_project_settings(project_name)
staging_dir_profiles = settings["core"]["tools"]["publish"][
"custom_staging_dir_profiles"
]
if not staging_dir_profiles:
return None
if not log:
log = Logger.get_logger("get_staging_dir_config")
filtering_criteria = {
"hosts": host_name,
"task_types": task_type,
"task_names": task_name,
"product_types": product_type,
"product_names": product_name,
}
profile = filter_profiles(
staging_dir_profiles, filtering_criteria, logger=log)
if not profile or not profile["active"]:
return None
if not anatomy:
anatomy = Anatomy(project_name)
# get template from template name
template_name = profile["template_name"]
_validate_template_name(project_name, template_name, anatomy)
template = anatomy.get_template_item("staging", template_name)
if not template:
# template should always be found either from anatomy or from profile
raise KeyError(
f"Staging template '{template_name}' was not found."
"Check project anatomy or settings at: "
"'ayon+settings://core/tools/publish/custom_staging_dir_profiles'"
)
data_persistence = profile["custom_staging_dir_persistent"]
return {"template": template, "persistence": data_persistence}
def _validate_template_name(project_name, template_name, anatomy):
"""Check that staging dir section with appropriate template exist.
Raises:
ValueError - if misconfigured template
"""
if template_name not in anatomy.templates["staging"]:
raise ValueError(
f'Anatomy of project "{project_name}" does not have set'
f' "{template_name}" template key at Staging Dir category!'
)
def get_staging_dir_info(
project_entity,
folder_entity,
task_entity,
product_type,
product_name,
host_name,
anatomy=None,
project_settings=None,
template_data=None,
always_return_path=True,
force_tmp_dir=False,
logger=None,
prefix=None,
suffix=None,
):
"""Get staging dir info data.
If `force_temp` is set, staging dir will be created as tempdir.
If `always_get_some_dir` is set, staging dir will be created as tempdir if
no staging dir profile is found.
If `prefix` or `suffix` is not set, default values will be used.
Arguments:
project_entity (Dict[str, Any]): Project entity.
folder_entity (Optional[Dict[str, Any]]): Folder entity.
task_entity (Optional[Dict[str, Any]]): Task entity.
product_type (str): Type of product.
product_name (str): Name of product.
host_name (str): Name of host.
anatomy (Optional[Anatomy]): Anatomy object.
project_settings (Optional[Dict[str, Any]]): Prepared project settings.
template_data (Optional[Dict[str, Any]]): Additional data for
formatting staging dir template.
always_return_path (Optional[bool]): If True, staging dir will be
created as tempdir if no staging dir profile is found. Input value
False will return None if no staging dir profile is found.
force_tmp_dir (Optional[bool]): If True, staging dir will be created as
tempdir.
logger (Optional[logging.Logger]): Logger instance.
prefix (Optional[str]) Optional prefix for staging dir name.
suffix (Optional[str]): Optional suffix for staging dir name.
Returns:
Optional[StagingDir]: Staging dir info data
"""
log = logger or Logger.get_logger("get_staging_dir_info")
if anatomy is None:
anatomy = Anatomy(
project_entity["name"], project_entity=project_entity
)
if force_tmp_dir:
return get_temp_dir(
project_name=project_entity["name"],
anatomy=anatomy,
prefix=prefix,
suffix=suffix,
)
# making few queries to database
ctx_data = get_template_data(
project_entity, folder_entity, task_entity, host_name
)
# add additional data
ctx_data["product"] = {
"type": product_type,
"name": product_name
}
# add additional template formatting data
if template_data:
ctx_data.update(template_data)
task_name = task_type = None
if task_entity:
task_name = task_entity["name"]
task_type = task_entity["taskType"]
# get staging dir config
staging_dir_config = get_staging_dir_config(
project_entity["name"],
task_type,
task_name ,
product_type,
product_name,
host_name,
project_settings=project_settings,
anatomy=anatomy,
log=log,
)
if staging_dir_config:
dir_template = staging_dir_config["template"]["directory"]
return StagingDir(
dir_template.format_strict(ctx_data),
staging_dir_config["persistence"],
)
# no config found but force an output
if always_return_path:
return StagingDir(
get_temp_dir(
project_name=project_entity["name"],
anatomy=anatomy,
prefix=prefix,
suffix=suffix,
),
False,
)
return None

92
client/ayon_core/pipeline/tempdir.py
View file

@ -3,11 +3,74 @@ Temporary folder operations
"""
import os
import tempfile
from pathlib import Path
from ayon_core.lib import StringTemplate
from ayon_core.pipeline import Anatomy
def create_custom_tempdir(project_name, anatomy=None):
def get_temp_dir(
project_name, anatomy=None, prefix=None, suffix=None, use_local_temp=False
):
"""Get temporary dir path.
If `use_local_temp` is set, tempdir will be created in local tempdir.
If `anatomy` is not set, default anatomy will be used.
If `prefix` or `suffix` is not set, default values will be used.
It also supports `AYON_TMPDIR`, so studio can define own temp
shared repository per project or even per more granular context.
Template formatting is supported also with optional keys. Folder is
created in case it doesn't exists.
Args:
project_name (str): Name of project.
anatomy (Optional[Anatomy]): Project Anatomy object.
suffix (Optional[str]): Suffix for tempdir.
prefix (Optional[str]): Prefix for tempdir.
use_local_temp (Optional[bool]): If True, temp dir will be created in
local tempdir.
Returns:
str: Path to staging dir of instance.
"""
if prefix is None:
prefix = "ay_tmp_"
suffix = suffix or ""
if use_local_temp:
return _create_local_staging_dir(prefix, suffix)
# make sure anatomy is set
if not anatomy:
anatomy = Anatomy(project_name)
# get customized tempdir path from `OPENPYPE_TMPDIR` env var
custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy)
return _create_local_staging_dir(prefix, suffix, dirpath=custom_temp_dir)
def _create_local_staging_dir(prefix, suffix, dirpath=None):
"""Create local staging dir
Args:
prefix (str): prefix for tempdir
suffix (str): suffix for tempdir
dirpath (Optional[str]): path to tempdir
Returns:
str: path to tempdir
"""
# use pathlib for creating tempdir
return tempfile.mkdtemp(
prefix=prefix, suffix=suffix, dir=dirpath
)
def _create_custom_tempdir(project_name, anatomy):
""" Create custom tempdir
Template path formatting is supporting:
@ -18,42 +81,35 @@ def create_custom_tempdir(project_name, anatomy=None):
Args:
project_name (str): project name
anatomy (ayon_core.pipeline.Anatomy)[optional]: Anatomy object
anatomy (ayon_core.pipeline.Anatomy): Anatomy object
Returns:
str | None: formatted path or None
"""
env_tmpdir = os.getenv("AYON_TMPDIR")
if not env_tmpdir:
return
return None
custom_tempdir = None
if "{" in env_tmpdir:
if anatomy is None:
anatomy = Anatomy(project_name)
# create base formate data
data = {
template_data = {
"root": anatomy.roots,
"project": {
"name": anatomy.project_name,
"code": anatomy.project_code,
}
},
}
# path is anatomy template
custom_tempdir = StringTemplate.format_template(
env_tmpdir, data).normalized()
env_tmpdir, template_data)
custom_tempdir_path = Path(custom_tempdir)
else:
# path is absolute
custom_tempdir = env_tmpdir
custom_tempdir_path = Path(env_tmpdir)
# create the dir path if it doesn't exists
if not os.path.exists(custom_tempdir):
try:
# create it if it doesn't exists
os.makedirs(custom_tempdir)
except IOError as error:
raise IOError(
"Path couldn't be created: {}".format(error))
custom_tempdir_path.mkdir(parents=True, exist_ok=True)
return custom_tempdir
return custom_tempdir_path.as_posix()

14
client/ayon_core/pipeline/template_data.py
View file

@ -87,14 +87,13 @@ def get_folder_template_data(folder_entity, project_name):
"""
path = folder_entity["path"]
hierarchy_parts = path.split("/")
# Remove empty string from the beginning
hierarchy_parts.pop(0)
# Remove empty string from the beginning and split by '/'
parents = path.lstrip("/").split("/")
# Remove last part which is folder name
folder_name = hierarchy_parts.pop(-1)
hierarchy = "/".join(hierarchy_parts)
if hierarchy_parts:
parent_name = hierarchy_parts[-1]
folder_name = parents.pop(-1)
hierarchy = "/".join(parents)
if parents:
parent_name = parents[-1]
else:
parent_name = project_name
@ -103,6 +102,7 @@ def get_folder_template_data(folder_entity, project_name):
"name": folder_name,
"type": folder_entity["folderType"],
"path": path,
"parents": parents,
},
"asset": folder_name,
"hierarchy": hierarchy,

1
client/ayon_core/plugins/publish/collect_anatomy_instance_data.py
View file

@ -434,6 +434,7 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin):
# Using 'Shot' is current default behavior of editorial
# (or 'newHierarchyIntegration') publishing.
"type": "Shot",
"parents": parents,
},
})

76
client/ayon_core/plugins/publish/collect_custom_staging_dir.py
View file

@ -1,76 +0,0 @@
"""
Requires:
anatomy
Provides:
instance.data -> stagingDir (folder path)
-> stagingDir_persistent (bool)
"""
import copy
import os.path
import pyblish.api
from ayon_core.pipeline.publish.lib import get_custom_staging_dir_info
class CollectCustomStagingDir(pyblish.api.InstancePlugin):
"""Looks through profiles if stagingDir should be persistent and in special
location.
Transient staging dir could be useful in specific use cases where is
desirable to have temporary renders in specific, persistent folders, could
be on disks optimized for speed for example.
It is studio responsibility to clean up obsolete folders with data.
Location of the folder is configured in `project_anatomy/templates/others`.
('transient' key is expected, with 'folder' key)
Which family/task type/product is applicable is configured in:
`project_settings/global/tools/publish/custom_staging_dir_profiles`
"""
label = "Collect Custom Staging Directory"
order = pyblish.api.CollectorOrder + 0.4990
template_key = "transient"
def process(self, instance):
product_type = instance.data["productType"]
product_name = instance.data["productName"]
host_name = instance.context.data["hostName"]
project_name = instance.context.data["projectName"]
project_settings = instance.context.data["project_settings"]
anatomy = instance.context.data["anatomy"]
task = instance.data["anatomyData"].get("task", {})
transient_tml, is_persistent = get_custom_staging_dir_info(
project_name,
host_name,
product_type,
product_name,
task.get("name"),
task.get("type"),
project_settings=project_settings,
anatomy=anatomy,
log=self.log)
if transient_tml:
anatomy_data = copy.deepcopy(instance.data["anatomyData"])
anatomy_data["root"] = anatomy.roots
scene_name = instance.context.data.get("currentFile")
if scene_name:
anatomy_data["scene_name"] = os.path.basename(scene_name)
transient_dir = transient_tml.format(**anatomy_data)
instance.data["stagingDir"] = transient_dir
instance.data["stagingDir_persistent"] = is_persistent
result_str = "Adding '{}' as".format(transient_dir)
else:
result_str = "Not adding"
self.log.debug("{} custom staging dir for instance with '{}'".format(
result_str, product_type
))

47
client/ayon_core/plugins/publish/collect_managed_staging_dir.py Normal file
View file

@ -0,0 +1,47 @@
"""
Requires:
anatomy
Provides:
instance.data -> stagingDir (folder path)
-> stagingDir_persistent (bool)
"""
import pyblish.api
from ayon_core.pipeline.publish import get_instance_staging_dir
class CollectManagedStagingDir(pyblish.api.InstancePlugin):
"""Apply matching Staging Dir profile to a instance.
Apply Staging dir via profiles could be useful in specific use cases
where is desirable to have temporary renders in specific,
persistent folders, could be on disks optimized for speed for example.
It is studio's responsibility to clean up obsolete folders with data.
Location of the folder is configured in:
`ayon+anatomy://_/templates/staging`.
Which family/task type/subset is applicable is configured in:
`ayon+settings://core/tools/publish/custom_staging_dir_profiles`
"""
label = "Collect Managed Staging Directory"
order = pyblish.api.CollectorOrder + 0.4990
def process(self, instance):
""" Collect the staging data and stores it to the instance.
Args:
instance (object): The instance to inspect.
"""
staging_dir_path = get_instance_staging_dir(instance)
persistance = instance.data.get("stagingDir_persistent", False)
self.log.info((
f"Instance staging dir was set to `{staging_dir_path}` "
f"and persistence is set to `{persistance}`"
))

26
client/ayon_core/plugins/publish/collect_otio_subset_resources.py
View file

@ -149,6 +149,7 @@ class CollectOtioSubsetResources(
self.log.info(
"frame_start-frame_end: {}-{}".format(frame_start, frame_end))
review_repre = None
if is_sequence:
# file sequence way
@ -177,6 +178,11 @@ class CollectOtioSubsetResources(
repre = self._create_representation(
frame_start, frame_end, collection=collection)
if "review" in instance.data["families"]:
review_repre = self._create_representation(
frame_start, frame_end, collection=collection,
delete=True, review=True)
else:
_trim = False
dirname, filename = os.path.split(media_ref.target_url)
@ -191,17 +197,26 @@ class CollectOtioSubsetResources(
repre = self._create_representation(
frame_start, frame_end, file=filename, trim=_trim)
if "review" in instance.data["families"]:
review_repre = self._create_representation(
frame_start, frame_end,
file=filename, delete=True, review=True)
instance.data["originalDirname"] = self.staging_dir
# add representation to instance data
if repre:
colorspace = instance.data.get("colorspace")
# add colorspace data to representation
self.set_representation_colorspace(
repre, instance.context, colorspace)
# add representation to instance data
instance.data["representations"].append(repre)
# add review representation to instance data
if review_repre:
instance.data["representations"].append(review_repre)
self.log.debug(instance.data)
def _create_representation(self, start, end, **kwargs):
@ -221,7 +236,8 @@ class CollectOtioSubsetResources(
representation_data = {
"frameStart": start,
"frameEnd": end,
"stagingDir": self.staging_dir
"stagingDir": self.staging_dir,
"tags": [],
}
if kwargs.get("collection"):
@ -247,8 +263,10 @@ class CollectOtioSubsetResources(
"frameEnd": end,
})
if kwargs.get("trim") is True:
representation_data["tags"] = ["trim"]
for tag_name in ("trim", "delete", "review"):
if kwargs.get(tag_name) is True:
representation_data["tags"].append(tag_name)
return representation_data
def get_template_name(self, instance):

11
client/ayon_core/plugins/publish/extract_burnin.py
View file

@ -9,11 +9,13 @@ import clique
import pyblish.api
from ayon_core import resources, AYON_CORE_ROOT
from ayon_core.pipeline import publish
from ayon_core.pipeline import (
publish,
get_temp_dir
)
from ayon_core.lib import (
run_ayon_launcher_process,
get_transcode_temp_directory,
convert_input_paths_for_ffmpeg,
should_convert_for_ffmpeg
)
@ -250,7 +252,10 @@ class ExtractBurnin(publish.Extractor):
# - change staging dir of source representation
# - must be set back after output definitions processing
if do_convert:
new_staging_dir = get_transcode_temp_directory()
new_staging_dir = get_temp_dir(
project_name=instance.context.data["projectName"],
use_local_temp=True,
)
repre["stagingDir"] = new_staging_dir
convert_input_paths_for_ffmpeg(

14
client/ayon_core/plugins/publish/extract_color_transcode.py
View file

@ -3,14 +3,15 @@ import copy
import clique
import pyblish.api
from ayon_core.pipeline import publish
from ayon_core.pipeline import (
publish,
get_temp_dir
)
from ayon_core.lib import (
is_oiio_supported,
)
from ayon_core.lib.transcoding import (
convert_colorspace,
get_transcode_temp_directory,
)
from ayon_core.lib.profiles_filtering import filter_profiles
@ -103,7 +104,10 @@ class ExtractOIIOTranscode(publish.Extractor):
new_repre = copy.deepcopy(repre)
original_staging_dir = new_repre["stagingDir"]
new_staging_dir = get_transcode_temp_directory()
new_staging_dir = get_temp_dir(
project_name=instance.context.data["projectName"],
use_local_temp=True,
)
new_repre["stagingDir"] = new_staging_dir
if isinstance(new_repre["files"], list):
@ -265,7 +269,7 @@ class ExtractOIIOTranscode(publish.Extractor):
(list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr]
"""
pattern = [clique.PATTERNS["frames"]]
collections, remainder = clique.assemble(
collections, _ = clique.assemble(
files_to_convert, patterns=pattern,
assume_padded_when_ambiguous=True)

1
client/ayon_core/plugins/publish/extract_otio_review.py
View file

@ -78,6 +78,7 @@ class ExtractOTIOReview(
if otio_review_clips is None:
self.log.info(f"Instance `{instance}` has no otioReviewClips")
return
# add plugin wide attributes
self.representation_files = []

7
client/ayon_core/plugins/publish/extract_review.py
View file

@ -22,8 +22,8 @@ from ayon_core.lib.transcoding import (
should_convert_for_ffmpeg,
get_review_layer_name,
convert_input_paths_for_ffmpeg,
get_transcode_temp_directory,
)
from ayon_core.pipeline import get_temp_dir
from ayon_core.pipeline.publish import (
KnownPublishError,
get_publish_instance_label,
@ -310,7 +310,10 @@ class ExtractReview(pyblish.api.InstancePlugin):
# - change staging dir of source representation
# - must be set back after output definitions processing
if do_convert:
new_staging_dir = get_transcode_temp_directory()
new_staging_dir = get_temp_dir(
project_name=instance.context.data["projectName"],
use_local_temp=True,
)
repre["stagingDir"] = new_staging_dir
convert_input_paths_for_ffmpeg(

16
client/ayon_core/tools/loader/control.py
View file

@ -372,17 +372,17 @@ class LoaderController(BackendLoaderController, FrontendLoaderController):
repre_ids = set()
for container in containers:
repre_id = container.get("representation")
# Ignore invalid representation ids.
# - invalid representation ids may be available if e.g. is
# opened scene from OpenPype whe 'ObjectId' was used instead
# of 'uuid'.
# NOTE: Server call would crash if there is any invalid id.
# That would cause crash we won't get any information.
try:
repre_id = container.get("representation")
# Ignore invalid representation ids.
# - invalid representation ids may be available if e.g. is
# opened scene from OpenPype whe 'ObjectId' was used
# instead of 'uuid'.
# NOTE: Server call would crash if there is any invalid id.
# That would cause crash we won't get any information.
uuid.UUID(repre_id)
repre_ids.add(repre_id)
except ValueError:
except (ValueError, TypeError, AttributeError):
pass
product_ids = self._products_model.get_product_ids_by_repre_ids(

32
client/ayon_core/tools/sceneinventory/control.py
View file

@ -86,8 +86,9 @@ class SceneInventoryController:
self._current_folder_set = True
return self._current_folder_id
def get_project_status_items(self):
project_name = self.get_current_project_name()
def get_project_status_items(self, project_name=None):
if project_name is None:
project_name = self.get_current_project_name()
return self._projects_model.get_project_status_items(
project_name, None
)
@ -105,32 +106,39 @@ class SceneInventoryController:
def get_container_items_by_id(self, item_ids):
return self._containers_model.get_container_items_by_id(item_ids)
def get_representation_info_items(self, representation_ids):
def get_representation_info_items(self, project_name, representation_ids):
return self._containers_model.get_representation_info_items(
representation_ids
project_name, representation_ids
)
def get_version_items(self, product_ids):
return self._containers_model.get_version_items(product_ids)
def get_version_items(self, project_name, product_ids):
return self._containers_model.get_version_items(
project_name, product_ids)
# Site Sync methods
def is_sitesync_enabled(self):
return self._sitesync_model.is_sitesync_enabled()
def get_sites_information(self):
return self._sitesync_model.get_sites_information()
def get_sites_information(self, project_name):
return self._sitesync_model.get_sites_information(project_name)
def get_site_provider_icons(self):
return self._sitesync_model.get_site_provider_icons()
def get_representations_site_progress(self, representation_ids):
def get_representations_site_progress(
self, project_name, representation_ids
):
return self._sitesync_model.get_representations_site_progress(
representation_ids
project_name, representation_ids
)
def resync_representations(self, representation_ids, site_type):
def resync_representations(
self, project_name, representation_ids, site_type
):
return self._sitesync_model.resync_representations(
representation_ids, site_type
project_name,
representation_ids,
site_type
)
# Switch dialog methods

319
client/ayon_core/tools/sceneinventory/model.py
View file

@ -36,6 +36,7 @@ REMOTE_SITE_ICON_ROLE = QtCore.Qt.UserRole + 23
# This value hold unique value of container that should be used to identify
# containers inbetween refresh.
ITEM_UNIQUE_NAME_ROLE = QtCore.Qt.UserRole + 24
PROJECT_NAME_ROLE = QtCore.Qt.UserRole + 25
class InventoryModel(QtGui.QStandardItemModel):
@ -52,6 +53,7 @@ class InventoryModel(QtGui.QStandardItemModel):
"Object name",
"Active site",
"Remote site",
"Project",
]
name_col = column_labels.index("Name")
version_col = column_labels.index("Version")
@ -63,6 +65,7 @@ class InventoryModel(QtGui.QStandardItemModel):
object_name_col = column_labels.index("Object name")
active_site_col = column_labels.index("Active site")
remote_site_col = column_labels.index("Remote site")
project_col = column_labels.index("Project")
display_role_by_column = {
name_col: QtCore.Qt.DisplayRole,
version_col: VERSION_LABEL_ROLE,
@ -72,6 +75,7 @@ class InventoryModel(QtGui.QStandardItemModel):
product_group_col: PRODUCT_GROUP_NAME_ROLE,
loader_col: LOADER_NAME_ROLE,
object_name_col: OBJECT_NAME_ROLE,
project_col: PROJECT_NAME_ROLE,
active_site_col: ACTIVE_SITE_PROGRESS_ROLE,
remote_site_col: REMOTE_SITE_PROGRESS_ROLE,
}
@ -85,7 +89,7 @@ class InventoryModel(QtGui.QStandardItemModel):
foreground_role_by_column = {
name_col: NAME_COLOR_ROLE,
version_col: VERSION_COLOR_ROLE,
status_col: STATUS_COLOR_ROLE
status_col: STATUS_COLOR_ROLE,
}
width_by_column = {
name_col: 250,
@ -95,6 +99,7 @@ class InventoryModel(QtGui.QStandardItemModel):
product_type_col: 150,
product_group_col: 120,
loader_col: 150,
project_col: 150,
}
OUTDATED_COLOR = QtGui.QColor(235, 30, 30)
@ -116,8 +121,8 @@ class InventoryModel(QtGui.QStandardItemModel):
self._default_icon_color = get_default_entity_icon_color()
self._last_project_statuses = {}
self._last_status_icons_by_name = {}
self._last_project_statuses = collections.defaultdict(dict)
self._last_status_icons_by_name = collections.defaultdict(dict)
def outdated(self, item):
return item.get("isOutdated", True)
@ -129,45 +134,73 @@ class InventoryModel(QtGui.QStandardItemModel):
self._clear_items()
items_by_repre_id = {}
project_names = set()
repre_ids_by_project = collections.defaultdict(set)
version_items_by_project = collections.defaultdict(dict)
repre_info_by_id_by_project = collections.defaultdict(dict)
item_by_repre_id_by_project = collections.defaultdict(
lambda: collections.defaultdict(list))
for container_item in container_items:
# if (
# selected is not None
# and container_item.item_id not in selected
# ):
# continue
repre_id = container_item.representation_id
items = items_by_repre_id.setdefault(repre_id, [])
items.append(container_item)
project_name = container_item.project_name
representation_id = container_item.representation_id
project_names.add(project_name)
repre_ids_by_project[project_name].add(representation_id)
(
item_by_repre_id_by_project
[project_name]
[representation_id]
).append(container_item)
for project_name, representation_ids in repre_ids_by_project.items():
repre_info = self._controller.get_representation_info_items(
project_name, representation_ids
)
repre_info_by_id_by_project[project_name] = repre_info
product_ids = {
repre_info.product_id
for repre_info in repre_info.values()
if repre_info.is_valid
}
version_items = self._controller.get_version_items(
project_name, product_ids
)
version_items_by_project[project_name] = version_items
repre_id = set(items_by_repre_id.keys())
repre_info_by_id = self._controller.get_representation_info_items(
repre_id
)
product_ids = {
repre_info.product_id
for repre_info in repre_info_by_id.values()
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
# SiteSync addon information
progress_by_id = self._controller.get_representations_site_progress(
repre_id
)
sites_info = self._controller.get_sites_information()
progress_by_project = {
project_name: self._controller.get_representations_site_progress(
project_name, repre_ids
)
for project_name, repre_ids in repre_ids_by_project.items()
}
sites_info_by_project_name = {
project_name: self._controller.get_sites_information(project_name)
for project_name in project_names
}
site_icons = {
provider: get_qt_icon(icon_def)
for provider, icon_def in (
self._controller.get_site_provider_icons().items()
)
}
self._last_project_statuses = {
status_item.name: status_item
for status_item in self._controller.get_project_status_items()
}
self._last_status_icons_by_name = {}
last_project_statuses = collections.defaultdict(dict)
for project_name in project_names:
status_items_by_name = {
status_item.name: status_item
for status_item in self._controller.get_project_status_items(
project_name
)
}
last_project_statuses[project_name] = status_items_by_name
self._last_project_statuses = last_project_statuses
self._last_status_icons_by_name = collections.defaultdict(dict)
group_item_icon = qtawesome.icon(
"fa.folder", color=self._default_icon_color
@ -187,118 +220,130 @@ class InventoryModel(QtGui.QStandardItemModel):
group_item_font = QtGui.QFont()
group_item_font.setBold(True)
active_site_icon = site_icons.get(sites_info["active_site_provider"])
remote_site_icon = site_icons.get(sites_info["remote_site_provider"])
root_item = self.invisibleRootItem()
group_items = []
for repre_id, container_items in items_by_repre_id.items():
repre_info = repre_info_by_id[repre_id]
version_color = None
if not repre_info.is_valid:
version_label = "N/A"
group_name = "< Entity N/A >"
item_icon = invalid_item_icon
is_latest = False
is_hero = False
status_name = None
for project_name, items_by_repre_id in (
item_by_repre_id_by_project.items()
):
sites_info = sites_info_by_project_name[project_name]
active_site_icon = site_icons.get(
sites_info["active_site_provider"]
)
remote_site_icon = site_icons.get(
sites_info["remote_site_provider"]
)
else:
group_name = "{}_{}: ({})".format(
repre_info.folder_path.rsplit("/")[-1],
repre_info.product_name,
repre_info.representation_name
progress_by_id = progress_by_project[project_name]
repre_info_by_id = repre_info_by_id_by_project[project_name]
version_items_by_product_id = (
version_items_by_project[project_name]
)
for repre_id, container_items in items_by_repre_id.items():
repre_info = repre_info_by_id[repre_id]
version_color = None
if not repre_info.is_valid:
version_label = "N/A"
group_name = "< Entity N/A >"
item_icon = invalid_item_icon
is_latest = False
is_hero = False
status_name = None
else:
group_name = "{}_{}: ({})".format(
repre_info.folder_path.rsplit("/")[-1],
repre_info.product_name,
repre_info.representation_name
)
item_icon = valid_item_icon
version_items = (
version_items_by_product_id[repre_info.product_id]
)
version_item = version_items[repre_info.version_id]
version_label = format_version(version_item.version)
is_hero = version_item.version < 0
is_latest = version_item.is_latest
if not version_item.is_latest:
version_color = self.OUTDATED_COLOR
status_name = version_item.status
(
status_color, status_short, status_icon
) = self._get_status_data(project_name, status_name)
repre_name = (
repre_info.representation_name or
"<unknown representation>"
)
item_icon = valid_item_icon
container_model_items = []
for container_item in container_items:
object_name = container_item.object_name or "<none>"
unique_name = repre_name + object_name
item = QtGui.QStandardItem()
item.setColumnCount(root_item.columnCount())
item.setData(container_item.namespace,
QtCore.Qt.DisplayRole)
item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE)
item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE)
item.setData(item_icon, QtCore.Qt.DecorationRole)
item.setData(repre_info.product_id, PRODUCT_ID_ROLE)
item.setData(container_item.item_id, ITEM_ID_ROLE)
item.setData(version_label, VERSION_LABEL_ROLE)
item.setData(container_item.loader_name, LOADER_NAME_ROLE)
item.setData(container_item.object_name, OBJECT_NAME_ROLE)
item.setData(True, IS_CONTAINER_ITEM_ROLE)
item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE)
container_model_items.append(item)
version_items = (
version_items_by_product_id[repre_info.product_id]
progress = progress_by_id[repre_id]
active_site_progress = "{}%".format(
max(progress["active_site"], 0) * 100
)
remote_site_progress = "{}%".format(
max(progress["remote_site"], 0) * 100
)
version_item = version_items[repre_info.version_id]
version_label = format_version(version_item.version)
is_hero = version_item.version < 0
is_latest = version_item.is_latest
if not version_item.is_latest:
version_color = self.OUTDATED_COLOR
status_name = version_item.status
status_color, status_short, status_icon = self._get_status_data(
status_name
)
group_item = QtGui.QStandardItem()
group_item.setColumnCount(root_item.columnCount())
group_item.setData(group_name, QtCore.Qt.DisplayRole)
group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE)
group_item.setData(group_item_icon, QtCore.Qt.DecorationRole)
group_item.setData(group_item_font, QtCore.Qt.FontRole)
group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE)
group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE)
group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE)
group_item.setData(is_latest, VERSION_IS_LATEST_ROLE)
group_item.setData(is_hero, VERSION_IS_HERO_ROLE)
group_item.setData(version_label, VERSION_LABEL_ROLE)
group_item.setData(len(container_items), COUNT_ROLE)
group_item.setData(status_name, STATUS_NAME_ROLE)
group_item.setData(status_short, STATUS_SHORT_ROLE)
group_item.setData(status_color, STATUS_COLOR_ROLE)
group_item.setData(status_icon, STATUS_ICON_ROLE)
group_item.setData(project_name, PROJECT_NAME_ROLE)
repre_name = (
repre_info.representation_name or "<unknown representation>"
)
container_model_items = []
for container_item in container_items:
object_name = container_item.object_name or "<none>"
unique_name = repre_name + object_name
item = QtGui.QStandardItem()
item.setColumnCount(root_item.columnCount())
item.setData(container_item.namespace, QtCore.Qt.DisplayRole)
item.setData(self.GRAYOUT_COLOR, NAME_COLOR_ROLE)
item.setData(self.GRAYOUT_COLOR, VERSION_COLOR_ROLE)
item.setData(item_icon, QtCore.Qt.DecorationRole)
item.setData(repre_info.product_id, PRODUCT_ID_ROLE)
item.setData(container_item.item_id, ITEM_ID_ROLE)
item.setData(version_label, VERSION_LABEL_ROLE)
item.setData(container_item.loader_name, LOADER_NAME_ROLE)
item.setData(container_item.object_name, OBJECT_NAME_ROLE)
item.setData(True, IS_CONTAINER_ITEM_ROLE)
item.setData(unique_name, ITEM_UNIQUE_NAME_ROLE)
container_model_items.append(item)
if not container_model_items:
continue
progress = progress_by_id[repre_id]
active_site_progress = "{}%".format(
max(progress["active_site"], 0) * 100
)
remote_site_progress = "{}%".format(
max(progress["remote_site"], 0) * 100
)
group_item = QtGui.QStandardItem()
group_item.setColumnCount(root_item.columnCount())
group_item.setData(group_name, QtCore.Qt.DisplayRole)
group_item.setData(group_name, ITEM_UNIQUE_NAME_ROLE)
group_item.setData(group_item_icon, QtCore.Qt.DecorationRole)
group_item.setData(group_item_font, QtCore.Qt.FontRole)
group_item.setData(repre_info.product_id, PRODUCT_ID_ROLE)
group_item.setData(repre_info.product_type, PRODUCT_TYPE_ROLE)
group_item.setData(product_type_icon, PRODUCT_TYPE_ICON_ROLE)
group_item.setData(is_latest, VERSION_IS_LATEST_ROLE)
group_item.setData(is_hero, VERSION_IS_HERO_ROLE)
group_item.setData(version_label, VERSION_LABEL_ROLE)
group_item.setData(len(container_items), COUNT_ROLE)
group_item.setData(status_name, STATUS_NAME_ROLE)
group_item.setData(status_short, STATUS_SHORT_ROLE)
group_item.setData(status_color, STATUS_COLOR_ROLE)
group_item.setData(status_icon, STATUS_ICON_ROLE)
group_item.setData(
active_site_progress, ACTIVE_SITE_PROGRESS_ROLE
)
group_item.setData(
remote_site_progress, REMOTE_SITE_PROGRESS_ROLE
)
group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE)
group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE)
group_item.setData(False, IS_CONTAINER_ITEM_ROLE)
if version_color is not None:
group_item.setData(version_color, VERSION_COLOR_ROLE)
if repre_info.product_group:
group_item.setData(
repre_info.product_group, PRODUCT_GROUP_NAME_ROLE
active_site_progress, ACTIVE_SITE_PROGRESS_ROLE
)
group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE)
group_item.setData(
remote_site_progress, REMOTE_SITE_PROGRESS_ROLE
)
group_item.setData(active_site_icon, ACTIVE_SITE_ICON_ROLE)
group_item.setData(remote_site_icon, REMOTE_SITE_ICON_ROLE)
group_item.setData(False, IS_CONTAINER_ITEM_ROLE)
group_item.appendRows(container_model_items)
group_items.append(group_item)
if version_color is not None:
group_item.setData(version_color, VERSION_COLOR_ROLE)
if repre_info.product_group:
group_item.setData(
repre_info.product_group, PRODUCT_GROUP_NAME_ROLE
)
group_item.setData(group_icon, PRODUCT_GROUP_ICON_ROLE)
group_item.appendRows(container_model_items)
group_items.append(group_item)
if group_items:
root_item.appendRows(group_items)
@ -359,17 +404,21 @@ class InventoryModel(QtGui.QStandardItemModel):
root_item = self.invisibleRootItem()
root_item.removeRows(0, root_item.rowCount())
def _get_status_data(self, status_name):
status_item = self._last_project_statuses.get(status_name)
status_icon = self._get_status_icon(status_name, status_item)
def _get_status_data(self, project_name, status_name):
status_item = self._last_project_statuses[project_name].get(
status_name
)
status_icon = self._get_status_icon(
project_name, status_name, status_item
)
status_color = status_short = None
if status_item is not None:
status_color = status_item.color
status_short = status_item.short
return status_color, status_short, status_icon
def _get_status_icon(self, status_name, status_item):
icon = self._last_status_icons_by_name.get(status_name)
def _get_status_icon(self, project_name, status_name, status_item):
icon = self._last_status_icons_by_name[project_name].get(status_name)
if icon is not None:
return icon
@ -382,7 +431,7 @@ class InventoryModel(QtGui.QStandardItemModel):
})
if icon is None:
icon = QtGui.QIcon()
self._last_status_icons_by_name[status_name] = icon
self._last_status_icons_by_name[project_name][status_name] = icon
return icon

53
client/ayon_core/tools/sceneinventory/models/containers.py
View file

@ -4,6 +4,7 @@ import collections
import ayon_api
from ayon_api.graphql import GraphQlQuery
from ayon_core.lib import Logger
from ayon_core.host import ILoadHost
from ayon_core.tools.common_models.projects import StatusStates
@ -93,22 +94,27 @@ class ContainerItem:
loader_name,
namespace,
object_name,
item_id
item_id,
project_name
):
self.representation_id = representation_id
self.loader_name = loader_name
self.object_name = object_name
self.namespace = namespace
self.item_id = item_id
self.project_name = project_name
@classmethod
def from_container_data(cls, container):
def from_container_data(cls, current_project_name, container):
return cls(
representation_id=container["representation"],
loader_name=container["loader"],
namespace=container["namespace"],
object_name=container["objectName"],
item_id=uuid.uuid4().hex,
project_name=container.get(
"project_name", current_project_name
)
)
@ -191,6 +197,7 @@ class ContainersModel:
self._container_items_by_id = {}
self._version_items_by_product_id = {}
self._repre_info_by_id = {}
self._log = Logger.get_logger("ContainersModel")
def reset(self):
self._items_cache = None
@ -219,26 +226,23 @@ class ContainersModel:
for item_id in item_ids
}
def get_representation_info_items(self, representation_ids):
def get_representation_info_items(self, project_name, representation_ids):
output = {}
missing_repre_ids = set()
for repre_id in representation_ids:
try:
uuid.UUID(repre_id)
except ValueError:
except (ValueError, TypeError, AttributeError):
output[repre_id] = RepresentationInfo.new_invalid()
continue
repre_info = self._repre_info_by_id.get(repre_id)
if repre_info is None:
missing_repre_ids.add(repre_id)
else:
output[repre_id] = repre_info
if not missing_repre_ids:
return output
project_name = self._controller.get_current_project_name()
repre_hierarchy_by_id = get_representations_hierarchy(
project_name, missing_repre_ids
)
@ -276,10 +280,9 @@ class ContainersModel:
output[repre_id] = repre_info
return output
def get_version_items(self, product_ids):
def get_version_items(self, project_name, product_ids):
if not product_ids:
return {}
missing_ids = {
product_id
for product_id in product_ids
@ -294,7 +297,6 @@ class ContainersModel:
def version_sorted(entity):
return entity["version"]
project_name = self._controller.get_current_project_name()
version_entities_by_product_id = {
product_id: []
for product_id in missing_ids
@ -348,34 +350,45 @@ class ContainersModel:
return
host = self._controller.get_host()
if isinstance(host, ILoadHost):
containers = list(host.get_containers())
elif hasattr(host, "ls"):
containers = list(host.ls())
else:
containers = []
containers = []
try:
if isinstance(host, ILoadHost):
containers = list(host.get_containers())
elif hasattr(host, "ls"):
containers = list(host.ls())
except Exception:
self._log.error("Failed to get containers", exc_info=True)
container_items = []
containers_by_id = {}
container_items_by_id = {}
invalid_ids_mapping = {}
current_project_name = self._controller.get_current_project_name()
for container in containers:
if not container:
continue
try:
item = ContainerItem.from_container_data(container)
item = ContainerItem.from_container_data(
current_project_name, container)
repre_id = item.representation_id
try:
uuid.UUID(repre_id)
except (ValueError, TypeError, AttributeError):
self._log.warning(
"Container contains invalid representation id."
f"\n{container}"
)
# Fake not existing representation id so container
# is shown in UI but as invalid
item.representation_id = invalid_ids_mapping.setdefault(
repre_id, uuid.uuid4().hex
)
except Exception as e:
except Exception:
# skip item if required data are missing
self._controller.log_error(
f"Failed to create item: {e}"
self._log.warning(
"Failed to create container item", exc_info=True
)
continue

133
client/ayon_core/tools/sceneinventory/models/sitesync.py
View file

@ -11,18 +11,18 @@ class SiteSyncModel:
self._sitesync_addon = NOT_SET
self._sitesync_enabled = None
self._active_site = NOT_SET
self._remote_site = NOT_SET
self._active_site_provider = NOT_SET
self._remote_site_provider = NOT_SET
self._active_site = {}
self._remote_site = {}
self._active_site_provider = {}
self._remote_site_provider = {}
def reset(self):
self._sitesync_addon = NOT_SET
self._sitesync_enabled = None
self._active_site = NOT_SET
self._remote_site = NOT_SET
self._active_site_provider = NOT_SET
self._remote_site_provider = NOT_SET
self._active_site = {}
self._remote_site = {}
self._active_site_provider = {}
self._remote_site_provider = {}
def is_sitesync_enabled(self):
"""Site sync is enabled.
@ -46,15 +46,21 @@ class SiteSyncModel:
sitesync_addon = self._get_sitesync_addon()
return sitesync_addon.get_site_icons()
def get_sites_information(self):
def get_sites_information(self, project_name):
return {
"active_site": self._get_active_site(),
"active_site_provider": self._get_active_site_provider(),
"remote_site": self._get_remote_site(),
"remote_site_provider": self._get_remote_site_provider()
"active_site": self._get_active_site(project_name),
"remote_site": self._get_remote_site(project_name),
"active_site_provider": self._get_active_site_provider(
project_name
),
"remote_site_provider": self._get_remote_site_provider(
project_name
)
}
def get_representations_site_progress(self, representation_ids):
def get_representations_site_progress(
self, project_name, representation_ids
):
"""Get progress of representations sync."""
representation_ids = set(representation_ids)
@ -68,13 +74,12 @@ class SiteSyncModel:
if not self.is_sitesync_enabled():
return output
project_name = self._controller.get_current_project_name()
sitesync_addon = self._get_sitesync_addon()
repre_entities = ayon_api.get_representations(
project_name, representation_ids
)
active_site = self._get_active_site()
remote_site = self._get_remote_site()
active_site = self._get_active_site(project_name)
remote_site = self._get_remote_site(project_name)
for repre_entity in repre_entities:
repre_output = output[repre_entity["id"]]
@ -86,20 +91,21 @@ class SiteSyncModel:
return output
def resync_representations(self, representation_ids, site_type):
def resync_representations(
self, project_name, representation_ids, site_type
):
"""
Args:
project_name (str): Project name.
representation_ids (Iterable[str]): Representation ids.
site_type (Literal[active_site, remote_site]): Site type.
"""
project_name = self._controller.get_current_project_name()
sitesync_addon = self._get_sitesync_addon()
active_site = self._get_active_site()
remote_site = self._get_remote_site()
active_site = self._get_active_site(project_name)
remote_site = self._get_remote_site(project_name)
progress = self.get_representations_site_progress(
representation_ids
project_name, representation_ids
)
for repre_id in representation_ids:
repre_progress = progress.get(repre_id)
@ -132,48 +138,49 @@ class SiteSyncModel:
self._sitesync_addon = sitesync_addon
self._sitesync_enabled = sync_enabled
def _get_active_site(self):
if self._active_site is NOT_SET:
self._cache_sites()
return self._active_site
def _get_active_site(self, project_name):
if project_name not in self._active_site:
self._cache_sites(project_name)
return self._active_site[project_name]
def _get_remote_site(self):
if self._remote_site is NOT_SET:
self._cache_sites()
return self._remote_site
def _get_remote_site(self, project_name):
if project_name not in self._remote_site:
self._cache_sites(project_name)
return self._remote_site[project_name]
def _get_active_site_provider(self):
if self._active_site_provider is NOT_SET:
self._cache_sites()
return self._active_site_provider
def _get_active_site_provider(self, project_name):
if project_name not in self._active_site_provider:
self._cache_sites(project_name)
return self._active_site_provider[project_name]
def _get_remote_site_provider(self):
if self._remote_site_provider is NOT_SET:
self._cache_sites()
return self._remote_site_provider
def _get_remote_site_provider(self, project_name):
if project_name not in self._remote_site_provider:
self._cache_sites(project_name)
return self._remote_site_provider[project_name]
def _cache_sites(self):
active_site = None
remote_site = None
active_site_provider = None
remote_site_provider = None
if self.is_sitesync_enabled():
sitesync_addon = self._get_sitesync_addon()
project_name = self._controller.get_current_project_name()
active_site = sitesync_addon.get_active_site(project_name)
remote_site = sitesync_addon.get_remote_site(project_name)
active_site_provider = "studio"
remote_site_provider = "studio"
if active_site != "studio":
active_site_provider = sitesync_addon.get_provider_for_site(
project_name, active_site
)
if remote_site != "studio":
remote_site_provider = sitesync_addon.get_provider_for_site(
project_name, remote_site
)
def _cache_sites(self, project_name):
self._active_site[project_name] = None
self._remote_site[project_name] = None
self._active_site_provider[project_name] = None
self._remote_site_provider[project_name] = None
if not self.is_sitesync_enabled():
return
self._active_site = active_site
self._remote_site = remote_site
self._active_site_provider = active_site_provider
self._remote_site_provider = remote_site_provider
sitesync_addon = self._get_sitesync_addon()
active_site = sitesync_addon.get_active_site(project_name)
remote_site = sitesync_addon.get_remote_site(project_name)
active_site_provider = "studio"
remote_site_provider = "studio"
if active_site != "studio":
active_site_provider = sitesync_addon.get_provider_for_site(
project_name, active_site
)
if remote_site != "studio":
remote_site_provider = sitesync_addon.get_provider_for_site(
project_name, remote_site
)
self._active_site[project_name] = active_site
self._remote_site[project_name] = remote_site
self._active_site_provider[project_name] = active_site_provider
self._remote_site_provider[project_name] = remote_site_provider

18
client/ayon_core/tools/sceneinventory/switch_dialog/dialog.py
View file

@ -46,8 +46,13 @@ class SwitchAssetDialog(QtWidgets.QDialog):
switched = QtCore.Signal()
def __init__(self, controller, parent=None, items=None):
super(SwitchAssetDialog, self).__init__(parent)
def __init__(self, controller, project_name, items, parent=None):
super().__init__(parent)
current_project_name = controller.get_current_project_name()
folder_id = None
if current_project_name == project_name:
folder_id = controller.get_current_folder_id()
self.setWindowTitle("Switch selected items ...")
@ -147,11 +152,10 @@ class SwitchAssetDialog(QtWidgets.QDialog):
self._init_repre_name = None
self._fill_check = False
self._project_name = project_name
self._folder_id = folder_id
self._project_name = controller.get_current_project_name()
self._folder_id = controller.get_current_folder_id()
self._current_folder_btn.setEnabled(self._folder_id is not None)
self._current_folder_btn.setEnabled(folder_id is not None)
self._controller = controller
@ -159,7 +163,7 @@ class SwitchAssetDialog(QtWidgets.QDialog):
self._prepare_content_data()
def showEvent(self, event):
super(SwitchAssetDialog, self).showEvent(event)
super().showEvent(event)
self._show_timer.start()
def refresh(self, init_refresh=False):

340
client/ayon_core/tools/sceneinventory/view.py
View file

@ -192,29 +192,46 @@ class SceneInventoryView(QtWidgets.QTreeView):
container_item = container_items_by_id[item_id]
active_repre_id = container_item.representation_id
break
repre_ids_by_project = collections.defaultdict(set)
for container_item in container_items_by_id.values():
repre_id = container_item.representation_id
project_name = container_item.project_name
repre_ids_by_project[project_name].add(repre_id)
repre_info_by_id = self._controller.get_representation_info_items({
container_item.representation_id
for container_item in container_items_by_id.values()
})
valid_repre_ids = {
repre_id
for repre_id, repre_info in repre_info_by_id.items()
if repre_info.is_valid
}
repre_info_by_project = {}
repre_ids_by_project_name = {}
version_ids_by_project = {}
product_ids_by_project = {}
for project_name, repre_ids in repre_ids_by_project.items():
repres_info = self._controller.get_representation_info_items(
project_name, repre_ids
)
repre_info_by_project[project_name] = repres_info
repre_ids = set()
version_ids = set()
product_ids = set()
for repre_id, repre_info in repres_info.items():
if not repre_info.is_valid:
continue
repre_ids.add(repre_id)
version_ids.add(repre_info.version_id)
product_ids.add(repre_info.product_id)
repre_ids_by_project_name[project_name] = repre_ids
version_ids_by_project[project_name] = version_ids
product_ids_by_project[project_name] = product_ids
# Exclude items that are "NOT FOUND" since setting versions, updating
# and removal won't work for those items.
filtered_items = []
product_ids = set()
version_ids = set()
for container_item in container_items_by_id.values():
project_name = container_item.project_name
repre_id = container_item.representation_id
repre_info_by_id = repre_info_by_project.get(project_name, {})
repre_info = repre_info_by_id.get(repre_id)
if repre_info and repre_info.is_valid:
filtered_items.append(container_item)
version_ids.add(repre_info.version_id)
product_ids.add(repre_info.product_id)
# remove
remove_icon = qtawesome.icon("fa.remove", color=DEFAULT_COLOR)
@ -227,43 +244,51 @@ class SceneInventoryView(QtWidgets.QTreeView):
menu.addAction(remove_action)
return
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
version_items_by_project = {
project_name: self._controller.get_version_items(
project_name, product_ids
)
for project_name, product_ids in product_ids_by_project.items()
}
has_outdated = False
has_loaded_hero_versions = False
has_available_hero_version = False
has_outdated_approved = False
last_version_by_product_id = {}
for product_id, version_items_by_id in (
version_items_by_product_id.items()
for project_name, version_items_by_product_id in (
version_items_by_project.items()
):
_has_outdated_approved = False
_last_approved_version_item = None
for version_item in version_items_by_id.values():
if version_item.is_hero:
has_available_hero_version = True
elif version_item.is_last_approved:
_last_approved_version_item = version_item
_has_outdated_approved = True
if version_item.version_id not in version_ids:
continue
if version_item.is_hero:
has_loaded_hero_versions = True
elif not version_item.is_latest:
has_outdated = True
if (
_has_outdated_approved
and _last_approved_version_item is not None
version_ids = version_ids_by_project[project_name]
for product_id, version_items_by_id in (
version_items_by_product_id.items()
):
last_version_by_product_id[product_id] = (
_last_approved_version_item
)
has_outdated_approved = True
_has_outdated_approved = False
_last_approved_version_item = None
for version_item in version_items_by_id.values():
if version_item.is_hero:
has_available_hero_version = True
elif version_item.is_last_approved:
_last_approved_version_item = version_item
_has_outdated_approved = True
if version_item.version_id not in version_ids:
continue
if version_item.is_hero:
has_loaded_hero_versions = True
elif not version_item.is_latest:
has_outdated = True
if (
_has_outdated_approved
and _last_approved_version_item is not None
):
last_version_by_product_id[product_id] = (
_last_approved_version_item
)
has_outdated_approved = True
switch_to_versioned = None
if has_loaded_hero_versions:
@ -284,8 +309,9 @@ class SceneInventoryView(QtWidgets.QTreeView):
approved_version_by_item_id = {}
if has_outdated_approved:
for container_item in container_items_by_id.values():
project_name = container_item.project_name
repre_id = container_item.representation_id
repre_info = repre_info_by_id.get(repre_id)
repre_info = repre_info_by_project[project_name][repre_id]
if not repre_info or not repre_info.is_valid:
continue
version_item = last_version_by_product_id.get(
@ -397,14 +423,15 @@ class SceneInventoryView(QtWidgets.QTreeView):
menu.addAction(remove_action)
self._handle_sitesync(menu, valid_repre_ids)
self._handle_sitesync(menu, repre_ids_by_project_name)
def _handle_sitesync(self, menu, repre_ids):
def _handle_sitesync(self, menu, repre_ids_by_project_name):
"""Adds actions for download/upload when SyncServer is enabled
Args:
menu (OptionMenu)
repre_ids (list) of object_ids
repre_ids_by_project_name (Dict[str, Set[str]]): Representation
ids by project name.
Returns:
(OptionMenu)
@ -413,7 +440,7 @@ class SceneInventoryView(QtWidgets.QTreeView):
if not self._controller.is_sitesync_enabled():
return
if not repre_ids:
if not repre_ids_by_project_name:
return
menu.addSeparator()
@ -425,7 +452,10 @@ class SceneInventoryView(QtWidgets.QTreeView):
menu
)
download_active_action.triggered.connect(
lambda: self._add_sites(repre_ids, "active_site"))
lambda: self._add_sites(
repre_ids_by_project_name, "active_site"
)
)
upload_icon = qtawesome.icon("fa.upload", color=DEFAULT_COLOR)
upload_remote_action = QtWidgets.QAction(
@ -434,23 +464,30 @@ class SceneInventoryView(QtWidgets.QTreeView):
menu
)
upload_remote_action.triggered.connect(
lambda: self._add_sites(repre_ids, "remote_site"))
lambda: self._add_sites(
repre_ids_by_project_name, "remote_site"
)
)
menu.addAction(download_active_action)
menu.addAction(upload_remote_action)
def _add_sites(self, repre_ids, site_type):
def _add_sites(self, repre_ids_by_project_name, site_type):
"""(Re)sync all 'repre_ids' to specific site.
It checks if opposite site has fully available content to limit
accidents. (ReSync active when no remote >> losing active content)
Args:
repre_ids (list)
repre_ids_by_project_name (Dict[str, Set[str]]): Representation
ids by project name.
site_type (Literal[active_site, remote_site]): Site type.
"""
self._controller.resync_representations(repre_ids, site_type)
"""
for project_name, repre_ids in repre_ids_by_project_name.items():
self._controller.resync_representations(
project_name, repre_ids, site_type
)
self.data_changed.emit()
@ -735,42 +772,68 @@ class SceneInventoryView(QtWidgets.QTreeView):
container_items_by_id = self._controller.get_container_items_by_id(
item_ids
)
repre_ids = {
container_item.representation_id
for container_item in container_items_by_id.values()
}
repre_info_by_id = self._controller.get_representation_info_items(
repre_ids
)
project_names = set()
repre_ids_by_project = collections.defaultdict(set)
for container_item in container_items_by_id.values():
repre_id = container_item.representation_id
project_name = container_item.project_name
project_names.add(project_name)
repre_ids_by_project[project_name].add(repre_id)
# active_project_name = None
active_repre_info = None
repre_info_by_project = {}
version_items_by_project = {}
for project_name, repre_ids in repre_ids_by_project.items():
repres_info = self._controller.get_representation_info_items(
project_name, repre_ids
)
if active_repre_info is None:
# active_project_name = project_name
active_repre_info = repres_info.get(active_repre_id)
product_ids = {
repre_info.product_id
for repre_info in repres_info.values()
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
project_name, product_ids
)
repre_info_by_project[project_name] = repres_info
version_items_by_project[project_name] = (
version_items_by_product_id
)
product_ids = {
repre_info.product_id
for repre_info in repre_info_by_id.values()
}
active_repre_info = repre_info_by_id[active_repre_id]
active_version_id = active_repre_info.version_id
active_product_id = active_repre_info.product_id
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
version_items = list(
version_items_by_product_id[active_product_id].values()
)
versions = {version_item.version for version_item in version_items}
product_ids_by_version = collections.defaultdict(set)
for version_items_by_id in version_items_by_product_id.values():
for version_item in version_items_by_id.values():
version = version_item.version
_prod_version = version
if _prod_version < 0:
_prod_version = -1
product_ids_by_version[_prod_version].add(
version_item.product_id
)
if version in versions:
continue
versions.add(version)
version_items.append(version_item)
# active_product_id = active_repre_info.product_id
versions = set()
product_ids = set()
version_items = []
product_ids_by_version_by_project = {}
for project_name, version_items_by_product_id in (
version_items_by_project.items()
):
product_ids_by_version = collections.defaultdict(set)
product_ids_by_version_by_project[project_name] = (
product_ids_by_version
)
for version_items_by_id in version_items_by_product_id.values():
for version_item in version_items_by_id.values():
version = version_item.version
_prod_version = version
if _prod_version < 0:
_prod_version = -1
product_ids_by_version[_prod_version].add(
version_item.product_id
)
product_ids.add(version_item.product_id)
if version in versions:
continue
versions.add(version)
version_items.append(version_item)
def version_sorter(item):
hero_value = 0
@ -831,12 +894,15 @@ class SceneInventoryView(QtWidgets.QTreeView):
product_version = -1
version = HeroVersionType(version)
product_ids = product_ids_by_version[product_version]
filtered_item_ids = set()
for container_item in container_items_by_id.values():
project_name = container_item.project_name
product_ids_by_version = (
product_ids_by_version_by_project[project_name]
)
product_ids = product_ids_by_version[product_version]
repre_id = container_item.representation_id
repre_info = repre_info_by_id[repre_id]
repre_info = repre_info_by_project[project_name][repre_id]
if repre_info.product_id in product_ids:
filtered_item_ids.add(container_item.item_id)
@ -846,14 +912,28 @@ class SceneInventoryView(QtWidgets.QTreeView):
def _show_switch_dialog(self, item_ids):
"""Display Switch dialog"""
containers_by_id = self._controller.get_containers_by_item_ids(
container_items_by_id = self._controller.get_container_items_by_id(
item_ids
)
dialog = SwitchAssetDialog(
self._controller, self, list(containers_by_id.values())
)
dialog.switched.connect(self.data_changed.emit)
dialog.show()
container_ids_by_project_name = collections.defaultdict(set)
for container_id, container_item in container_items_by_id.items():
project_name = container_item.project_name
container_ids_by_project_name[project_name].add(container_id)
for project_name, container_ids in (
container_ids_by_project_name.items()
):
containers_by_id = self._controller.get_containers_by_item_ids(
container_ids
)
dialog = SwitchAssetDialog(
self._controller,
project_name,
list(containers_by_id.values()),
self
)
dialog.switched.connect(self.data_changed.emit)
dialog.show()
def _show_remove_warning_dialog(self, item_ids):
"""Prompt a dialog to inform the user the action will remove items"""
@ -927,38 +1007,58 @@ class SceneInventoryView(QtWidgets.QTreeView):
self._update_containers_to_version(item_ids, version=-1)
def _on_switch_to_versioned(self, item_ids):
# Get container items by ID
containers_items_by_id = self._controller.get_container_items_by_id(
item_ids
)
repre_ids = {
container_item.representation_id
for container_item in containers_items_by_id.values()
}
repre_info_by_id = self._controller.get_representation_info_items(
repre_ids
)
product_ids = {
repre_info.product_id
for repre_info in repre_info_by_id.values()
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
product_ids
)
item_ids)
# Extract project names and their corresponding representation IDs
repre_ids_by_project = collections.defaultdict(set)
for container_item in containers_items_by_id.values():
project_name = container_item.project_name
repre_id = container_item.representation_id
repre_ids_by_project[project_name].add(repre_id)
# Get representation info items by ID
repres_info_by_project = {}
version_items_by_project = {}
for project_name, repre_ids in repre_ids_by_project.items():
repre_info_by_id = self._controller.get_representation_info_items(
project_name, repre_ids
)
repres_info_by_project[project_name] = repre_info_by_id
product_ids = {
repre_info.product_id
for repre_info in repre_info_by_id.values()
if repre_info.is_valid
}
version_items_by_product_id = self._controller.get_version_items(
project_name, product_ids
)
version_items_by_project[project_name] = (
version_items_by_product_id
)
update_containers = []
update_versions = []
for item_id, container_item in containers_items_by_id.items():
for container_item in containers_items_by_id.values():
project_name = container_item.project_name
repre_id = container_item.representation_id
repre_info_by_id = repres_info_by_project[project_name]
repre_info = repre_info_by_id[repre_id]
version_items_by_product_id = (
version_items_by_project[project_name]
)
product_id = repre_info.product_id
version_items_id = version_items_by_product_id[product_id]
version_item = version_items_id.get(repre_info.version_id, {})
version_items_by_id = version_items_by_product_id[product_id]
version_item = version_items_by_id.get(repre_info.version_id, {})
if not version_item or not version_item.is_hero:
continue
version = abs(version_item.version)
version_found = False
for version_item in version_items_id.values():
for version_item in version_items_by_id.values():
if version_item.is_hero:
continue
if version_item.version == version:
@ -971,8 +1071,8 @@ class SceneInventoryView(QtWidgets.QTreeView):
update_containers.append(container_item.item_id)
update_versions.append(version)
# Specify version per item to update to
self._update_containers(update_containers, update_versions)
# Specify version per item to update to
self._update_containers(update_containers, update_versions)
def _update_containers(self, item_ids, versions):
"""Helper to update items to given version (or version per item)

2
client/ayon_core/version.py
View file

@ -1,3 +1,3 @@
# -*- coding: utf-8 -*-
"""Package declaring AYON addon 'core' version."""
__version__ = "1.0.10+dev"
__version__ = "1.0.11+dev"

2
package.py
View file

@ -1,6 +1,6 @@
name = "core"
title = "Core"
version = "1.0.10+dev"
version = "1.0.11+dev"
client_dir = "ayon_core"

2
pyproject.toml
View file

@ -5,7 +5,7 @@
[tool.poetry]
name = "ayon-core"
version = "1.0.10+dev"
version = "1.0.11+dev"
description = ""
authors = ["Ynput Team <team@ynput.io>"]
readme = "README.md"