Merge branch 'develop' into chore/cleanup_abstract_collect_render

This commit is contained in:
Roy Nieterau 2024-04-11 17:26:39 +02:00 committed by GitHub
commit af00f43ce8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 184 additions and 135 deletions

View file

@ -15,6 +15,7 @@ from abc import ABCMeta, abstractmethod
import six
import appdirs
import ayon_api
from semver import VersionInfo
from ayon_core import AYON_CORE_ROOT
from ayon_core.lib import Logger, is_dev_mode_enabled
@ -46,6 +47,11 @@ IGNORED_HOSTS_IN_AYON = {
}
IGNORED_MODULES_IN_AYON = set()
# When addon was moved from ayon-core codebase
# - this is used to log the missing addon
MOVED_ADDON_MILESTONE_VERSIONS = {
"applications": VersionInfo(2, 0, 0),
}
# Inherit from `object` for Python 2 hosts
class _ModuleClass(object):
@ -192,6 +198,45 @@ def _get_ayon_addons_information(bundle_info):
return output
def _handle_moved_addons(addon_name, milestone_version, log):
"""Log message that addon version is not compatible with current core.
The function can return path to addon client code, but that can happen
only if ayon-core is used from code (for development), but still
logs a warning.
Args:
addon_name (str): Addon name.
milestone_version (str): Milestone addon version.
log (logging.Logger): Logger object.
Returns:
Union[str, None]: Addon dir or None.
"""
# Handle addons which were moved out of ayon-core
# - Try to fix it by loading it directly from server addons dir in
# ayon-core repository. But that will work only if ayon-core is
# used from code.
addon_dir = os.path.join(
os.path.dirname(os.path.dirname(AYON_CORE_ROOT)),
"server_addon",
addon_name,
"client",
)
if not os.path.exists(addon_dir):
log.error((
"Addon '{}' is not be available."
" Please update applications addon to '{}' or higher."
).format(addon_name, milestone_version))
return None
log.warning((
"Please update '{}' addon to '{}' or higher."
" Using client code from ayon-core repository."
).format(addon_name, milestone_version))
return addon_dir
def _load_ayon_addons(openpype_modules, modules_key, log):
"""Load AYON addons based on information from server.
@ -249,6 +294,7 @@ def _load_ayon_addons(openpype_modules, modules_key, log):
use_dev_path = dev_addon_info.get("enabled", False)
addon_dir = None
milestone_version = MOVED_ADDON_MILESTONE_VERSIONS.get(addon_name)
if use_dev_path:
addon_dir = dev_addon_info["path"]
if not addon_dir or not os.path.exists(addon_dir):
@ -257,6 +303,16 @@ def _load_ayon_addons(openpype_modules, modules_key, log):
).format(addon_name, addon_version, addon_dir))
continue
elif (
milestone_version is not None
and VersionInfo.parse(addon_version) < milestone_version
):
addon_dir = _handle_moved_addons(
addon_name, milestone_version, log
)
if not addon_dir:
continue
elif addons_dir_exists:
folder_name = "{}_{}".format(addon_name, addon_version)
addon_dir = os.path.join(addons_dir, folder_name)
@ -336,66 +392,9 @@ def _load_ayon_addons(openpype_modules, modules_key, log):
return addons_to_skip_in_core
def _load_ayon_core_addons_dir(
ignore_addon_names, openpype_modules, modules_key, log
):
addons_dir = os.path.join(AYON_CORE_ROOT, "addons")
if not os.path.exists(addons_dir):
return
imported_modules = []
# Make sure that addons which already have client code are not loaded
# from core again, with older code
filtered_paths = []
for name in os.listdir(addons_dir):
if name in ignore_addon_names:
continue
path = os.path.join(addons_dir, name)
if os.path.isdir(path):
filtered_paths.append(path)
for path in filtered_paths:
while path in sys.path:
sys.path.remove(path)
sys.path.insert(0, path)
for name in os.listdir(path):
fullpath = os.path.join(path, name)
if os.path.isfile(fullpath):
basename, ext = os.path.splitext(name)
if ext != ".py":
continue
else:
basename = name
try:
module = __import__(basename, fromlist=("",))
for attr_name in dir(module):
attr = getattr(module, attr_name)
if (
inspect.isclass(attr)
and issubclass(attr, AYONAddon)
):
new_import_str = "{}.{}".format(modules_key, basename)
sys.modules[new_import_str] = module
setattr(openpype_modules, basename, module)
imported_modules.append(module)
break
except Exception:
log.error(
"Failed to import addon '{}'.".format(fullpath),
exc_info=True
)
return imported_modules
def _load_addons_in_core(
ignore_addon_names, openpype_modules, modules_key, log
):
_load_ayon_core_addons_dir(
ignore_addon_names, openpype_modules, modules_key, log
)
# Add current directory at first place
# - has small differences in import logic
hosts_dir = os.path.join(AYON_CORE_ROOT, "hosts")

View file

@ -139,7 +139,6 @@ class InstallPySideToBlender(PreLaunchHook):
administration rights.
"""
try:
import win32api
import win32con
import win32process
import win32event

View file

@ -3,8 +3,8 @@ import sys
import re
import contextlib
from ayon_core.lib import Logger
from ayon_core.lib import Logger, BoolDef, UILabelDef
from ayon_core.style import load_stylesheet
from ayon_core.pipeline import registered_host
from ayon_core.pipeline.create import CreateContext
from ayon_core.pipeline.context_tools import get_current_folder_entity
@ -181,7 +181,6 @@ def validate_comp_prefs(comp=None, force_repair=False):
from . import menu
from ayon_core.tools.utils import SimplePopup
from ayon_core.style import load_stylesheet
dialog = SimplePopup(parent=menu.menu)
dialog.setWindowTitle("Fusion comp has invalid configuration")
@ -340,9 +339,7 @@ def prompt_reset_context():
from ayon_core.tools.attribute_defs.dialog import (
AttributeDefinitionsDialog
)
from ayon_core.style import load_stylesheet
from ayon_core.lib import BoolDef, UILabelDef
from qtpy import QtWidgets, QtCore
from qtpy import QtCore
definitions = [
UILabelDef(

View file

@ -85,7 +85,6 @@ class InstallPySideToFusion(PreLaunchHook):
administration rights.
"""
try:
import win32api
import win32con
import win32process
import win32event

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating alembic camera products."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance, CreatorError
from ayon_core.pipeline import CreatorError
import hou
@ -23,7 +23,7 @@ class CreateAlembicCamera(plugin.HoudiniCreator):
instance = super(CreateAlembicCamera, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))
parms = {

View file

@ -29,7 +29,7 @@ class CreateArnoldAss(plugin.HoudiniCreator):
instance = super(CreateArnoldAss, self).create(
product_name,
instance_data,
pre_create_data) # type: plugin.CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -31,7 +31,7 @@ class CreateArnoldRop(plugin.HoudiniCreator):
instance = super(CreateArnoldRop, self).create(
product_name,
instance_data,
pre_create_data) # type: plugin.CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating pointcache bgeo files."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance, CreatorError
from ayon_core.pipeline import CreatorError
import hou
from ayon_core.lib import EnumDef, BoolDef
@ -25,7 +25,7 @@ class CreateBGEO(plugin.HoudiniCreator):
instance = super(CreateBGEO, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating composite sequences."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance, CreatorError
from ayon_core.pipeline import CreatorError
import hou
@ -25,7 +25,7 @@ class CreateCompositeSequence(plugin.HoudiniCreator):
instance = super(CreateCompositeSequence, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))
filepath = "{}{}".format(

View file

@ -78,7 +78,7 @@ class CreateHDA(plugin.HoudiniCreator):
instance = super(CreateHDA, self).create(
product_name,
instance_data,
pre_create_data) # type: plugin.CreatedInstance
pre_create_data)
return instance

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin to create Karma ROP."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
from ayon_core.lib import BoolDef, EnumDef, NumberDef
@ -25,7 +24,7 @@ class CreateKarmaROP(plugin.HoudiniCreator):
instance = super(CreateKarmaROP, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating pointcache alembics."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
from ayon_core.lib import BoolDef
@ -22,7 +21,7 @@ class CreateMantraIFD(plugin.HoudiniCreator):
instance = super(CreateMantraIFD, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin to create Mantra ROP."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
from ayon_core.lib import EnumDef, BoolDef
@ -28,7 +27,7 @@ class CreateMantraROP(plugin.HoudiniCreator):
instance = super(CreateMantraROP, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating USDs."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
import hou
@ -22,7 +21,7 @@ class CreateUSD(plugin.HoudiniCreator):
instance = super(CreateUSD, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating USD renders."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
class CreateUSDRender(plugin.HoudiniCreator):
@ -23,7 +22,7 @@ class CreateUSDRender(plugin.HoudiniCreator):
instance = super(CreateUSDRender, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
"""Creator plugin for creating VDB Caches."""
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance
from ayon_core.lib import BoolDef
import hou
@ -26,7 +25,7 @@ class CreateVDBCache(plugin.HoudiniCreator):
instance = super(CreateVDBCache, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))
file_path = "{}{}".format(

View file

@ -3,7 +3,7 @@
import hou
from ayon_core.hosts.houdini.api import plugin
from ayon_core.pipeline import CreatedInstance, CreatorError
from ayon_core.pipeline import CreatorError
from ayon_core.lib import EnumDef, BoolDef
@ -31,7 +31,7 @@ class CreateVrayROP(plugin.HoudiniCreator):
instance = super(CreateVrayROP, self).create(
product_name,
instance_data,
pre_create_data) # type: CreatedInstance
pre_create_data)
instance_node = hou.node(instance.get("instance_node"))

View file

@ -3,7 +3,6 @@ import pyblish.api
from ayon_core.lib import version_up
from ayon_core.pipeline import registered_host
from ayon_core.pipeline.publish import get_errored_plugins_from_context
from ayon_core.hosts.houdini.api import HoudiniHost
from ayon_core.pipeline.publish import KnownPublishError
@ -39,7 +38,7 @@ class IncrementCurrentFile(pyblish.api.ContextPlugin):
)
# Filename must not have changed since collecting
host = registered_host() # type: HoudiniHost
host = registered_host()
current_file = host.current_file()
if context.data["currentFile"] != current_file:
raise KnownPublishError(

View file

@ -2,8 +2,6 @@
"""Tools to work with FBX."""
import logging
from pyblish.api import Instance
from maya import cmds # noqa
import maya.mel as mel # noqa
from ayon_core.hosts.maya.api.lib import maintained_selection
@ -146,7 +144,6 @@ class FBXExtractor:
return options
def set_options_from_instance(self, instance):
# type: (Instance) -> None
"""Sets FBX export options from data in the instance.
Args:

View file

@ -2,7 +2,6 @@ from maya import cmds
import pyblish.api
from ayon_core.pipeline.publish import (
ValidateContentsOrder,
RepairContextAction,
PublishValidationError
)

View file

@ -5,7 +5,7 @@ import sys
import six
import random
import string
from collections import OrderedDict, defaultdict
from collections import defaultdict
from ayon_core.settings import get_current_project_settings
from ayon_core.lib import (

View file

@ -144,7 +144,8 @@ class CreateTextures(Creator):
9: "512",
10: "1024",
11: "2048",
12: "4096"
12: "4096",
13: "8192"
},
default=None,
label="Size"),

View file

@ -1,13 +1,14 @@
# -*- coding: utf-8 -*-
"""Wrapper around Royal Render API."""
import sys
import os
import sys
from ayon_core.lib.local_settings import AYONSettingsRegistry
from ayon_core.lib import Logger, run_subprocess
from .rr_job import RRJob, SubmitFile, SubmitterParameter
from ayon_core.lib import Logger, run_subprocess, AYONSettingsRegistry
from ayon_core.lib.vendor_bin_utils import find_tool_in_custom_paths
from .rr_job import SubmitFile
from .rr_job import RRjob, SubmitterParameter # noqa F401
class Api:

View file

@ -3,7 +3,6 @@
import os
import attr
import json
import re
import pyblish.api

View file

@ -549,7 +549,7 @@ class Anatomy(BaseAnatomy):
)
else:
# Ask sync server to get roots overrides
roots_overrides = sitesync.get_site_root_overrides(
roots_overrides = sitesync_addon.get_site_root_overrides(
project_name, site_name
)
site_cache.update_data(roots_overrides)

View file

@ -14,7 +14,6 @@ from .exceptions import (
TemplateMissingKey,
AnatomyTemplateUnsolved,
)
from .roots import RootItem
_PLACEHOLDER = object()

View file

@ -1,7 +1,6 @@
"""Core pipeline functionality"""
import os
import types
import logging
import platform
import uuid
@ -21,7 +20,6 @@ from .anatomy import Anatomy
from .template_data import get_template_data_with_names
from .workfile import (
get_workdir,
get_workfile_template_key,
get_custom_workfile_template_by_string_context,
)
from . import (

View file

@ -6,13 +6,11 @@ from copy import deepcopy
import attr
import ayon_api
import pyblish.api
import clique
from ayon_core.pipeline import (
get_current_project_name,
get_representation_path,
Anatomy,
)
from ayon_core.lib import Logger
from ayon_core.pipeline.publish import KnownPublishError
@ -137,7 +135,7 @@ def get_transferable_representations(instance):
list of dicts: List of transferable representations.
"""
anatomy = instance.context.data["anatomy"] # type: Anatomy
anatomy = instance.context.data["anatomy"]
to_transfer = []
for representation in instance.data.get("representations", []):
@ -166,7 +164,6 @@ def get_transferable_representations(instance):
def create_skeleton_instance(
instance, families_transfer=None, instance_transfer=None):
# type: (pyblish.api.Instance, list, dict) -> dict
"""Create skeleton instance from original instance data.
This will create dictionary containing skeleton
@ -191,7 +188,7 @@ def create_skeleton_instance(
context = instance.context
data = instance.data.copy()
anatomy = instance.context.data["anatomy"] # type: Anatomy
anatomy = instance.context.data["anatomy"]
# get time related data from instance (or context)
time_data = get_time_data_from_instance_or_context(instance)
@ -751,7 +748,6 @@ def get_resources(project_name, version_entity, extension=None):
def create_skeleton_instance_cache(instance):
# type: (pyblish.api.Instance, list, dict) -> dict
"""Create skeleton instance from original instance data.
This will create dictionary containing skeleton
@ -771,7 +767,7 @@ def create_skeleton_instance_cache(instance):
context = instance.context
data = instance.data.copy()
anatomy = instance.context.data["anatomy"] # type: Anatomy
anatomy = instance.context.data["anatomy"]
# get time related data from instance (or context)
time_data = get_time_data_from_instance_or_context(instance)
@ -1005,7 +1001,7 @@ def copy_extend_frames(instance, representation):
start = instance.data.get("frameStart")
end = instance.data.get("frameEnd")
project_name = instance.context.data["project"]
anatomy = instance.context.data["anatomy"] # type: Anatomy
anatomy = instance.context.data["anatomy"]
folder_entity = ayon_api.get_folder_by_path(
project_name, instance.data.get("folderPath")

View file

@ -13,7 +13,6 @@ Resources:
"""
import os
import re
import json
import logging

View file

@ -3,8 +3,6 @@ import platform
import subprocess
from string import Formatter
import ayon_api
from ayon_core.pipeline import (
Anatomy,
LauncherAction,

View file

@ -0,0 +1,3 @@
name = "applications"
title = "Applications"
version = "0.2.0"

View file

@ -6,7 +6,6 @@ from ayon_server.addons import BaseServerAddon, AddonLibrary
from ayon_server.entities.core import attribute_library
from ayon_server.lib.postgres import Postgres
from .version import __version__
from .settings import ApplicationsAddonSettings, DEFAULT_VALUES
try:
@ -87,9 +86,6 @@ def get_enum_items_from_groups(groups):
class ApplicationsAddon(BaseServerAddon):
name = "applications"
title = "Applications"
version = __version__
settings_model = ApplicationsAddonSettings
async def get_default_settings(self):

View file

@ -1 +0,0 @@
__version__ = "0.1.9"

View file

@ -4,6 +4,8 @@ import re
import shutil
import argparse
import zipfile
import types
import importlib
import platform
import collections
from pathlib import Path
@ -44,6 +46,11 @@ version = "{addon_version}"
plugin_for = ["ayon_server"]
"""
CLIENT_VERSION_CONTENT = '''# -*- coding: utf-8 -*-
"""Package declaring AYON core addon version."""
__version__ = "{}"
'''
class ZipFileLongPaths(zipfile.ZipFile):
"""Allows longer paths in zip files.
@ -175,13 +182,75 @@ def create_addon_zip(
shutil.rmtree(str(output_dir / addon_name))
def prepare_client_code(
addon_dir: Path,
addon_output_dir: Path,
addon_version: str
):
client_dir = addon_dir / "client"
if not client_dir.exists():
return
# Prepare private dir in output
private_dir = addon_output_dir / "private"
private_dir.mkdir(parents=True, exist_ok=True)
# Copy pyproject toml if available
pyproject_toml = client_dir / "pyproject.toml"
if pyproject_toml.exists():
shutil.copy(pyproject_toml, private_dir)
for subpath in client_dir.iterdir():
if subpath.name == "pyproject.toml":
continue
if subpath.is_file():
continue
# Update version.py with server version if 'version.py' is available
version_path = subpath / "version.py"
if version_path.exists():
with open(version_path, "w") as stream:
stream.write(CLIENT_VERSION_CONTENT.format(addon_version))
zip_filepath = private_dir / "client.zip"
with ZipFileLongPaths(zip_filepath, "w", zipfile.ZIP_DEFLATED) as zipf:
# Add client code content to zip
for path, sub_path in find_files_in_subdir(str(subpath)):
sub_path = os.path.join(subpath.name, sub_path)
zipf.write(path, sub_path)
def import_filepath(path: Path, module_name: Optional[str] = None):
if not module_name:
module_name = os.path.splitext(path.name)[0]
# Convert to string
path = str(path)
module = types.ModuleType(module_name)
module.__file__ = path
# Use loader so module has full specs
module_loader = importlib.machinery.SourceFileLoader(
module_name, path
)
module_loader.exec_module(module)
return module
def create_addon_package(
addon_dir: Path,
output_dir: Path,
create_zip: bool,
keep_source: bool,
):
addon_version = get_addon_version(addon_dir)
src_package_py = addon_dir / "package.py"
package = None
if src_package_py.exists():
package = import_filepath(src_package_py)
addon_version = package.version
else:
addon_version = get_addon_version(addon_dir)
addon_output_dir = output_dir / addon_dir.name / addon_version
if addon_output_dir.exists():
@ -189,22 +258,27 @@ def create_addon_package(
addon_output_dir.mkdir(parents=True)
# Copy server content
package_py = addon_output_dir / "package.py"
addon_name = addon_dir.name
if addon_name == "royal_render":
addon_name = "royalrender"
package_py_content = PACKAGE_PY_TEMPLATE.format(
addon_name=addon_name, addon_version=addon_version
)
dst_package_py = addon_output_dir / "package.py"
if package is not None:
shutil.copy(src_package_py, dst_package_py)
else:
addon_name = addon_dir.name
if addon_name == "royal_render":
addon_name = "royalrender"
package_py_content = PACKAGE_PY_TEMPLATE.format(
addon_name=addon_name, addon_version=addon_version
)
with open(package_py, "w+") as pkg_py:
pkg_py.write(package_py_content)
with open(dst_package_py, "w+") as pkg_py:
pkg_py.write(package_py_content)
server_dir = addon_dir / "server"
shutil.copytree(
server_dir, addon_output_dir / "server", dirs_exist_ok=True
)
prepare_client_code(addon_dir, addon_output_dir, addon_version)
if create_zip:
create_addon_zip(
output_dir, addon_dir.name, addon_version, keep_source

View file

@ -1,3 +1,4 @@
from typing import TYPE_CHECKING
from pydantic import validator
from ayon_server.settings import (
@ -5,6 +6,8 @@ from ayon_server.settings import (
SettingsField,
ensure_unique_names,
)
if TYPE_CHECKING:
from ayon_server.addons import BaseServerAddon
from .publish_plugins import (
PublishPluginsModel,
@ -19,7 +22,7 @@ class ServerListSubmodel(BaseSettingsModel):
async def defined_deadline_ws_name_enum_resolver(
addon: "BaseServerAddon",
addon: BaseServerAddon,
settings_variant: str = "production",
project_name: str | None = None,
) -> list[str]:

View file

@ -1,5 +1,5 @@
from ayon_server.settings import BaseSettingsModel, SettingsField
from ayon_server.types import ColorRGB_float, ColorRGBA_uint8
from ayon_server.types import ColorRGBA_uint8
class LoaderEnabledModel(BaseSettingsModel):

View file

@ -6,7 +6,7 @@ from ayon_server.settings import (
ensure_unique_names,
task_types_enum,
)
from ayon_server.types import ColorRGBA_uint8, ColorRGB_float
from ayon_server.types import ColorRGBA_uint8
def hardware_falloff_enum():

View file

@ -1,5 +1,5 @@
from ayon_server.settings import BaseSettingsModel, SettingsField
from ayon_server.types import ColorRGBA_uint8, ColorRGB_uint8
from ayon_server.types import ColorRGBA_uint8
class CollectRenderInstancesModel(BaseSettingsModel):