Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

AYON: Bundle distribution (#5209)

Browse source 
* modified distribution to use bundles

* use bundles in modules discovery logic

* removed unused import

* added support for bundle settings getter

* added script launch mechanism to ayon start script

* show login UI through subprocess

* removed silent mode

* removed unused variable

* match env variables to ayon launcher

* moved ui lib function to ayon common

* raise custom exception on missing bundle name

* implemented missing bundle window to show issues with bundles

* implemented helper function to show dialog about issues with bundle

* handle issues with bundles

* removed unused import

* dont convert passed addons infor

* access keys only in server getters

* fix accessed attribute

* fix test

* fixed missing 'message' variable

* removed duplicated data

* removed unnecessary 'sha256' variable

* use lstrip instead of replacement

* use f-string

* move import to the top of file

* added some dosctrings

* change type

* use f-string

* fix grammar

* set default settings variant in global connection creation

* reuse new function

* added init file

* safe access to optional keys

* removed unnecessary condition

* modified print messages on issues with bundles

* Changed message in missing bundle window

* updated ayon_api to 0.3.2
This commit is contained in:
Jakub Trllo 2023-07-05 17:14:22 +02:00 • committed by Jakub Trllo
parent 301375c401
commit 2791aa84bb
25 changed files with 1741 additions and 917 deletions
Download patch file Download diff file Expand all files Collapse all files

139
ayon_start.py
View file

@ -9,6 +9,7 @@ import site
import traceback
import contextlib
# Enabled logging debug mode when "--debug" is passed
if "--verbose" in sys.argv:
expected_values = (
@ -48,35 +49,55 @@ if "--verbose" in sys.argv:
))
os.environ["OPENPYPE_LOG_LEVEL"] = str(log_level)
os.environ["AYON_LOG_LEVEL"] = str(log_level)
# Enable debug mode, may affect log level if log level is not defined
if "--debug" in sys.argv:
sys.argv.remove("--debug")
os.environ["AYON_DEBUG"] = "1"
os.environ["OPENPYPE_DEBUG"] = "1"
if "--automatic-tests" in sys.argv:
sys.argv.remove("--automatic-tests")
os.environ["IS_TEST"] = "1"
SKIP_HEADERS = False
if "--skip-headers" in sys.argv:
sys.argv.remove("--skip-headers")
SKIP_HEADERS = True
SKIP_BOOTSTRAP = False
if "--skip-bootstrap" in sys.argv:
sys.argv.remove("--skip-bootstrap")
SKIP_BOOTSTRAP = True
if "--use-staging" in sys.argv:
sys.argv.remove("--use-staging")
os.environ["AYON_USE_STAGING"] = "1"
os.environ["OPENPYPE_USE_STAGING"] = "1"
_silent_commands = {
"run",
"standalonepublisher",
"extractenvironments",
"version"
}
if "--headless" in sys.argv:
os.environ["AYON_HEADLESS_MODE"] = "1"
os.environ["OPENPYPE_HEADLESS_MODE"] = "1"
sys.argv.remove("--headless")
elif os.getenv("OPENPYPE_HEADLESS_MODE") != "1":
elif (
os.getenv("AYON_HEADLESS_MODE") != "1"
or os.getenv("OPENPYPE_HEADLESS_MODE") != "1"
):
os.environ.pop("AYON_HEADLESS_MODE", None)
os.environ.pop("OPENPYPE_HEADLESS_MODE", None)
elif (
os.getenv("AYON_HEADLESS_MODE")
!= os.getenv("OPENPYPE_HEADLESS_MODE")
):
os.environ["OPENPYPE_HEADLESS_MODE"] = (
os.environ["AYON_HEADLESS_MODE"]
)
IS_BUILT_APPLICATION = getattr(sys, "frozen", False)
HEADLESS_MODE_ENABLED = os.environ.get("OPENPYPE_HEADLESS_MODE") == "1"
SILENT_MODE_ENABLED = any(arg in _silent_commands for arg in sys.argv)
HEADLESS_MODE_ENABLED = os.getenv("AYON_HEADLESS_MODE") == "1"
_pythonpath = os.getenv("PYTHONPATH", "")
_python_paths = _pythonpath.split(os.pathsep)
@ -129,10 +150,12 @@ os.environ["PYTHONPATH"] = os.pathsep.join(_python_paths)
os.environ["USE_AYON_SERVER"] = "1"
# Set this to point either to `python` from venv in case of live code
# or to `ayon` or `ayon_console` in case of frozen code
os.environ["AYON_EXECUTABLE"] = sys.executable
os.environ["OPENPYPE_EXECUTABLE"] = sys.executable
os.environ["AYON_ROOT"] = AYON_ROOT
os.environ["OPENPYPE_ROOT"] = AYON_ROOT
os.environ["OPENPYPE_REPOS_ROOT"] = AYON_ROOT
os.environ["AYON_MENU_LABEL"] = "AYON"
os.environ["AVALON_LABEL"] = "AYON"
# Set name of pyblish UI import
os.environ["PYBLISH_GUI"] = "pyblish_pype"
@ -153,9 +176,7 @@ if sys.__stdout__:
term = blessed.Terminal()
def _print(message: str):
if SILENT_MODE_ENABLED:
pass
elif message.startswith("!!! "):
if message.startswith("!!! "):
print(f'{term.orangered2("!!! ")}{message[4:]}')
elif message.startswith(">>> "):
print(f'{term.aquamarine3(">>> ")}{message[4:]}')
@ -179,8 +200,7 @@ if sys.__stdout__:
print(message)
else:
def _print(message: str):
if not SILENT_MODE_ENABLED:
print(message)
print(message)
# if SSL_CERT_FILE is not set prior to OpenPype launch, we set it to point
@ -190,7 +210,9 @@ if not os.getenv("SSL_CERT_FILE"):
elif os.getenv("SSL_CERT_FILE") != certifi.where():
_print("--- your system is set to use custom CA certificate bundle.")
from ayon_api import get_base_url
from ayon_api.constants import SERVER_URL_ENV_KEY, SERVER_API_ENV_KEY
from ayon_common import is_staging_enabled
from ayon_common.connection.credentials import (
ask_to_login_ui,
add_server,
@ -200,7 +222,11 @@ from ayon_common.connection.credentials import (
create_global_connection,
confirm_server_login,
)
from ayon_common.distribution.addon_distribution import AyonDistribution
from ayon_common.distribution import (
AyonDistribution,
BundleNotFoundError,
show_missing_bundle_information,
)
def set_global_environments() -> None:
@ -286,8 +312,44 @@ def _check_and_update_from_ayon_server():
"""
distribution = AyonDistribution()
bundle = None
bundle_name = None
try:
bundle = distribution.bundle_to_use
if bundle is not None:
bundle_name = bundle.name
except BundleNotFoundError as exc:
bundle_name = exc.bundle_name
if bundle is None:
url = get_base_url()
if not HEADLESS_MODE_ENABLED:
show_missing_bundle_information(url, bundle_name)
elif bundle_name:
_print((
f"!!! Requested release bundle '{bundle_name}'"
" is not available on server."
))
_print(
"!!! Check if selected release bundle"
f" is available on the server '{url}'."
)
else:
mode = "staging" if is_staging_enabled() else "production"
_print(
f"!!! No release bundle is set as {mode} on the AYON server."
)
_print(
"!!! Make sure there is a release bundle set"
f" as \"{mode}\" on the AYON server '{url}'."
)
sys.exit(1)
distribution.distribute()
distribution.validate_distribution()
os.environ["AYON_BUNDLE_NAME"] = bundle_name
python_paths = [
path
@ -311,8 +373,6 @@ def boot():
os.environ["OPENPYPE_VERSION"] = __version__
os.environ["AYON_VERSION"] = __version__
use_staging = os.environ.get("OPENPYPE_USE_STAGING") == "1"
_connect_to_ayon_server()
_check_and_update_from_ayon_server()
@ -328,7 +388,10 @@ def boot():
with contextlib.suppress(AttributeError, KeyError):
del sys.modules[module_name]
def main_cli():
from openpype import cli
from openpype.version import __version__
from openpype.lib import terminal as t
_print(">>> loading environments ...")
@ -338,8 +401,8 @@ def boot():
set_addons_environments()
# print info when not running scripts defined in 'silent commands'
if not SILENT_MODE_ENABLED:
info = get_info(use_staging)
if not SKIP_HEADERS:
info = get_info(is_staging_enabled())
info.insert(0, f">>> Using AYON from [ {AYON_ROOT} ]")
t_width = 20
@ -361,6 +424,29 @@ def boot():
sys.exit(1)
def script_cli():
"""Run and execute script."""
filepath = os.path.abspath(sys.argv[1])
# Find '__main__.py' in directory
if os.path.isdir(filepath):
new_filepath = os.path.join(filepath, "__main__.py")
if not os.path.exists(new_filepath):
raise RuntimeError(
f"can't find '__main__' module in '{filepath}'")
filepath = new_filepath
# Add parent dir to sys path
sys.path.insert(0, os.path.dirname(filepath))
# Read content and execute
with open(filepath, "r") as stream:
content = stream.read()
exec(compile(content, filepath, "exec"), globals())
def get_info(use_staging=None) -> list:
"""Print additional information to console."""
@ -369,6 +455,7 @@ def get_info(use_staging=None) -> list:
inf.append(("AYON variant", "staging"))
else:
inf.append(("AYON variant", "production"))
inf.append(("AYON bundle", os.getenv("AYON_BUNDLE")))
# NOTE add addons information
@ -380,5 +467,17 @@ def get_info(use_staging=None) -> list:
return formatted
def main():
if not SKIP_BOOTSTRAP:
boot()
args = list(sys.argv)
args.pop(0)
if args and os.path.exists(args[0]):
script_cli()
else:
main_cli()
if __name__ == "__main__":
boot()
main()

16
common/ayon_common/__init__.py Normal file
View file

@ -0,0 +1,16 @@
from .utils import (
IS_BUILT_APPLICATION,
is_staging_enabled,
get_local_site_id,
get_ayon_appdirs,
get_ayon_launch_args,
)
__all__ = (
"IS_BUILT_APPLICATION",
"is_staging_enabled",
"get_local_site_id",
"get_ayon_appdirs",
"get_ayon_launch_args",
)

49
common/ayon_common/connection/credentials.py
View file

@ -13,6 +13,8 @@ import json
import platform
import datetime
import contextlib
import subprocess
import tempfile
from typing import Optional, Union, Any
import ayon_api
@ -25,7 +27,12 @@ from ayon_api.utils import (
logout_from_server,
)
from ayon_common.utils import get_ayon_appdirs, get_local_site_id
from ayon_common.utils import (
get_ayon_appdirs,
get_local_site_id,
get_ayon_launch_args,
is_staging_enabled,
)
class ChangeUserResult:
@ -258,6 +265,8 @@ def ask_to_login_ui(
credentials are invalid. To change credentials use 'change_user_ui'
function.
Use a subprocess to show UI.
Args:
url (Optional[str]): Server url that could be prefilled in UI.
always_on_top (Optional[bool]): Window will be drawn on top of
@ -267,12 +276,33 @@ def ask_to_login_ui(
tuple[str, str, str]: Url, user's token and username.
"""
from .ui import ask_to_login
current_dir = os.path.dirname(os.path.abspath(__file__))
ui_dir = os.path.join(current_dir, "ui")
if url is None:
url = get_last_server()
username = get_last_username_by_url(url)
return ask_to_login(url, username, always_on_top=always_on_top)
data = {
"url": url,
"username": username,
"always_on_top": always_on_top,
}
with tempfile.TemporaryFile(
mode="w", prefix="ayon_login", suffix=".json", delete=False
) as tmp:
output = tmp.name
json.dump(data, tmp)
code = subprocess.call(
get_ayon_launch_args(ui_dir, "--skip-bootstrap", output))
if code != 0:
raise RuntimeError("Failed to show login UI")
with open(output, "r") as stream:
data = json.load(stream)
os.remove(output)
return data["output"]
def change_user_ui() -> ChangeUserResult:
@ -421,15 +451,18 @@ def set_environments(url: str, token: str):
def create_global_connection():
"""Create global connection with site id and client version.
Make sure the global connection in 'ayon_api' have entered site id and
client version.
Set default settings variant to use based on 'is_staging_enabled'.
"""
if hasattr(ayon_api, "create_connection"):
ayon_api.create_connection(
get_local_site_id(), os.environ.get("AYON_VERSION")
)
ayon_api.create_connection(
get_local_site_id(), os.environ.get("AYON_VERSION")
)
ayon_api.set_default_settings_variant(
"staging" if is_staging_enabled() else "production"
)
def need_server_or_login() -> bool:

23
common/ayon_common/connection/ui/__main__.py Normal file
View file

@ -0,0 +1,23 @@
import sys
import json
from ayon_common.connection.ui.login_window import ask_to_login
def main(output_path):
with open(output_path, "r") as stream:
data = json.load(stream)
url = data.get("url")
username = data.get("username")
always_on_top = data.get("always_on_top", False)
out_url, out_token, out_username = ask_to_login(
url, username, always_on_top=always_on_top)
data["output"] = [out_url, out_token, out_username]
with open(output_path, "w") as stream:
json.dump(data, stream)
if __name__ == "__main__":
main(sys.argv[-1])

25
common/ayon_common/connection/ui/login_window.py
View file

@ -10,12 +10,12 @@ from ayon_common.resources import (
get_icon_path,
load_stylesheet,
)
from ayon_common.ui_utils import set_style_property, get_qt_app
from .widgets import (
PressHoverButton,
PlaceholderLineEdit,
)
from .lib import set_style_property
class LogoutConfirmDialog(QtWidgets.QDialog):
@ -650,16 +650,7 @@ def ask_to_login(url=None, username=None, always_on_top=False):
be changed during dialog lifetime that's why the url is returned.
"""
app_instance = QtWidgets.QApplication.instance()
if app_instance is None:
for attr_name in (
"AA_EnableHighDpiScaling",
"AA_UseHighDpiPixmaps"
):
attr = getattr(QtCore.Qt, attr_name, None)
if attr is not None:
QtWidgets.QApplication.setAttribute(attr)
app_instance = QtWidgets.QApplication([])
app_instance = get_qt_app()
window = ServerLoginWindow()
if always_on_top:
@ -701,17 +692,7 @@ def change_user(url, username, api_key, always_on_top=False):
during dialog lifetime that's why the url is returned.
"""
app_instance = QtWidgets.QApplication.instance()
if app_instance is None:
for attr_name in (
"AA_EnableHighDpiScaling",
"AA_UseHighDpiPixmaps"
):
attr = getattr(QtCore.Qt, attr_name, None)
if attr is not None:
QtWidgets.QApplication.setAttribute(attr)
app_instance = QtWidgets.QApplication([])
app_instance = get_qt_app()
window = ServerLoginWindow()
if always_on_top:
window.setWindowFlags(

4
common/ayon_common/distribution/README.md
View file

@ -5,7 +5,7 @@ Code in this folder is backend portion of Addon distribution logic for v4 server
Each host, module will be separate Addon in the future. Each v4 server could run different set of Addons.
Client (running on artist machine) will in the first step ask v4 for list of enabled addons.
Client (running on artist machine) will in the first step ask v4 for list of enabled addons.
(It expects list of json documents matching to `addon_distribution.py:AddonInfo` object.)
Next it will compare presence of enabled addon version in local folder. In the case of missing version of
an addon, client will use information in the addon to download (from http/shared local disk/git) zip file
@ -15,4 +15,4 @@ Required part of addon distribution will be sharing of dependencies (python libr
Location of this folder might change in the future as it will be required for a clint to add this folder to sys.path reliably.
This code needs to be independent on Openpype code as much as possible!
This code needs to be independent on Openpype code as much as possible!

9
common/ayon_common/distribution/__init__.py
View file

@ -0,0 +1,9 @@
from .control import AyonDistribution, BundleNotFoundError
from .utils import show_missing_bundle_information
__all__ = (
"AyonDistribution",
"BundleNotFoundError",
"show_missing_bundle_information",
)

213
common/ayon_common/distribution/addon_info.py
View file

@ -1,213 +0,0 @@
import attr
from enum import Enum
class UrlType(Enum):
HTTP = "http"
GIT = "git"
FILESYSTEM = "filesystem"
SERVER = "server"
@attr.s
class MultiPlatformPath(object):
windows = attr.ib(default=None)
linux = attr.ib(default=None)
darwin = attr.ib(default=None)
@attr.s
class SourceInfo(object):
type = attr.ib()
@attr.s
class LocalSourceInfo(SourceInfo):
path = attr.ib(default=attr.Factory(MultiPlatformPath))
@attr.s
class WebSourceInfo(SourceInfo):
url = attr.ib(default=None)
headers = attr.ib(default=None)
filename = attr.ib(default=None)
@attr.s
class ServerSourceInfo(SourceInfo):
filename = attr.ib(default=None)
path = attr.ib(default=None)
def convert_source(source):
"""Create source object from data information.
Args:
source (Dict[str, any]): Information about source.
Returns:
Union[None, SourceInfo]: Object with source information if type is
known.
"""
source_type = source.get("type")
if not source_type:
return None
if source_type == UrlType.FILESYSTEM.value:
return LocalSourceInfo(
type=source_type,
path=source["path"]
)
if source_type == UrlType.HTTP.value:
url = source["path"]
return WebSourceInfo(
type=source_type,
url=url,
headers=source.get("headers"),
filename=source.get("filename")
)
if source_type == UrlType.SERVER.value:
return ServerSourceInfo(
type=source_type,
filename=source.get("filename"),
path=source.get("path")
)
@attr.s
class VersionData(object):
version_data = attr.ib(default=None)
@attr.s
class AddonInfo(object):
"""Object matching json payload from Server"""
name = attr.ib()
version = attr.ib()
full_name = attr.ib()
title = attr.ib(default=None)
require_distribution = attr.ib(default=False)
sources = attr.ib(default=attr.Factory(list))
unknown_sources = attr.ib(default=attr.Factory(list))
hash = attr.ib(default=None)
description = attr.ib(default=None)
license = attr.ib(default=None)
authors = attr.ib(default=None)
@classmethod
def from_dict_by_version(cls, data, addon_version):
"""Addon info for specific version.
Args:
data (dict[str, Any]): Addon information from server. Should
contain information about every version under 'versions'.
addon_version (str): Addon version for which is info requested.
Returns:
Union[AddonInfo, None]: Addon info, or None if version is not
available.
"""
if not addon_version:
return None
# server payload contains info about all versions
version_data = data.get("versions", {}).get(addon_version)
if not version_data:
return None
source_info = version_data.get("clientSourceInfo")
require_distribution = source_info is not None
sources = []
unknown_sources = []
for source in (source_info or []):
addon_source = convert_source(source)
if addon_source is not None:
sources.append(addon_source)
else:
unknown_sources.append(source)
print(f"Unknown source {source.get('type')}")
full_name = "{}_{}".format(data["name"], addon_version)
return cls(
name=data.get("name"),
version=addon_version,
full_name=full_name,
require_distribution=require_distribution,
sources=sources,
unknown_sources=unknown_sources,
hash=data.get("hash"),
description=data.get("description"),
title=data.get("title"),
license=data.get("license"),
authors=data.get("authors")
)
@classmethod
def from_dict(cls, data, use_staging=False):
"""Get Addon information for production or staging version.
Args:
data (dict[str, Any]): Addon information from server. Should
contain information about every version under 'versions'.
use_staging (bool): Use staging version if set to 'True' instead
of production.
Returns:
Union[AddonInfo, None]: Addon info, or None if version is not
set or available.
"""
# Active addon must have 'productionVersion' or 'stagingVersion'
# and matching version info.
if use_staging:
addon_version = data.get("stagingVersion")
else:
addon_version = data.get("productionVersion")
return cls.from_dict_by_version(data, addon_version)
@attr.s
class DependencyItem(object):
"""Object matching payload from Server about single dependency package"""
name = attr.ib()
platform = attr.ib()
checksum = attr.ib()
require_distribution = attr.ib()
sources = attr.ib(default=attr.Factory(list))
unknown_sources = attr.ib(default=attr.Factory(list))
addon_list = attr.ib(default=attr.Factory(list))
python_modules = attr.ib(default=attr.Factory(dict))
@classmethod
def from_dict(cls, package):
sources = []
unknown_sources = []
package_sources = package.get("sources")
require_distribution = package_sources is not None
for source in (package_sources or []):
dependency_source = convert_source(source)
if dependency_source is not None:
sources.append(dependency_source)
else:
print(f"Unknown source {source.get('type')}")
unknown_sources.append(source)
addon_list = [f"{name}_{version}"
for name, version in
package.get("supportedAddons").items()]
return cls(
name=package.get("name"),
platform=package.get("platform"),
require_distribution=require_distribution,
sources=sources,
unknown_sources=unknown_sources,
checksum=package.get("checksum"),
addon_list=addon_list,
python_modules=package.get("pythonModules")
)

752
common/ayon_common/distribution/addon_distribution.py → common/ayon_common/distribution/control.py
View file

@ -4,24 +4,43 @@ import json
import traceback
import collections
import datetime
from enum import Enum
from abc import abstractmethod
import attr
import logging
import platform
import shutil
import threading
from abc import ABCMeta
import platform
import attr
from enum import Enum
import ayon_api
from ayon_common.utils import get_ayon_appdirs
from .file_handler import RemoteFileHandler
from .addon_info import (
AddonInfo,
UrlType,
DependencyItem,
from ayon_common.utils import is_staging_enabled
from .utils import (
get_addons_dir,
get_dependencies_dir,
)
from .downloaders import get_default_download_factory
from .data_structures import (
AddonInfo,
DependencyItem,
Bundle,
)
NOT_SET = type("UNKNOWN", (), {"__bool__": lambda: False})()
class BundleNotFoundError(Exception):
"""Bundle name is defined but is not available on server.
Args:
bundle_name (str): Name of bundle that was not found.
"""
def __init__(self, bundle_name):
self.bundle_name = bundle_name
super().__init__(
f"Bundle '{bundle_name}' is not available on server"
)
class UpdateState(Enum):
@ -32,326 +51,6 @@ class UpdateState(Enum):
MISS_SOURCE_FILES = "miss_source_files"
def get_local_dir(*subdirs):
"""Get product directory in user's home directory.
Each user on machine have own local directory where are downloaded updates,
addons etc.
Returns:
str: Path to product local directory.
"""
if not subdirs:
raise ValueError("Must fill dir_name if nothing else provided!")
local_dir = get_ayon_appdirs(*subdirs)
if not os.path.isdir(local_dir):
try:
os.makedirs(local_dir)
except Exception: # TODO fix exception
raise RuntimeError(f"Cannot create {local_dir}")
return local_dir
def get_addons_dir():
"""Directory where addon packages are stored.
Path to addons is defined using python module 'appdirs' which
The path is stored into environment variable 'AYON_ADDONS_DIR'.
Value of environment variable can be overriden, but we highly recommended
to use that option only for development purposes.
Returns:
str: Path to directory where addons should be downloaded.
"""
addons_dir = os.environ.get("AYON_ADDONS_DIR")
if not addons_dir:
addons_dir = get_local_dir("addons")
os.environ["AYON_ADDONS_DIR"] = addons_dir
return addons_dir
def get_dependencies_dir():
"""Directory where dependency packages are stored.
Path to addons is defined using python module 'appdirs' which
The path is stored into environment variable 'AYON_DEPENDENCIES_DIR'.
Value of environment variable can be overriden, but we highly recommended
to use that option only for development purposes.
Returns:
str: Path to directory where dependency packages should be downloaded.
"""
dependencies_dir = os.environ.get("AYON_DEPENDENCIES_DIR")
if not dependencies_dir:
dependencies_dir = get_local_dir("dependency_packages")
os.environ["AYON_DEPENDENCIES_DIR"] = dependencies_dir
return dependencies_dir
class SourceDownloader(metaclass=ABCMeta):
log = logging.getLogger(__name__)
@classmethod
@abstractmethod
def download(cls, source, destination_dir, data, transfer_progress):
"""Returns url to downloaded addon zip file.
Tranfer progress can be ignored, in that case file transfer won't
be shown as 0-100% but as 'running'. First step should be to set
destination content size and then add transferred chunk sizes.
Args:
source (dict): {type:"http", "url":"https://} ...}
destination_dir (str): local folder to unzip
data (dict): More information about download content. Always have
'type' key in.
transfer_progress (ayon_api.TransferProgress): Progress of
transferred (copy/download) content.
Returns:
(str) local path to addon zip file
"""
pass
@classmethod
@abstractmethod
def cleanup(cls, source, destination_dir, data):
"""Cleanup files when distribution finishes or crashes.
Cleanup e.g. temporary files (downloaded zip) or other related stuff
to downloader.
"""
pass
@classmethod
def check_hash(cls, addon_path, addon_hash, hash_type="sha256"):
"""Compares 'hash' of downloaded 'addon_url' file.
Args:
addon_path (str): Local path to addon file.
addon_hash (str): Hash of downloaded file.
hash_type (str): Type of hash.
Raises:
ValueError if hashes doesn't match
"""
if not os.path.exists(addon_path):
raise ValueError(f"{addon_path} doesn't exist.")
if not RemoteFileHandler.check_integrity(addon_path,
addon_hash,
hash_type=hash_type):
raise ValueError(f"{addon_path} doesn't match expected hash.")
@classmethod
def unzip(cls, addon_zip_path, destination_dir):
"""Unzips local 'addon_zip_path' to 'destination'.
Args:
addon_zip_path (str): local path to addon zip file
destination_dir (str): local folder to unzip
"""
RemoteFileHandler.unzip(addon_zip_path, destination_dir)
os.remove(addon_zip_path)
class DownloadFactory:
def __init__(self):
self._downloaders = {}
def register_format(self, downloader_type, downloader):
"""Register downloader for download type.
Args:
downloader_type (UrlType): Type of source.
downloader (SourceDownloader): Downloader which cares about
download, hash check and unzipping.
"""
self._downloaders[downloader_type.value] = downloader
def get_downloader(self, downloader_type):
"""Registered downloader for type.
Args:
downloader_type (UrlType): Type of source.
Returns:
SourceDownloader: Downloader object which should care about file
distribution.
Raises:
ValueError: If type does not have registered downloader.
"""
if downloader := self._downloaders.get(downloader_type):
return downloader()
raise ValueError(f"{downloader_type} not implemented")
class OSDownloader(SourceDownloader):
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
# OS doesn't need to download, unzip directly
addon_url = source["path"].get(platform.system().lower())
if not os.path.exists(addon_url):
raise ValueError(f"{addon_url} is not accessible")
return addon_url
@classmethod
def cleanup(cls, source, destination_dir, data):
# Nothing to do - download does not copy anything
pass
class HTTPDownloader(SourceDownloader):
CHUNK_SIZE = 100000
@staticmethod
def get_filename(source):
source_url = source["url"]
filename = source.get("filename")
if not filename:
filename = os.path.basename(source_url)
basename, ext = os.path.splitext(filename)
allowed_exts = set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)
if ext.replace(".", "") not in allowed_exts:
filename = f"{basename}.zip"
return filename
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
source_url = source["url"]
cls.log.debug(f"Downloading {source_url} to {destination_dir}")
headers = source.get("headers")
filename = cls.get_filename(source)
# TODO use transfer progress
RemoteFileHandler.download_url(
source_url,
destination_dir,
filename,
headers=headers
)
return os.path.join(destination_dir, filename)
@classmethod
def cleanup(cls, source, destination_dir, data):
# Nothing to do - download does not copy anything
filename = cls.get_filename(source)
filepath = os.path.join(destination_dir, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.remove(filepath)
class AyonServerDownloader(SourceDownloader):
"""Downloads static resource file from v4 Server.
Expects filled env var AYON_SERVER_URL.
"""
CHUNK_SIZE = 8192
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
path = source["path"]
filename = source["filename"]
if path and not filename:
filename = path.split("/")[-1]
cls.log.debug(f"Downloading {filename} to {destination_dir}")
_, ext = os.path.splitext(filename)
clear_ext = ext.lower().replace(".", "")
valid_exts = set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)
if clear_ext not in valid_exts:
raise ValueError(
"Invalid file extension \"{}\". Expected {}".format(
clear_ext, ", ".join(valid_exts)
))
if path:
filepath = os.path.join(destination_dir, filename)
return ayon_api.download_file(
path,
filepath,
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
# dst_filepath = os.path.join(destination_dir, filename)
if data["type"] == "dependency_package":
return ayon_api.download_dependency_package(
data["name"],
destination_dir,
filename,
platform_name=data["platform"],
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
if data["type"] == "addon":
return ayon_api.download_addon_private_file(
data["name"],
data["version"],
filename,
destination_dir,
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
raise ValueError(f"Unknown type to download \"{data['type']}\"")
@classmethod
def cleanup(cls, source, destination_dir, data):
# Nothing to do - download does not copy anything
filename = source["filename"]
filepath = os.path.join(destination_dir, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.remove(filepath)
def get_dependency_package(package_name=None):
"""Returns info about currently used dependency package.
Dependency package means .venv created from all activated addons from the
server (plus libraries for core Tray app TODO confirm).
This package needs to be downloaded, unpacked and added to sys.path for
Tray app to work.
Args:
package_name (str): Name of package. Production package name is used
if not entered.
Returns:
Union[DependencyItem, None]: Item or None if package with the name was
not found.
"""
dependencies_info = ayon_api.get_dependencies_info()
dependency_list = dependencies_info["packages"]
# Use production package if package is not specified
if package_name is None:
package_name = dependencies_info["productionPackage"]
for dependency in dependency_list:
dependency_package = DependencyItem.from_dict(dependency)
if dependency_package.name == package_name:
return dependency_package
class DistributeTransferProgress:
"""Progress of single source item in 'DistributionItem'.
@ -612,7 +311,8 @@ class DistributionItem:
try:
downloader = self.factory.get_downloader(source.type)
except Exception:
source_progress.set_failed(f"Unknown downloader {source.type}")
message = f"Unknown downloader {source.type}"
source_progress.set_failed(message)
self.log.warning(message, exc_info=True)
continue
@ -733,12 +433,18 @@ class AyonDistribution:
dependency_dirpath (Optional[str]): Where dependencies will be stored.
dist_factory (Optional[DownloadFactory]): Factory which cares about
downloading of items based on source type.
addons_info (Optional[List[AddonInfo]]): List of prepared addons' info.
dependency_package_info (Optional[Union[Dict[str, Any], None]]): Info
about package from server. Defaults to '-1'.
addons_info (Optional[list[dict[str, Any]]): List of prepared
addons' info.
dependency_packages_info (Optional[list[dict[str, Any]]): Info
about packages from server.
bundles_info (Optional[Dict[str, Any]]): Info about
bundles.
bundle_name (Optional[str]): Name of bundle to use. If not passed
an environment variable 'AYON_BUNDLE_NAME' is checked for value.
When both are not available the bundle is defined by 'use_staging'
value.
use_staging (Optional[bool]): Use staging versions of an addon.
If not passed, an environment variable 'OPENPYPE_USE_STAGING' is
checked for value '1'.
If not passed, 'is_staging_enabled' is used as default value.
"""
def __init__(
@ -746,96 +452,304 @@ class AyonDistribution:
addon_dirpath=None,
dependency_dirpath=None,
dist_factory=None,
addons_info=None,
dependency_package_info=-1,
addons_info=NOT_SET,
dependency_packages_info=NOT_SET,
bundles_info=NOT_SET,
bundle_name=NOT_SET,
use_staging=None
):
self._log = None
self._dist_started = False
self._dist_finished = False
self._addons_dirpath = addon_dirpath or get_addons_dir()
self._dependency_dirpath = dependency_dirpath or get_dependencies_dir()
self._dist_factory = (
dist_factory or get_default_download_factory()
)
if isinstance(addons_info, list):
addons_info = {item.full_name: item for item in addons_info}
self._dist_started = False
self._dist_finished = False
self._log = None
if bundle_name is NOT_SET:
bundle_name = os.environ.get("AYON_BUNDLE_NAME", NOT_SET)
# Raw addons data from server
self._addons_info = addons_info
self._addons_dist_items = None
self._dependency_package = dependency_package_info
self._dependency_dist_item = -1
# Prepared data as Addon objects
self._addon_items = NOT_SET
# Distrubtion items of addons
# - only those addons and versions that should be distributed
self._addon_dist_items = NOT_SET
# Raw dependency packages data from server
self._dependency_packages_info = dependency_packages_info
# Prepared dependency packages as objects
self._dependency_packages_items = NOT_SET
# Dependency package item that should be used
self._dependency_package_item = NOT_SET
# Distribution item of dependency package
self._dependency_dist_item = NOT_SET
# Raw bundles data from server
self._bundles_info = bundles_info
# Bundles as objects
self._bundle_items = NOT_SET
# Bundle that should be used in production
self._production_bundle = NOT_SET
# Bundle that should be used in staging
self._staging_bundle = NOT_SET
# Boolean that defines if staging bundle should be used
self._use_staging = use_staging
# Specific bundle name should be used
self._bundle_name = bundle_name
# Final bundle that will be used
self._bundle = NOT_SET
@property
def use_staging(self):
"""Staging version of a bundle should be used.
This value is completely ignored if specific bundle name should
be used.
Returns:
bool: True if staging version should be used.
"""
if self._use_staging is None:
self._use_staging = os.getenv("OPENPYPE_USE_STAGING") == "1"
self._use_staging = is_staging_enabled()
return self._use_staging
@property
def log(self):
"""Helper to access logger.
Returns:
logging.Logger: Logger instance.
"""
if self._log is None:
self._log = logging.getLogger(self.__class__.__name__)
return self._log
@property
def bundles_info(self):
"""
Returns:
dict[str, dict[str, Any]]: Bundles information from server.
"""
if self._bundles_info is NOT_SET:
self._bundles_info = ayon_api.get_bundles()
return self._bundles_info
@property
def bundle_items(self):
"""
Returns:
list[Bundle]: List of bundles info.
"""
if self._bundle_items is NOT_SET:
self._bundle_items = [
Bundle.from_dict(info)
for info in self.bundles_info["bundles"]
]
return self._bundle_items
def _prepare_production_staging_bundles(self):
production_bundle = None
staging_bundle = None
for bundle in self.bundle_items:
if bundle.is_production:
production_bundle = bundle
if bundle.is_staging:
staging_bundle = bundle
self._production_bundle = production_bundle
self._staging_bundle = staging_bundle
@property
def production_bundle(self):
"""
Returns:
Union[Bundle, None]: Bundle that should be used in production.
"""
if self._production_bundle is NOT_SET:
self._prepare_production_staging_bundles()
return self._production_bundle
@property
def staging_bundle(self):
"""
Returns:
Union[Bundle, None]: Bundle that should be used in staging.
"""
if self._staging_bundle is NOT_SET:
self._prepare_production_staging_bundles()
return self._staging_bundle
@property
def bundle_to_use(self):
"""Bundle that will be used for distribution.
Bundle that should be used can be affected by 'bundle_name'
or 'use_staging'.
Returns:
Union[Bundle, None]: Bundle that will be used for distribution
or None.
Raises:
BundleNotFoundError: When bundle name to use is defined
but is not available on server.
"""
if self._bundle is NOT_SET:
if self._bundle_name is not NOT_SET:
bundle = next(
(
bundle
for bundle in self.bundle_items
if bundle.name == self._bundle_name
),
None
)
if bundle is None:
raise BundleNotFoundError(self._bundle_name)
self._bundle = bundle
elif self.use_staging:
self._bundle = self.staging_bundle
else:
self._bundle = self.production_bundle
return self._bundle
@property
def bundle_name_to_use(self):
bundle = self.bundle_to_use
return None if bundle is None else bundle.name
@property
def addons_info(self):
"""Server information about available addons.
Returns:
Dict[str, dict[str, Any]: Addon info by addon name.
"""
if self._addons_info is NOT_SET:
server_info = ayon_api.get_addons_info(details=True)
self._addons_info = server_info["addons"]
return self._addons_info
@property
def addon_items(self):
"""Information about available addons on server.
Addons may require distribution of files. For those addons will be
created 'DistributionItem' handling distribution itself.
Todos:
Add support for staging versions. Right now is supported only
production version.
Returns:
Dict[str, AddonInfo]: Addon info by full name.
Dict[str, AddonInfo]: Addon info object by addon name.
"""
if self._addons_info is None:
if self._addon_items is NOT_SET:
addons_info = {}
server_addons_info = ayon_api.get_addons_info(details=True)
for addon in server_addons_info["addons"]:
addon_info = AddonInfo.from_dict(addon, self.use_staging)
if addon_info is None:
continue
addons_info[addon_info.full_name] = addon_info
self._addons_info = addons_info
return self._addons_info
for addon in self.addons_info:
addon_info = AddonInfo.from_dict(addon)
addons_info[addon_info.name] = addon_info
self._addon_items = addons_info
return self._addon_items
@property
def dependency_package(self):
"""Information about dependency package from server.
Receive and cache dependency package information from server.
def dependency_packages_info(self):
"""Server information about available dependency packages.
Notes:
For testing purposes it is possible to pass dependency package
For testing purposes it is possible to pass dependency packages
information to '__init__'.
Returns:
Union[None, Dict[str, Any]]: None if server does not have specified
dependency package.
list[dict[str, Any]]: Dependency packages information.
"""
if self._dependency_package == -1:
self._dependency_package = get_dependency_package()
return self._dependency_package
if self._dependency_packages_info is NOT_SET:
self._dependency_packages_info = (
ayon_api.get_dependency_packages())["packages"]
return self._dependency_packages_info
def _prepare_current_addons_dist_items(self):
@property
def dependency_packages_items(self):
"""Dependency packages as objects.
Returns:
dict[str, DependencyItem]: Dependency packages as objects by name.
"""
if self._dependency_packages_items is NOT_SET:
dependenc_package_items = {}
for item in self.dependency_packages_info:
item = DependencyItem.from_dict(item)
dependenc_package_items[item.name] = item
self._dependency_packages_items = dependenc_package_items
return self._dependency_packages_items
@property
def dependency_package_item(self):
"""Dependency package item that should be used by bundle.
Returns:
Union[None, Dict[str, Any]]: None if bundle does not have
specified dependency package.
"""
if self._dependency_package_item is NOT_SET:
dependency_package_item = None
bundle = self.bundle_to_use
if bundle is not None:
package_name = bundle.dependency_packages.get(
platform.system().lower()
)
dependency_package_item = self.dependency_packages_items.get(
package_name)
self._dependency_package_item = dependency_package_item
return self._dependency_package_item
def _prepare_current_addon_dist_items(self):
addons_metadata = self.get_addons_metadata()
output = {}
for full_name, addon_info in self.addons_info.items():
if not addon_info.require_distribution:
output = []
addon_versions = {}
bundle = self.bundle_to_use
if bundle is not None:
addon_versions = bundle.addon_versions
for addon_name, addon_item in self.addon_items.items():
addon_version = addon_versions.get(addon_name)
# Addon is not in bundle -> Skip
if addon_version is None:
continue
addon_version_item = addon_item.versions.get(addon_version)
# Addon version is not available in addons info
# - TODO handle this case (raise error, skip, store, report, ...)
if addon_version_item is None:
print(
f"Version '{addon_version}' of addon '{addon_name}'"
" is not available on server."
)
continue
if not addon_version_item.require_distribution:
continue
full_name = addon_version_item.full_name
addon_dest = os.path.join(self._addons_dirpath, full_name)
self.log.debug(f"Checking {full_name} in {addon_dest}")
addon_in_metadata = (
addon_info.name in addons_metadata
and addon_info.version in addons_metadata[addon_info.name]
addon_name in addons_metadata
and addon_version_item.version in addons_metadata[addon_name]
)
if addon_in_metadata and os.path.isdir(addon_dest):
self.log.debug(
@ -848,25 +762,32 @@ class AyonDistribution:
downloader_data = {
"type": "addon",
"name": addon_info.name,
"version": addon_info.version
"name": addon_name,
"version": addon_version
}
output[full_name] = DistributionItem(
dist_item = DistributionItem(
state,
addon_dest,
addon_dest,
addon_info.hash,
addon_version_item.hash,
self._dist_factory,
list(addon_info.sources),
list(addon_version_item.sources),
downloader_data,
full_name,
self.log
)
output.append({
"dist_item": dist_item,
"addon_name": addon_name,
"addon_version": addon_version,
"addon_item": addon_item,
"addon_version_item": addon_version_item,
})
return output
def _prepare_dependency_progress(self):
package = self.dependency_package
package = self.dependency_package_item
if package is None or not package.require_distribution:
return None
@ -898,20 +819,34 @@ class AyonDistribution:
self.log,
)
def get_addons_dist_items(self):
def get_addon_dist_items(self):
"""Addon distribution items.
These items describe source files required by addon to be available on
machine. Each item may have 0-n source information from where can be
obtained. If file is already available it's state will be 'UPDATED'.
Example output:
[
{
"dist_item": DistributionItem,
"addon_name": str,
"addon_version": str,
"addon_item": AddonInfo,
"addon_version_item": AddonVersionInfo
}, {
...
}
]
Returns:
Dict[str, DistributionItem]: Distribution items by addon fullname.
list[dict[str, Any]]: Distribution items with addon version item.
"""
if self._addons_dist_items is None:
self._addons_dist_items = self._prepare_current_addons_dist_items()
return self._addons_dist_items
if self._addon_dist_items is NOT_SET:
self._addon_dist_items = (
self._prepare_current_addon_dist_items())
return self._addon_dist_items
def get_dependency_dist_item(self):
"""Dependency package distribution item.
@ -928,7 +863,7 @@ class AyonDistribution:
does not have specified any dependency package.
"""
if self._dependency_dist_item == -1:
if self._dependency_dist_item is NOT_SET:
self._dependency_dist_item = self._prepare_dependency_progress()
return self._dependency_dist_item
@ -1049,7 +984,8 @@ class AyonDistribution:
self.update_dependency_metadata(package.name, data)
addons_info = {}
for full_name, dist_item in self.get_addons_dist_items().items():
for item in self.get_addon_dist_items():
dist_item = item["dist_item"]
if (
not dist_item.need_distribution
or dist_item.state != UpdateState.UPDATED
@ -1059,10 +995,11 @@ class AyonDistribution:
source_data = dist_item.used_source
if not source_data:
continue
addon_info = self.addons_info[full_name]
if addon_info.name not in addons_info:
addons_info[addon_info.name] = {}
addons_info[addon_info.name][addon_info.version] = {
addon_name = item["addon_name"]
addon_version = item["addon_version"]
addons_info.setdefault(addon_name, {})
addons_info[addon_name][addon_version] = {
"source": source_data,
"file_hash": dist_item.file_hash,
"distributed_dt": stored_time
@ -1082,12 +1019,14 @@ class AyonDistribution:
List[DistributionItem]: Distribution items required by server.
"""
output = []
output = [
item["dist_item"]
for item in self.get_addon_dist_items()
]
dependency_dist_item = self.get_dependency_dist_item()
if dependency_dist_item is not None:
output.append(dependency_dist_item)
for dist_item in self.get_addons_dist_items().values():
output.append(dist_item)
output.insert(0, dependency_dist_item)
return output
def distribute(self, threaded=False):
@ -1136,9 +1075,10 @@ class AyonDistribution:
):
invalid.append("Dependency package")
for addon_name, dist_item in self.get_addons_dist_items().items():
for item in self.get_addon_dist_items():
dist_item = item["dist_item"]
if dist_item.state != UpdateState.UPDATED:
invalid.append(addon_name)
invalid.append(item["addon_name"])
if not invalid:
return
@ -1172,13 +1112,5 @@ class AyonDistribution:
return output
def get_default_download_factory():
download_factory = DownloadFactory()
download_factory.register_format(UrlType.FILESYSTEM, OSDownloader)
download_factory.register_format(UrlType.HTTP, HTTPDownloader)
download_factory.register_format(UrlType.SERVER, AyonServerDownloader)
return download_factory
def cli(*args):
raise NotImplementedError

261
common/ayon_common/distribution/data_structures.py Normal file
View file

@ -0,0 +1,261 @@
import attr
from enum import Enum
class UrlType(Enum):
HTTP = "http"
GIT = "git"
FILESYSTEM = "filesystem"
SERVER = "server"
@attr.s
class MultiPlatformValue(object):
windows = attr.ib(default=None)
linux = attr.ib(default=None)
darwin = attr.ib(default=None)
@attr.s
class SourceInfo(object):
type = attr.ib()
@attr.s
class LocalSourceInfo(SourceInfo):
path = attr.ib(default=attr.Factory(MultiPlatformValue))
@attr.s
class WebSourceInfo(SourceInfo):
url = attr.ib(default=None)
headers = attr.ib(default=None)
filename = attr.ib(default=None)
@attr.s
class ServerSourceInfo(SourceInfo):
filename = attr.ib(default=None)
path = attr.ib(default=None)
def convert_source(source):
"""Create source object from data information.
Args:
source (Dict[str, any]): Information about source.
Returns:
Union[None, SourceInfo]: Object with source information if type is
known.
"""
source_type = source.get("type")
if not source_type:
return None
if source_type == UrlType.FILESYSTEM.value:
return LocalSourceInfo(
type=source_type,
path=source["path"]
)
if source_type == UrlType.HTTP.value:
url = source["path"]
return WebSourceInfo(
type=source_type,
url=url,
headers=source.get("headers"),
filename=source.get("filename")
)
if source_type == UrlType.SERVER.value:
return ServerSourceInfo(
type=source_type,
filename=source.get("filename"),
path=source.get("path")
)
def prepare_sources(src_sources):
sources = []
unknown_sources = []
for source in (src_sources or []):
dependency_source = convert_source(source)
if dependency_source is not None:
sources.append(dependency_source)
else:
print(f"Unknown source {source.get('type')}")
unknown_sources.append(source)
return sources, unknown_sources
@attr.s
class VersionData(object):
version_data = attr.ib(default=None)
@attr.s
class AddonVersionInfo(object):
version = attr.ib()
full_name = attr.ib()
title = attr.ib(default=None)
require_distribution = attr.ib(default=False)
sources = attr.ib(default=attr.Factory(list))
unknown_sources = attr.ib(default=attr.Factory(list))
hash = attr.ib(default=None)
@classmethod
def from_dict(
cls, addon_name, addon_title, addon_version, version_data
):
"""Addon version info.
Args:
addon_name (str): Name of addon.
addon_title (str): Title of addon.
addon_version (str): Version of addon.
version_data (dict[str, Any]): Addon version information from
server.
Returns:
AddonVersionInfo: Addon version info.
"""
full_name = f"{addon_name}_{addon_version}"
title = f"{addon_title} {addon_version}"
source_info = version_data.get("clientSourceInfo")
require_distribution = source_info is not None
sources, unknown_sources = prepare_sources(source_info)
return cls(
version=addon_version,
full_name=full_name,
require_distribution=require_distribution,
sources=sources,
unknown_sources=unknown_sources,
hash=version_data.get("hash"),
title=title
)
@attr.s
class AddonInfo(object):
"""Object matching json payload from Server"""
name = attr.ib()
versions = attr.ib(default=attr.Factory(dict))
title = attr.ib(default=None)
description = attr.ib(default=None)
license = attr.ib(default=None)
authors = attr.ib(default=None)
@classmethod
def from_dict(cls, data):
"""Addon info by available versions.
Args:
data (dict[str, Any]): Addon information from server. Should
contain information about every version under 'versions'.
Returns:
AddonInfo: Addon info with available versions.
"""
# server payload contains info about all versions
addon_name = data["name"]
title = data.get("title") or addon_name
src_versions = data.get("versions") or {}
dst_versions = {
addon_version: AddonVersionInfo.from_dict(
addon_name, title, addon_version, version_data
)
for addon_version, version_data in src_versions.items()
}
return cls(
name=addon_name,
versions=dst_versions,
description=data.get("description"),
title=data.get("title") or addon_name,
license=data.get("license"),
authors=data.get("authors")
)
@attr.s
class DependencyItem(object):
"""Object matching payload from Server about single dependency package"""
name = attr.ib()
platform_name = attr.ib()
checksum = attr.ib()
sources = attr.ib(default=attr.Factory(list))
unknown_sources = attr.ib(default=attr.Factory(list))
source_addons = attr.ib(default=attr.Factory(dict))
python_modules = attr.ib(default=attr.Factory(dict))
@classmethod
def from_dict(cls, package):
sources, unknown_sources = prepare_sources(package.get("sources"))
return cls(
name=package["name"],
platform_name=package["platform"],
sources=sources,
unknown_sources=unknown_sources,
checksum=package["checksum"],
source_addons=package["sourceAddons"],
python_modules=package["pythonModules"]
)
@attr.s
class Installer:
version = attr.ib()
filename = attr.ib()
platform_name = attr.ib()
size = attr.ib()
checksum = attr.ib()
python_version = attr.ib()
python_modules = attr.ib()
sources = attr.ib(default=attr.Factory(list))
unknown_sources = attr.ib(default=attr.Factory(list))
@classmethod
def from_dict(cls, installer_info):
sources, unknown_sources = prepare_sources(
installer_info.get("sources"))
return cls(
version=installer_info["version"],
filename=installer_info["filename"],
platform_name=installer_info["platform"],
size=installer_info["size"],
sources=sources,
unknown_sources=unknown_sources,
checksum=installer_info["checksum"],
python_version=installer_info["pythonVersion"],
python_modules=installer_info["pythonModules"]
)
@attr.s
class Bundle:
"""Class representing bundle information."""
name = attr.ib()
installer_version = attr.ib()
addon_versions = attr.ib(default=attr.Factory(dict))
dependency_packages = attr.ib(default=attr.Factory(dict))
is_production = attr.ib(default=False)
is_staging = attr.ib(default=False)
@classmethod
def from_dict(cls, data):
return cls(
name=data["name"],
installer_version=data.get("installerVersion"),
addon_versions=data.get("addons", {}),
dependency_packages=data.get("dependencyPackages", {}),
is_production=data["isProduction"],
is_staging=data["isStaging"],
)

250
common/ayon_common/distribution/downloaders.py Normal file
View file

@ -0,0 +1,250 @@
import os
import logging
import platform
from abc import ABCMeta, abstractmethod
import ayon_api
from .file_handler import RemoteFileHandler
from .data_structures import UrlType
class SourceDownloader(metaclass=ABCMeta):
"""Abstract class for source downloader."""
log = logging.getLogger(__name__)
@classmethod
@abstractmethod
def download(cls, source, destination_dir, data, transfer_progress):
"""Returns url of downloaded addon zip file.
Tranfer progress can be ignored, in that case file transfer won't
be shown as 0-100% but as 'running'. First step should be to set
destination content size and then add transferred chunk sizes.
Args:
source (dict): {type:"http", "url":"https://} ...}
destination_dir (str): local folder to unzip
data (dict): More information about download content. Always have
'type' key in.
transfer_progress (ayon_api.TransferProgress): Progress of
transferred (copy/download) content.
Returns:
(str) local path to addon zip file
"""
pass
@classmethod
@abstractmethod
def cleanup(cls, source, destination_dir, data):
"""Cleanup files when distribution finishes or crashes.
Cleanup e.g. temporary files (downloaded zip) or other related stuff
to downloader.
"""
pass
@classmethod
def check_hash(cls, addon_path, addon_hash, hash_type="sha256"):
"""Compares 'hash' of downloaded 'addon_url' file.
Args:
addon_path (str): Local path to addon file.
addon_hash (str): Hash of downloaded file.
hash_type (str): Type of hash.
Raises:
ValueError if hashes doesn't match
"""
if not os.path.exists(addon_path):
raise ValueError(f"{addon_path} doesn't exist.")
if not RemoteFileHandler.check_integrity(
addon_path, addon_hash, hash_type=hash_type
):
raise ValueError(f"{addon_path} doesn't match expected hash.")
@classmethod
def unzip(cls, addon_zip_path, destination_dir):
"""Unzips local 'addon_zip_path' to 'destination'.
Args:
addon_zip_path (str): local path to addon zip file
destination_dir (str): local folder to unzip
"""
RemoteFileHandler.unzip(addon_zip_path, destination_dir)
os.remove(addon_zip_path)
class OSDownloader(SourceDownloader):
"""Downloader using files from file drive."""
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
# OS doesn't need to download, unzip directly
addon_url = source["path"].get(platform.system().lower())
if not os.path.exists(addon_url):
raise ValueError(f"{addon_url} is not accessible")
return addon_url
@classmethod
def cleanup(cls, source, destination_dir, data):
# Nothing to do - download does not copy anything
pass
class HTTPDownloader(SourceDownloader):
"""Downloader using http or https protocol."""
CHUNK_SIZE = 100000
@staticmethod
def get_filename(source):
source_url = source["url"]
filename = source.get("filename")
if not filename:
filename = os.path.basename(source_url)
basename, ext = os.path.splitext(filename)
allowed_exts = set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)
if ext.lower().lstrip(".") not in allowed_exts:
filename = f"{basename}.zip"
return filename
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
source_url = source["url"]
cls.log.debug(f"Downloading {source_url} to {destination_dir}")
headers = source.get("headers")
filename = cls.get_filename(source)
# TODO use transfer progress
RemoteFileHandler.download_url(
source_url,
destination_dir,
filename,
headers=headers
)
return os.path.join(destination_dir, filename)
@classmethod
def cleanup(cls, source, destination_dir, data):
filename = cls.get_filename(source)
filepath = os.path.join(destination_dir, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.remove(filepath)
class AyonServerDownloader(SourceDownloader):
"""Downloads static resource file from AYON Server.
Expects filled env var AYON_SERVER_URL.
"""
CHUNK_SIZE = 8192
@classmethod
def download(cls, source, destination_dir, data, transfer_progress):
path = source["path"]
filename = source["filename"]
if path and not filename:
filename = path.split("/")[-1]
cls.log.debug(f"Downloading {filename} to {destination_dir}")
_, ext = os.path.splitext(filename)
ext = ext.lower().lstrip(".")
valid_exts = set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)
if ext not in valid_exts:
raise ValueError((
f"Invalid file extension \"{ext}\"."
f" Expected {', '.join(valid_exts)}"
))
if path:
filepath = os.path.join(destination_dir, filename)
return ayon_api.download_file(
path,
filepath,
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
# dst_filepath = os.path.join(destination_dir, filename)
if data["type"] == "dependency_package":
return ayon_api.download_dependency_package(
data["name"],
destination_dir,
filename,
platform_name=data["platform"],
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
if data["type"] == "addon":
return ayon_api.download_addon_private_file(
data["name"],
data["version"],
filename,
destination_dir,
chunk_size=cls.CHUNK_SIZE,
progress=transfer_progress
)
raise ValueError(f"Unknown type to download \"{data['type']}\"")
@classmethod
def cleanup(cls, source, destination_dir, data):
filename = source["filename"]
filepath = os.path.join(destination_dir, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.remove(filepath)
class DownloadFactory:
"""Factory for downloaders."""
def __init__(self):
self._downloaders = {}
def register_format(self, downloader_type, downloader):
"""Register downloader for download type.
Args:
downloader_type (UrlType): Type of source.
downloader (SourceDownloader): Downloader which cares about
download, hash check and unzipping.
"""
self._downloaders[downloader_type.value] = downloader
def get_downloader(self, downloader_type):
"""Registered downloader for type.
Args:
downloader_type (UrlType): Type of source.
Returns:
SourceDownloader: Downloader object which should care about file
distribution.
Raises:
ValueError: If type does not have registered downloader.
"""
if downloader := self._downloaders.get(downloader_type):
return downloader()
raise ValueError(f"{downloader_type} not implemented")
def get_default_download_factory():
download_factory = DownloadFactory()
download_factory.register_format(UrlType.FILESYSTEM, OSDownloader)
download_factory.register_format(UrlType.HTTP, HTTPDownloader)
download_factory.register_format(UrlType.SERVER, AyonServerDownloader)
return download_factory

129
common/ayon_common/distribution/file_handler.py
View file

@ -9,21 +9,23 @@ import hashlib
import tarfile
import zipfile
import requests
USER_AGENT = "openpype"
USER_AGENT = "AYON-launcher"
class RemoteFileHandler:
"""Download file from url, might be GDrive shareable link"""
IMPLEMENTED_ZIP_FORMATS = ['zip', 'tar', 'tgz',
'tar.gz', 'tar.xz', 'tar.bz2']
IMPLEMENTED_ZIP_FORMATS = {
"zip", "tar", "tgz", "tar.gz", "tar.xz", "tar.bz2"
}
@staticmethod
def calculate_md5(fpath, chunk_size=10000):
md5 = hashlib.md5()
with open(fpath, 'rb') as f:
for chunk in iter(lambda: f.read(chunk_size), b''):
with open(fpath, "rb") as f:
for chunk in iter(lambda: f.read(chunk_size), b""):
md5.update(chunk)
return md5.hexdigest()
@ -45,7 +47,7 @@ class RemoteFileHandler:
h = hashlib.sha256()
b = bytearray(128 * 1024)
mv = memoryview(b)
with open(fpath, 'rb', buffering=0) as f:
with open(fpath, "rb", buffering=0) as f:
for n in iter(lambda: f.readinto(mv), 0):
h.update(mv[:n])
return h.hexdigest()
@ -69,21 +71,25 @@ class RemoteFileHandler:
@staticmethod
def download_url(
url, root, filename=None,
sha256=None, max_redirect_hops=3, headers=None
url,
root,
filename=None,
max_redirect_hops=3,
headers=None
):
"""Download a file from a url and place it in root.
"""Download a file from url and place it in root.
Args:
url (str): URL to download file from
root (str): Directory to place downloaded file in
filename (str, optional): Name to save the file under.
If None, use the basename of the URL
sha256 (str, optional): sha256 checksum of the download.
If None, do not check
max_redirect_hops (int, optional): Maximum number of redirect
max_redirect_hops (Optional[int]): Maximum number of redirect
hops allowed
headers (dict): additional required headers - Authentication etc..
headers (Optional[dict[str, str]]): Additional required headers
- Authentication etc..
"""
root = os.path.expanduser(root)
if not filename:
filename = os.path.basename(url)
@ -91,59 +97,44 @@ class RemoteFileHandler:
os.makedirs(root, exist_ok=True)
# check if file is already present locally
if RemoteFileHandler.check_integrity(fpath,
sha256, hash_type="sha256"):
print(f"Using downloaded and verified file: {fpath}")
return
# expand redirect chain if needed
url = RemoteFileHandler._get_redirect_url(url,
max_hops=max_redirect_hops,
headers=headers)
url = RemoteFileHandler._get_redirect_url(
url, max_hops=max_redirect_hops, headers=headers)
# check if file is located on Google Drive
file_id = RemoteFileHandler._get_google_drive_file_id(url)
if file_id is not None:
return RemoteFileHandler.download_file_from_google_drive(
file_id, root, filename, sha256)
file_id, root, filename)
# download the file
try:
print(f"Downloading {url} to {fpath}")
RemoteFileHandler._urlretrieve(url, fpath, headers=headers)
except (urllib.error.URLError, IOError) as e:
if url[:5] == "https":
url = url.replace("https:", "http:")
print((
"Failed download. Trying https -> http instead."
f" Downloading {url} to {fpath}"
))
RemoteFileHandler._urlretrieve(url, fpath,
headers=headers)
else:
raise e
except (urllib.error.URLError, IOError) as exc:
if url[:5] != "https":
raise exc
# check integrity of downloaded file
if not RemoteFileHandler.check_integrity(fpath,
sha256, hash_type="sha256"):
raise RuntimeError("File not found or corrupted.")
url = url.replace("https:", "http:")
print((
"Failed download. Trying https -> http instead."
f" Downloading {url} to {fpath}"
))
RemoteFileHandler._urlretrieve(url, fpath, headers=headers)
@staticmethod
def download_file_from_google_drive(file_id, root,
filename=None,
sha256=None):
def download_file_from_google_drive(
file_id, root, filename=None
):
"""Download a Google Drive file from and place it in root.
Args:
file_id (str): id of file to be downloaded
root (str): Directory to place downloaded file in
filename (str, optional): Name to save the file under.
If None, use the id of the file.
sha256 (str, optional): sha256 checksum of the download.
If None, do not check
"""
# Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa
import requests
url = "https://docs.google.com/uc?export=download"
root = os.path.expanduser(root)
@ -153,17 +144,16 @@ class RemoteFileHandler:
os.makedirs(root, exist_ok=True)
if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(
fpath, sha256, hash_type="sha256"):
print('Using downloaded and verified file: ' + fpath)
if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath):
print(f"Using downloaded and verified file: {fpath}")
else:
session = requests.Session()
response = session.get(url, params={'id': file_id}, stream=True)
response = session.get(url, params={"id": file_id}, stream=True)
token = RemoteFileHandler._get_confirm_token(response)
if token:
params = {'id': file_id, 'confirm': token}
params = {"id": file_id, "confirm": token}
response = session.get(url, params=params, stream=True)
response_content_generator = response.iter_content(32768)
@ -191,28 +181,28 @@ class RemoteFileHandler:
destination_path = os.path.dirname(path)
_, archive_type = os.path.splitext(path)
archive_type = archive_type.lstrip('.')
archive_type = archive_type.lstrip(".")
if archive_type in ['zip']:
print("Unzipping {}->{}".format(path, destination_path))
if archive_type in ["zip"]:
print(f"Unzipping {path}->{destination_path}")
zip_file = zipfile.ZipFile(path)
zip_file.extractall(destination_path)
zip_file.close()
elif archive_type in [
'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.bz2'
"tar", "tgz", "tar.gz", "tar.xz", "tar.bz2"
]:
print("Unzipping {}->{}".format(path, destination_path))
if archive_type == 'tar':
tar_type = 'r:'
elif archive_type.endswith('xz'):
tar_type = 'r:xz'
elif archive_type.endswith('gz'):
tar_type = 'r:gz'
elif archive_type.endswith('bz2'):
tar_type = 'r:bz2'
print(f"Unzipping {path}->{destination_path}")
if archive_type == "tar":
tar_type = "r:"
elif archive_type.endswith("xz"):
tar_type = "r:xz"
elif archive_type.endswith("gz"):
tar_type = "r:gz"
elif archive_type.endswith("bz2"):
tar_type = "r:bz2"
else:
tar_type = 'r:*'
tar_type = "r:*"
try:
tar_file = tarfile.open(path, tar_type)
except tarfile.ReadError:
@ -229,9 +219,8 @@ class RemoteFileHandler:
chunk_size = chunk_size or 8192
with open(filename, "wb") as fh:
with urllib.request.urlopen(
urllib.request.Request(url,
headers=final_headers)) \
as response:
urllib.request.Request(url, headers=final_headers)
) as response:
for chunk in iter(lambda: response.read(chunk_size), ""):
if not chunk:
break
@ -245,12 +234,12 @@ class RemoteFileHandler:
final_headers.update(headers)
for _ in range(max_hops + 1):
with urllib.request.urlopen(
urllib.request.Request(url,
headers=final_headers)) as response:
urllib.request.Request(url, headers=final_headers)
) as response:
if response.url == url or response.url is None:
return url
url = response.url
return response.url
else:
raise RecursionError(
f"Request to {initial_url} exceeded {max_hops} redirects. "
@ -260,7 +249,7 @@ class RemoteFileHandler:
@staticmethod
def _get_confirm_token(response):
for key, value in response.cookies.items():
if key.startswith('download_warning'):
if key.startswith("download_warning"):
return value
# handle antivirus warning for big zips

253
common/ayon_common/distribution/tests/test_addon_distributtion.py
View file

@ -1,20 +1,33 @@
import pytest
import attr
import os
import sys
import copy
import tempfile
from common.ayon_common.distribution.addon_distribution import (
import attr
import pytest
current_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.abspath(os.path.join(current_dir, "..", "..", "..", ".."))
sys.path.append(root_dir)
from common.ayon_common.distribution.downloaders import (
DownloadFactory,
OSDownloader,
HTTPDownloader,
AddonInfo,
AyonDistribution,
UpdateState
)
from common.ayon_common.distribution.addon_info import UrlType
from common.ayon_common.distribution.control import (
AyonDistribution,
UpdateState,
)
from common.ayon_common.distribution.data_structures import (
AddonInfo,
UrlType,
)
@pytest.fixture
def addon_download_factory():
def download_factory():
addon_downloader = DownloadFactory()
addon_downloader.register_format(UrlType.FILESYSTEM, OSDownloader)
addon_downloader.register_format(UrlType.HTTP, HTTPDownloader)
@ -23,65 +36,84 @@ def addon_download_factory():
@pytest.fixture
def http_downloader(addon_download_factory):
yield addon_download_factory.get_downloader(UrlType.HTTP.value)
def http_downloader(download_factory):
yield download_factory.get_downloader(UrlType.HTTP.value)
@pytest.fixture
def temp_folder():
yield tempfile.mkdtemp()
yield tempfile.mkdtemp(prefix="ayon_test_")
@pytest.fixture
def sample_bundles():
yield {
"bundles": [
{
"name": "TestBundle",
"createdAt": "2023-06-29T00:00:00.0+00:00",
"installerVersion": None,
"addons": {
"slack": "1.0.0"
},
"dependencyPackages": {},
"isProduction": True,
"isStaging": False
}
],
"productionBundle": "TestBundle",
"stagingBundle": None
}
@pytest.fixture
def sample_addon_info():
addon_info = {
"versions": {
yield {
"name": "slack",
"title": "Slack addon",
"versions": {
"1.0.0": {
"clientPyproject": {
"tool": {
"poetry": {
"hasSettings": True,
"hasSiteSettings": False,
"clientPyproject": {
"tool": {
"poetry": {
"dependencies": {
"nxtools": "^1.6",
"orjson": "^3.6.7",
"typer": "^0.4.1",
"email-validator": "^1.1.3",
"python": "^3.10",
"fastapi": "^0.73.0"
"nxtools": "^1.6",
"orjson": "^3.6.7",
"typer": "^0.4.1",
"email-validator": "^1.1.3",
"python": "^3.10",
"fastapi": "^0.73.0"
}
}
}
},
"hasSettings": True,
"clientSourceInfo": [
{
"type": "http",
"path": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa
"filename": "dummy.zip"
},
{
"type": "filesystem",
"path": {
"windows": ["P:/sources/some_file.zip",
"W:/sources/some_file.zip"], # noqa
"linux": ["/mnt/srv/sources/some_file.zip"],
"darwin": ["/Volumes/srv/sources/some_file.zip"]
}
}
],
"frontendScopes": {
"project": {
"sidebar": "hierarchy"
}
}
}
}
},
"clientSourceInfo": [
{
"type": "http",
"path": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa
"filename": "dummy.zip"
},
{
"type": "filesystem",
"path": {
"windows": "P:/sources/some_file.zip",
"linux": "/mnt/srv/sources/some_file.zip",
"darwin": "/Volumes/srv/sources/some_file.zip"
}
}
],
"frontendScopes": {
"project": {
"sidebar": "hierarchy",
}
},
"hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa
}
},
"description": "",
"title": "Slack addon",
"name": "openpype_slack",
"productionVersion": "1.0.0",
"hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa
},
"description": ""
}
yield addon_info
def test_register(printer):
@ -103,21 +135,16 @@ def test_get_downloader(printer, download_factory):
def test_addon_info(printer, sample_addon_info):
"""Tests parsing of expected payload from v4 server into AadonInfo."""
valid_minimum = {
"name": "openpype_slack",
"productionVersion": "1.0.0",
"versions": {
"1.0.0": {
"clientSourceInfo": [
{
"type": "filesystem",
"path": {
"windows": [
"P:/sources/some_file.zip",
"W:/sources/some_file.zip"],
"linux": [
"/mnt/srv/sources/some_file.zip"],
"darwin": [
"/Volumes/srv/sources/some_file.zip"] # noqa
"name": "slack",
"versions": {
"1.0.0": {
"clientSourceInfo": [
{
"type": "filesystem",
"path": {
"windows": "P:/sources/some_file.zip",
"linux": "/mnt/srv/sources/some_file.zip",
"darwin": "/Volumes/srv/sources/some_file.zip"
}
}
]
@ -127,18 +154,10 @@ def test_addon_info(printer, sample_addon_info):
assert AddonInfo.from_dict(valid_minimum), "Missing required fields"
valid_minimum["versions"].pop("1.0.0")
with pytest.raises(KeyError):
assert not AddonInfo.from_dict(valid_minimum), "Must fail without version data" # noqa
valid_minimum.pop("productionVersion")
assert not AddonInfo.from_dict(
valid_minimum), "none if not productionVersion" # noqa
addon = AddonInfo.from_dict(sample_addon_info)
assert addon, "Should be created"
assert addon.name == "openpype_slack", "Incorrect name"
assert addon.version == "1.0.0", "Incorrect version"
assert addon.name == "slack", "Incorrect name"
assert "1.0.0" in addon.versions, "Version is not in versions"
with pytest.raises(TypeError):
assert addon["name"], "Dict approach not implemented"
@ -147,37 +166,83 @@ def test_addon_info(printer, sample_addon_info):
assert addon_as_dict["name"], "Dict approach should work"
def test_update_addon_state(printer, sample_addon_info,
temp_folder, download_factory):
def _get_dist_item(dist_items, name, version):
final_dist_info = next(
(
dist_info
for dist_info in dist_items
if (
dist_info["addon_name"] == name
and dist_info["addon_version"] == version
)
),
{}
)
return final_dist_info["dist_item"]
def test_update_addon_state(
printer, sample_addon_info, temp_folder, download_factory, sample_bundles
):
"""Tests possible cases of addon update."""
addon_info = AddonInfo.from_dict(sample_addon_info)
orig_hash = addon_info.hash
addon_version = list(sample_addon_info["versions"])[0]
broken_addon_info = copy.deepcopy(sample_addon_info)
# Cause crash because of invalid hash
addon_info.hash = "brokenhash"
broken_addon_info["versions"][addon_version]["hash"] = "brokenhash"
distribution = AyonDistribution(
temp_folder, temp_folder, download_factory, [addon_info], None
addon_dirpath=temp_folder,
dependency_dirpath=temp_folder,
dist_factory=download_factory,
addons_info=[broken_addon_info],
dependency_packages_info=[],
bundles_info=sample_bundles
)
distribution.distribute()
dist_items = distribution.get_addons_dist_items()
slack_state = dist_items["openpype_slack_1.0.0"].state
dist_items = distribution.get_addon_dist_items()
slack_dist_item = _get_dist_item(
dist_items,
sample_addon_info["name"],
addon_version
)
slack_state = slack_dist_item.state
assert slack_state == UpdateState.UPDATE_FAILED, (
"Update should have failed because of wrong hash")
# Fix cache and validate if was updated
addon_info.hash = orig_hash
distribution = AyonDistribution(
temp_folder, temp_folder, download_factory, [addon_info], None
addon_dirpath=temp_folder,
dependency_dirpath=temp_folder,
dist_factory=download_factory,
addons_info=[sample_addon_info],
dependency_packages_info=[],
bundles_info=sample_bundles
)
distribution.distribute()
dist_items = distribution.get_addons_dist_items()
assert dist_items["openpype_slack_1.0.0"].state == UpdateState.UPDATED, (
dist_items = distribution.get_addon_dist_items()
slack_dist_item = _get_dist_item(
dist_items,
sample_addon_info["name"],
addon_version
)
assert slack_dist_item.state == UpdateState.UPDATED, (
"Addon should have been updated")
# Is UPDATED without calling distribute
distribution = AyonDistribution(
temp_folder, temp_folder, download_factory, [addon_info], None
addon_dirpath=temp_folder,
dependency_dirpath=temp_folder,
dist_factory=download_factory,
addons_info=[sample_addon_info],
dependency_packages_info=[],
bundles_info=sample_bundles
)
dist_items = distribution.get_addons_dist_items()
assert dist_items["openpype_slack_1.0.0"].state == UpdateState.UPDATED, (
dist_items = distribution.get_addon_dist_items()
slack_dist_item = _get_dist_item(
dist_items,
sample_addon_info["name"],
addon_version
)
assert slack_dist_item.state == UpdateState.UPDATED, (
"Addon should already exist")

146
common/ayon_common/distribution/ui/missing_bundle_window.py Normal file
View file

@ -0,0 +1,146 @@
import sys
from qtpy import QtWidgets, QtGui
from ayon_common import is_staging_enabled
from ayon_common.resources import (
get_icon_path,
load_stylesheet,
)
from ayon_common.ui_utils import get_qt_app
class MissingBundleWindow(QtWidgets.QDialog):
default_width = 410
default_height = 170
def __init__(
self, url=None, bundle_name=None, use_staging=None, parent=None
):
super().__init__(parent)
icon_path = get_icon_path()
icon = QtGui.QIcon(icon_path)
self.setWindowIcon(icon)
self.setWindowTitle("Missing Bundle")
self._url = url
self._bundle_name = bundle_name
self._use_staging = use_staging
self._first_show = True
info_label = QtWidgets.QLabel("", self)
info_label.setWordWrap(True)
btns_widget = QtWidgets.QWidget(self)
confirm_btn = QtWidgets.QPushButton("Exit", btns_widget)
btns_layout = QtWidgets.QHBoxLayout(btns_widget)
btns_layout.setContentsMargins(0, 0, 0, 0)
btns_layout.addStretch(1)
btns_layout.addWidget(confirm_btn, 0)
main_layout = QtWidgets.QVBoxLayout(self)
main_layout.addWidget(info_label, 0)
main_layout.addStretch(1)
main_layout.addWidget(btns_widget, 0)
confirm_btn.clicked.connect(self._on_confirm_click)
self._info_label = info_label
self._confirm_btn = confirm_btn
self._update_label()
def set_url(self, url):
if url == self._url:
return
self._url = url
self._update_label()
def set_bundle_name(self, bundle_name):
if bundle_name == self._bundle_name:
return
self._bundle_name = bundle_name
self._update_label()
def set_use_staging(self, use_staging):
if self._use_staging == use_staging:
return
self._use_staging = use_staging
self._update_label()
def showEvent(self, event):
super().showEvent(event)
if self._first_show:
self._first_show = False
self._on_first_show()
self._recalculate_sizes()
def resizeEvent(self, event):
super().resizeEvent(event)
self._recalculate_sizes()
def _recalculate_sizes(self):
hint = self._confirm_btn.sizeHint()
new_width = max((hint.width(), hint.height() * 3))
self._confirm_btn.setMinimumWidth(new_width)
def _on_first_show(self):
self.setStyleSheet(load_stylesheet())
self.resize(self.default_width, self.default_height)
def _on_confirm_click(self):
self.accept()
self.close()
def _update_label(self):
self._info_label.setText(self._get_label())
def _get_label(self):
url_part = f" <b>{self._url}</b>" if self._url else ""
if self._bundle_name:
return (
f"Requested release bundle <b>{self._bundle_name}</b>"
f" is not available on server{url_part}."
"<br/><br/>Try to restart AYON desktop launcher. Please"
" contact your administrator if issue persist."
)
mode = "staging" if self._use_staging else "production"
return (
f"No release bundle is set as {mode} on the AYON"
f" server{url_part} so there is nothing to launch."
"<br/><br/>Please contact your administrator"
" to resolve the issue."
)
def main():
"""Show message that server does not have set bundle to use.
It is possible to pass url as argument to show it in the message. To use
this feature, pass `--url <url>` as argument to this script.
"""
url = None
bundle_name = None
if "--url" in sys.argv:
url_index = sys.argv.index("--url") + 1
if url_index < len(sys.argv):
url = sys.argv[url_index]
if "--bundle" in sys.argv:
bundle_index = sys.argv.index("--bundle") + 1
if bundle_index < len(sys.argv):
bundle_name = sys.argv[bundle_index]
use_staging = is_staging_enabled()
app = get_qt_app()
window = MissingBundleWindow(url, bundle_name, use_staging)
window.show()
app.exec_()
if __name__ == "__main__":
main()

90
common/ayon_common/distribution/utils.py Normal file
View file

@ -0,0 +1,90 @@
import os
import subprocess
from ayon_common.utils import get_ayon_appdirs, get_ayon_launch_args
def get_local_dir(*subdirs):
"""Get product directory in user's home directory.
Each user on machine have own local directory where are downloaded updates,
addons etc.
Returns:
str: Path to product local directory.
"""
if not subdirs:
raise ValueError("Must fill dir_name if nothing else provided!")
local_dir = get_ayon_appdirs(*subdirs)
if not os.path.isdir(local_dir):
try:
os.makedirs(local_dir)
except Exception: # TODO fix exception
raise RuntimeError(f"Cannot create {local_dir}")
return local_dir
def get_addons_dir():
"""Directory where addon packages are stored.
Path to addons is defined using python module 'appdirs' which
The path is stored into environment variable 'AYON_ADDONS_DIR'.
Value of environment variable can be overriden, but we highly recommended
to use that option only for development purposes.
Returns:
str: Path to directory where addons should be downloaded.
"""
addons_dir = os.environ.get("AYON_ADDONS_DIR")
if not addons_dir:
addons_dir = get_local_dir("addons")
os.environ["AYON_ADDONS_DIR"] = addons_dir
return addons_dir
def get_dependencies_dir():
"""Directory where dependency packages are stored.
Path to addons is defined using python module 'appdirs' which
The path is stored into environment variable 'AYON_DEPENDENCIES_DIR'.
Value of environment variable can be overriden, but we highly recommended
to use that option only for development purposes.
Returns:
str: Path to directory where dependency packages should be downloaded.
"""
dependencies_dir = os.environ.get("AYON_DEPENDENCIES_DIR")
if not dependencies_dir:
dependencies_dir = get_local_dir("dependency_packages")
os.environ["AYON_DEPENDENCIES_DIR"] = dependencies_dir
return dependencies_dir
def show_missing_bundle_information(url, bundle_name=None):
"""Show missing bundle information window.
This function should be called when server does not have set bundle for
production or staging, or when bundle that should be used is not available
on server.
Using subprocess to show the dialog. Is blocking and is waiting until
dialog is closed.
Args:
url (str): Server url where bundle is not set.
bundle_name (Optional[str]): Name of bundle that was not found.
"""
ui_dir = os.path.join(os.path.dirname(__file__), "ui")
script_path = os.path.join(ui_dir, "missing_bundle_window.py")
args = get_ayon_launch_args(script_path, "--skip-bootstrap", "--url", url)
if bundle_name:
args.extend(["--bundle", bundle_name])
subprocess.call(args)

4
common/ayon_common/resources/__init__.py
View file

@ -1,5 +1,7 @@
import os
from ayon_common.utils import is_staging_enabled
RESOURCES_DIR = os.path.dirname(os.path.abspath(__file__))
@ -10,7 +12,7 @@ def get_resource_path(*args):
def get_icon_path():
if os.environ.get("OPENPYPE_USE_STAGING") == "1":
if is_staging_enabled():
return get_resource_path("AYON_staging.png")
return get_resource_path("AYON.png")

2
common/ayon_common/resources/stylesheet.css
View file

@ -81,4 +81,4 @@ QLineEdit[state="invalid"] {
}
#LikeDisabledInput:focus {
border-color: #373D48;
}
}

0
common/ayon_common/connection/ui/lib.py → common/ayon_common/ui_utils.py
View file

40
common/ayon_common/utils.py
View file

@ -1,6 +1,9 @@
import os
import sys
import appdirs
IS_BUILT_APPLICATION = getattr(sys, "frozen", False)
def get_ayon_appdirs(*args):
"""Local app data directory of AYON client.
@ -18,8 +21,23 @@ def get_ayon_appdirs(*args):
)
def is_staging_enabled():
"""Check if staging is enabled.
Returns:
bool: True if staging is enabled.
"""
return os.getenv("AYON_USE_STAGING") == "1"
def _create_local_site_id():
"""Create a local site identifier."""
"""Create a local site identifier.
Returns:
str: Randomly generated site id.
"""
from coolname import generate_slug
new_id = generate_slug(3)
@ -33,6 +51,9 @@ def get_local_site_id():
"""Get local site identifier.
Site id is created if does not exist yet.
Returns:
str: Site id.
"""
# used for background syncing
@ -50,3 +71,20 @@ def get_local_site_id():
with open(site_id_path, "w") as stream:
stream.write(site_id)
return site_id
def get_ayon_launch_args(*args):
"""Launch arguments that can be used to launch ayon process.
Args:
*args (str): Additional arguments.
Returns:
list[str]: Launch arguments.
"""
output = [sys.executable]
if not IS_BUILT_APPLICATION:
output.append(sys.argv[0])
output.extend(args)
return output

39
openpype/modules/base.py
View file

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
"""Base class for Pype Modules."""
import copy
import os
import sys
import json
@ -319,7 +320,35 @@ def _get_ayon_addons_information():
List[Dict[str, Any]]: List of addon information to use.
"""
return ayon_api.get_addons_info()["addons"]
output = []
bundle_name = os.getenv("AYON_BUNDLE_NAME")
bundles = ayon_api.get_bundles()["bundles"]
final_bundle = next(
(
bundle
for bundle in bundles
if bundle["name"] == bundle_name
),
None
)
if final_bundle is None:
return output
bundle_addons = final_bundle["addons"]
addons = ayon_api.get_addons_info()["addons"]
for addon in addons:
name = addon["name"]
versions = addon.get("versions")
addon_version = bundle_addons.get(name)
if addon_version is None or not versions:
continue
version = versions.get(addon_version)
if version:
version = copy.deepcopy(version)
version["name"] = name
version["version"] = addon_version
output.append(version)
return output
def _load_ayon_addons(openpype_modules, modules_key, log):
@ -354,15 +383,9 @@ def _load_ayon_addons(openpype_modules, modules_key, log):
))
return v3_addons_to_skip
version_key = (
"stagingVersion" if is_staging_enabled()
else "productionVersion"
)
for addon_info in addons_info:
addon_name = addon_info["name"]
addon_version = addon_info.get(version_key)
if not addon_version:
continue
addon_version = addon_info["version"]
folder_name = "{}_{}".format(addon_name, addon_version)
addon_dir = os.path.join(addons_dir, folder_name)

115
openpype/settings/ayon_settings.py
View file

@ -13,7 +13,8 @@ Main entrypoints are functions:
- get_ayon_project_settings - replacement for 'get_project_settings'
- get_ayon_system_settings - replacement for 'get_system_settings'
"""
import os
import collections
import json
import copy
import time
@ -1275,9 +1276,15 @@ def convert_project_settings(ayon_settings, default_settings):
class CacheItem:
lifetime = 10
def __init__(self, value):
def __init__(self, value, outdate_time=None):
self._value = value
self._outdate_time = time.time() + self.lifetime
if outdate_time is None:
outdate_time = time.time() + self.lifetime
self._outdate_time = outdate_time
@classmethod
def create_outdated(cls):
return cls({}, 0)
def get_value(self):
return copy.deepcopy(self._value)
@ -1291,57 +1298,89 @@ class CacheItem:
return time.time() > self._outdate_time
class AyonSettingsCache:
_cache_by_project_name = {}
_production_settings = None
class _AyonSettingsCache:
use_bundles = None
variant = None
addon_versions = CacheItem.create_outdated()
studio_settings = CacheItem.create_outdated()
cache_by_project_name = collections.defaultdict(
CacheItem.create_outdated)
@classmethod
def get_production_settings(cls):
if (
cls._production_settings is None
or cls._production_settings.is_outdated
):
def _use_bundles(cls):
if _AyonSettingsCache.use_bundles is None:
major, minor, _, _, _ = ayon_api.get_server_version_tuple()
_AyonSettingsCache.use_bundles = major == 0 and minor >= 3
return _AyonSettingsCache.use_bundles
@classmethod
def _get_variant(cls):
if _AyonSettingsCache.variant is None:
from openpype.lib.openpype_version import is_staging_enabled
variant = "staging" if is_staging_enabled() else "production"
value = ayon_api.get_addons_settings(
only_values=False, variant=variant)
if cls._production_settings is None:
cls._production_settings = CacheItem(value)
else:
cls._production_settings.update_value(value)
return cls._production_settings.get_value()
_AyonSettingsCache.variant = (
"staging" if is_staging_enabled() else "production"
)
return _AyonSettingsCache.variant
@classmethod
def _get_bundle_name(cls):
return os.environ["AYON_BUNDLE_NAME"]
@classmethod
def get_value_by_project(cls, project_name):
production_settings = cls.get_production_settings()
addon_versions = production_settings["versions"]
if project_name is None:
return production_settings["settings"], addon_versions
cache_item = cls._cache_by_project_name.get(project_name)
if cache_item is None or cache_item.is_outdated:
value = ayon_api.get_addons_settings(project_name)
if cache_item is None:
cache_item = CacheItem(value)
cls._cache_by_project_name[project_name] = cache_item
cache_item = _AyonSettingsCache.cache_by_project_name[project_name]
if cache_item.is_outdated:
if cls._use_bundles():
value = ayon_api.get_addons_settings(
bundle_name=cls._get_bundle_name(),
project_name=project_name
)
else:
cache_item.update_value(value)
value = ayon_api.get_addons_settings(project_name)
cache_item.update_value(value)
return cache_item.get_value()
return cache_item.get_value(), addon_versions
@classmethod
def _get_addon_versions_from_bundle(cls):
expected_bundle = cls._get_bundle_name()
bundles = ayon_api.get_bundles()["bundles"]
bundle = next(
(
bundle
for bundle in bundles
if bundle["name"] == expected_bundle
),
None
)
if bundle is not None:
return bundle["addons"]
return {}
@classmethod
def get_addon_versions(cls):
cache_item = _AyonSettingsCache.addon_versions
if cache_item.is_outdated:
if cls._use_bundles():
addons = cls._get_addon_versions_from_bundle()
else:
settings_data = ayon_api.get_addons_settings(
only_values=False, variant=cls._get_variant())
addons = settings_data["versions"]
cache_item.update_value(addons)
return cache_item.get_value()
def get_ayon_project_settings(default_values, project_name):
ayon_settings, addon_versions = (
AyonSettingsCache.get_value_by_project(project_name)
)
ayon_settings = _AyonSettingsCache.get_value_by_project(project_name)
return convert_project_settings(ayon_settings, default_values)
def get_ayon_system_settings(default_values):
ayon_settings, addon_versions = (
AyonSettingsCache.get_value_by_project(None)
)
addon_versions = _AyonSettingsCache.get_addon_versions()
ayon_settings = _AyonSettingsCache.get_value_by_project(None)
return convert_system_settings(
ayon_settings, default_values, addon_versions
)

5
openpype/vendor/python/common/ayon_api/__init__.py vendored
View file

@ -1,6 +1,8 @@
from .version import __version__
from .utils import (
TransferProgress,
slugify_string,
create_dependency_package_basename,
)
from .server_api import (
ServerAPI,
@ -183,8 +185,11 @@ from ._api import (
__all__ = (
"__version__",
"TransferProgress",
"slugify_string",
"create_dependency_package_basename",
"ServerAPI",

72
openpype/vendor/python/common/ayon_api/server_api.py vendored
View file

@ -61,6 +61,7 @@ from .utils import (
entity_data_json_default,
failed_json_default,
TransferProgress,
create_dependency_package_basename,
)
PatternType = type(re.compile(""))
@ -114,6 +115,10 @@ class RestApiResponse(object):
self.status = status_code
self._data = data
@property
def text(self):
return self._response.text
@property
def orig_response(self):
return self._response
@ -150,11 +155,13 @@ class RestApiResponse(object):
def status_code(self):
return self.status
def raise_for_status(self):
def raise_for_status(self, message=None):
try:
self._response.raise_for_status()
except requests.exceptions.HTTPError as exc:
raise HTTPRequestError(str(exc), exc.response)
if message is None:
message = str(exc)
raise HTTPRequestError(message, exc.response)
def __enter__(self, *args, **kwargs):
return self._response.__enter__(*args, **kwargs)
@ -1303,13 +1310,15 @@ class ServerAPI(object):
progress.set_transfer_done()
return progress
def _upload_file(self, url, filepath, progress):
def _upload_file(self, url, filepath, progress, request_type=None):
if request_type is None:
request_type = RequestTypes.put
kwargs = {}
if self._session is None:
kwargs["headers"] = self.get_headers()
post_func = self._base_functions_mapping[RequestTypes.post]
post_func = self._base_functions_mapping[request_type]
else:
post_func = self._session_functions_mapping[RequestTypes.post]
post_func = self._session_functions_mapping[request_type]
with open(filepath, "rb") as stream:
stream.seek(0, io.SEEK_END)
@ -1320,7 +1329,9 @@ class ServerAPI(object):
response.raise_for_status()
progress.set_transferred_size(size)
def upload_file(self, endpoint, filepath, progress=None):
def upload_file(
self, endpoint, filepath, progress=None, request_type=None
):
"""Upload file to server.
Todos:
@ -1331,6 +1342,8 @@ class ServerAPI(object):
filepath (str): Source filepath.
progress (Optional[TransferProgress]): Object that gives ability
to track upload progress.
request_type (Optional[RequestType]): Type of request that will
be used to upload file.
"""
if endpoint.startswith(self._base_url):
@ -1349,7 +1362,7 @@ class ServerAPI(object):
progress.set_started()
try:
self._upload_file(url, filepath, progress)
self._upload_file(url, filepath, progress, request_type)
except Exception as exc:
progress.set_failed(str(exc))
@ -1486,13 +1499,11 @@ class ServerAPI(object):
"""
response = self.delete("attributes/{}".format(attribute_name))
if response.status_code != 204:
# TODO raise different exception
raise ValueError(
"Attribute \"{}\" was not created/updated. {}".format(
attribute_name, response.detail
)
response.raise_for_status(
"Attribute \"{}\" was not created/updated. {}".format(
attribute_name, response.detail
)
)
self.reset_attributes_schema()
@ -1732,8 +1743,9 @@ class ServerAPI(object):
python_version,
platform_name,
python_modules,
runtime_python_modules,
checksum,
checksum_type,
checksum_algorithm,
file_size,
sources=None,
):
@ -1742,6 +1754,10 @@ class ServerAPI(object):
This step will create only metadata. Make sure to upload installer
to the server using 'upload_installer' method.
Runtime python modules are modules that are required to run AYON
desktop application, but are not added to PYTHONPATH for any
subprocess.
Args:
filename (str): Installer filename.
version (str): Version of installer.
@ -1749,8 +1765,10 @@ class ServerAPI(object):
platform_name (str): Name of platform.
python_modules (dict[str, str]): Python modules that are available
in installer.
runtime_python_modules (dict[str, str]): Runtime python modules
that are available in installer.
checksum (str): Installer file checksum.
checksum_type (str): Type of checksum used to create checksum.
checksum_algorithm (str): Type of checksum used to create checksum.
file_size (int): File size.
sources (Optional[list[dict[str, Any]]]): List of sources that
can be used to download file.
@ -1762,8 +1780,9 @@ class ServerAPI(object):
"pythonVersion": python_version,
"platform": platform_name,
"pythonModules": python_modules,
"runtimePythonModules": runtime_python_modules,
"checksum": checksum,
"checksumType": checksum_type,
"checksumAlgorithm": checksum_algorithm,
"size": file_size,
}
if sources:
@ -1781,7 +1800,7 @@ class ServerAPI(object):
can be used to download file. Fully replaces existing sources.
"""
response = self.post(
response = self.patch(
"desktop/installers/{}".format(filename),
sources=sources
)
@ -1794,7 +1813,7 @@ class ServerAPI(object):
filename (str): Installer filename.
"""
response = self.delete("dekstop/installers/{}".format(filename))
response = self.delete("desktop/installers/{}".format(filename))
response.raise_for_status()
def download_installer(
@ -1929,8 +1948,7 @@ class ServerAPI(object):
checksum=checksum,
**kwargs
)
if response.status not in (200, 201, 204):
raise ServerError("Failed to create/update dependency")
response.raise_for_status("Failed to create/update dependency")
return response.data
def get_dependency_packages(self):
@ -2065,8 +2083,7 @@ class ServerAPI(object):
route = self._get_dependency_package_route(filename, platform_name)
response = self.delete(route)
if response.status != 200:
raise ServerError("Failed to delete dependency file")
response.raise_for_status("Failed to delete dependency file")
return response.data
def download_dependency_package(
@ -2131,6 +2148,10 @@ class ServerAPI(object):
def create_dependency_package_basename(self, platform_name=None):
"""Create basename for dependency package file.
Deprecated:
Use 'create_dependency_package_basename' from `ayon_api` or
`ayon_api.utils` instead.
Args:
platform_name (Optional[str]): Name of platform for which the
bundle is targeted. Default value is current platform.
@ -2139,12 +2160,7 @@ class ServerAPI(object):
str: Dependency package name with timestamp and platform.
"""
if platform_name is None:
platform_name = platform.system().lower()
now_date = datetime.datetime.now()
time_stamp = now_date.strftime("%y%m%d%H%M")
return "ayon_{}_{}".format(time_stamp, platform_name)
return create_dependency_package_basename(platform_name)
def _get_bundles_route(self):
major, minor, patch, _, _ = self.server_version_tuple

20
openpype/vendor/python/common/ayon_api/utils.py vendored
View file

@ -2,6 +2,7 @@ import re
import datetime
import uuid
import string
import platform
import collections
try:
# Python 3
@ -449,3 +450,22 @@ class TransferProgress:
destination_url = property(get_destination_url, set_destination_url)
content_size = property(get_content_size, set_content_size)
transferred_size = property(get_transferred_size, set_transferred_size)
def create_dependency_package_basename(platform_name=None):
"""Create basename for dependency package file.
Args:
platform_name (Optional[str]): Name of platform for which the
bundle is targeted. Default value is current platform.
Returns:
str: Dependency package name with timestamp and platform.
"""
if platform_name is None:
platform_name = platform.system().lower()
now_date = datetime.datetime.now()
time_stamp = now_date.strftime("%y%m%d%H%M")
return "ayon_{}_{}".format(time_stamp, platform_name)

2
openpype/vendor/python/common/ayon_api/version.py vendored
View file

@ -1,2 +1,2 @@
"""Package declaring Python API for Ayon server."""
__version__ = "0.3.1"
__version__ = "0.3.2"