diff --git a/common/openpype_common/distribution/README.md b/common/openpype_common/distribution/README.md new file mode 100644 index 0000000000..212eb267b8 --- /dev/null +++ b/common/openpype_common/distribution/README.md @@ -0,0 +1,18 @@ +Addon distribution tool +------------------------ + +Code in this folder is backend portion of Addon distribution logic for v4 server. + +Each host, module will be separate Addon in the future. Each v4 server could run different set of Addons. + +Client (running on artist machine) will in the first step ask v4 for list of enabled addons. +(It expects list of json documents matching to `addon_distribution.py:AddonInfo` object.) +Next it will compare presence of enabled addon version in local folder. In the case of missing version of +an addon, client will use information in the addon to download (from http/shared local disk/git) zip file +and unzip it. + +Required part of addon distribution will be sharing of dependencies (python libraries, utilities) which is not part of this folder. + +Location of this folder might change in the future as it will be required for a clint to add this folder to sys.path reliably. + +This code needs to be independent on Openpype code as much as possible! \ No newline at end of file diff --git a/common/openpype_common/distribution/__init__.py b/common/openpype_common/distribution/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py new file mode 100644 index 0000000000..5e48639dec --- /dev/null +++ b/common/openpype_common/distribution/addon_distribution.py @@ -0,0 +1,208 @@ +import os +from enum import Enum +from abc import abstractmethod +import attr +import logging +import requests +import platform +import shutil + +from .file_handler import RemoteFileHandler +from .addon_info import AddonInfo + + +class UpdateState(Enum): + EXISTS = "exists" + UPDATED = "updated" + FAILED = "failed" + + +class AddonDownloader: + log = logging.getLogger(__name__) + + def __init__(self): + self._downloaders = {} + + def register_format(self, downloader_type, downloader): + self._downloaders[downloader_type.value] = downloader + + def get_downloader(self, downloader_type): + downloader = self._downloaders.get(downloader_type) + if not downloader: + raise ValueError(f"{downloader_type} not implemented") + return downloader() + + @classmethod + @abstractmethod + def download(cls, source, destination): + """Returns url to downloaded addon zip file. + + Args: + source (dict): {type:"http", "url":"https://} ...} + destination (str): local folder to unzip + Returns: + (str) local path to addon zip file + """ + pass + + @classmethod + def check_hash(cls, addon_path, addon_hash): + """Compares 'hash' of downloaded 'addon_url' file. + + Args: + addon_path (str): local path to addon zip file + addon_hash (str): sha256 hash of zip file + Raises: + ValueError if hashes doesn't match + """ + if not os.path.exists(addon_path): + raise ValueError(f"{addon_path} doesn't exist.") + if not RemoteFileHandler.check_integrity(addon_path, + addon_hash, + hash_type="sha256"): + raise ValueError(f"{addon_path} doesn't match expected hash.") + + @classmethod + def unzip(cls, addon_zip_path, destination): + """Unzips local 'addon_zip_path' to 'destination'. + + Args: + addon_zip_path (str): local path to addon zip file + destination (str): local folder to unzip + """ + RemoteFileHandler.unzip(addon_zip_path, destination) + os.remove(addon_zip_path) + + @classmethod + def remove(cls, addon_url): + pass + + +class OSAddonDownloader(AddonDownloader): + + @classmethod + def download(cls, source, destination): + # OS doesnt need to download, unzip directly + addon_url = source["path"].get(platform.system().lower()) + if not os.path.exists(addon_url): + raise ValueError("{} is not accessible".format(addon_url)) + return addon_url + + +class HTTPAddonDownloader(AddonDownloader): + CHUNK_SIZE = 100000 + + @classmethod + def download(cls, source, destination): + source_url = source["url"] + cls.log.debug(f"Downloading {source_url} to {destination}") + file_name = os.path.basename(destination) + _, ext = os.path.splitext(file_name) + if (ext.replace(".", '') not + in set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)): + file_name += ".zip" + RemoteFileHandler.download_url(source_url, + destination, + filename=file_name) + + return os.path.join(destination, file_name) + + +def get_addons_info(server_endpoint): + """Returns list of addon information from Server""" + # TODO temp + # addon_info = AddonInfo( + # **{"name": "openpype_slack", + # "version": "1.0.0", + # "addon_url": "c:/projects/openpype_slack_1.0.0.zip", + # "type": UrlType.FILESYSTEM, + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + # + # http_addon = AddonInfo( + # **{"name": "openpype_slack", + # "version": "1.0.0", + # "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa + # "type": UrlType.HTTP, + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + + response = requests.get(server_endpoint) + if not response.ok: + raise Exception(response.text) + + addons_info = [] + for addon in response.json(): + addons_info.append(AddonInfo(**addon)) + return addons_info + + +def update_addon_state(addon_infos, destination_folder, factory, + log=None): + """Loops through all 'addon_infos', compares local version, unzips. + + Loops through server provided list of dictionaries with information about + available addons. Looks if each addon is already present and deployed. + If isn't, addon zip gets downloaded and unzipped into 'destination_folder'. + Args: + addon_infos (list of AddonInfo) + destination_folder (str): local path + factory (AddonDownloader): factory to get appropriate downloader per + addon type + log (logging.Logger) + Returns: + (dict): {"addon_full_name": UpdateState.value + (eg. "exists"|"updated"|"failed") + """ + if not log: + log = logging.getLogger(__name__) + + download_states = {} + for addon in addon_infos: + full_name = "{}_{}".format(addon.name, addon.version) + addon_dest = os.path.join(destination_folder, full_name) + + if os.path.isdir(addon_dest): + log.debug(f"Addon version folder {addon_dest} already exists.") + download_states[full_name] = UpdateState.EXISTS.value + continue + + for source in addon.sources: + download_states[full_name] = UpdateState.FAILED.value + try: + downloader = factory.get_downloader(source.type) + zip_file_path = downloader.download(attr.asdict(source), + addon_dest) + downloader.check_hash(zip_file_path, addon.hash) + downloader.unzip(zip_file_path, addon_dest) + download_states[full_name] = UpdateState.UPDATED.value + break + except Exception: + log.warning(f"Error happened during updating {addon.name}", + exc_info=True) + if os.path.isdir(addon_dest): + log.debug(f"Cleaning {addon_dest}") + shutil.rmtree(addon_dest) + + return download_states + + +def check_addons(server_endpoint, addon_folder, downloaders): + """Main entry point to compare existing addons with those on server. + + Args: + server_endpoint (str): url to v4 server endpoint + addon_folder (str): local dir path for addons + downloaders (AddonDownloader): factory of downloaders + + Raises: + (RuntimeError) if any addon failed update + """ + addons_info = get_addons_info(server_endpoint) + result = update_addon_state(addons_info, + addon_folder, + downloaders) + if UpdateState.FAILED.value in result.values(): + raise RuntimeError(f"Unable to update some addons {result}") + + +def cli(*args): + raise NotImplementedError diff --git a/common/openpype_common/distribution/addon_info.py b/common/openpype_common/distribution/addon_info.py new file mode 100644 index 0000000000..00ece11f3b --- /dev/null +++ b/common/openpype_common/distribution/addon_info.py @@ -0,0 +1,80 @@ +import attr +from enum import Enum + + +class UrlType(Enum): + HTTP = "http" + GIT = "git" + FILESYSTEM = "filesystem" + + +@attr.s +class MultiPlatformPath(object): + windows = attr.ib(default=None) + linux = attr.ib(default=None) + darwin = attr.ib(default=None) + + +@attr.s +class AddonSource(object): + type = attr.ib() + + +@attr.s +class LocalAddonSource(AddonSource): + path = attr.ib(default=attr.Factory(MultiPlatformPath)) + + +@attr.s +class WebAddonSource(AddonSource): + url = attr.ib(default=None) + + +@attr.s +class VersionData(object): + version_data = attr.ib(default=None) + + +@attr.s +class AddonInfo(object): + """Object matching json payload from Server""" + name = attr.ib() + version = attr.ib() + title = attr.ib(default=None) + sources = attr.ib(default=attr.Factory(dict)) + hash = attr.ib(default=None) + description = attr.ib(default=None) + license = attr.ib(default=None) + authors = attr.ib(default=None) + + @classmethod + def from_dict(cls, data): + sources = [] + + production_version = data.get("productionVersion") + if not production_version: + return + + # server payload contains info about all versions + # active addon must have 'productionVersion' and matching version info + version_data = data.get("versions", {})[production_version] + + for source in version_data.get("clientSourceInfo", []): + if source.get("type") == UrlType.FILESYSTEM.value: + source_addon = LocalAddonSource(type=source["type"], + path=source["path"]) + if source.get("type") == UrlType.HTTP.value: + source_addon = WebAddonSource(type=source["type"], + url=source["url"]) + + sources.append(source_addon) + + return cls(name=data.get("name"), + version=production_version, + sources=sources, + hash=data.get("hash"), + description=data.get("description"), + title=data.get("title"), + license=data.get("license"), + authors=data.get("authors")) + diff --git a/tests/lib/file_handler.py b/common/openpype_common/distribution/file_handler.py similarity index 83% rename from tests/lib/file_handler.py rename to common/openpype_common/distribution/file_handler.py index ee3abc6ecb..f585c77632 100644 --- a/tests/lib/file_handler.py +++ b/common/openpype_common/distribution/file_handler.py @@ -21,7 +21,7 @@ class RemoteFileHandler: 'tar.gz', 'tar.xz', 'tar.bz2'] @staticmethod - def calculate_md5(fpath, chunk_size): + def calculate_md5(fpath, chunk_size=10000): md5 = hashlib.md5() with open(fpath, 'rb') as f: for chunk in iter(lambda: f.read(chunk_size), b''): @@ -33,17 +33,45 @@ class RemoteFileHandler: return md5 == RemoteFileHandler.calculate_md5(fpath, **kwargs) @staticmethod - def check_integrity(fpath, md5=None): + def calculate_sha256(fpath): + """Calculate sha256 for content of the file. + + Args: + fpath (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(fpath, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + + @staticmethod + def check_sha256(fpath, sha256, **kwargs): + return sha256 == RemoteFileHandler.calculate_sha256(fpath, **kwargs) + + @staticmethod + def check_integrity(fpath, hash_value=None, hash_type=None): if not os.path.isfile(fpath): return False - if md5 is None: + if hash_value is None: return True - return RemoteFileHandler.check_md5(fpath, md5) + if not hash_type: + raise ValueError("Provide hash type, md5 or sha256") + if hash_type == 'md5': + return RemoteFileHandler.check_md5(fpath, hash_value) + if hash_type == "sha256": + return RemoteFileHandler.check_sha256(fpath, hash_value) @staticmethod def download_url( url, root, filename=None, - md5=None, max_redirect_hops=3 + sha256=None, max_redirect_hops=3 ): """Download a file from a url and place it in root. Args: @@ -51,7 +79,7 @@ class RemoteFileHandler: root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the basename of the URL - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check max_redirect_hops (int, optional): Maximum number of redirect hops allowed @@ -64,7 +92,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) # check if file is already present locally - if RemoteFileHandler.check_integrity(fpath, md5): + if RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) return @@ -76,7 +105,7 @@ class RemoteFileHandler: file_id = RemoteFileHandler._get_google_drive_file_id(url) if file_id is not None: return RemoteFileHandler.download_file_from_google_drive( - file_id, root, filename, md5) + file_id, root, filename, sha256) # download the file try: @@ -92,20 +121,21 @@ class RemoteFileHandler: raise e # check integrity of downloaded file - if not RemoteFileHandler.check_integrity(fpath, md5): + if not RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): raise RuntimeError("File not found or corrupted.") @staticmethod def download_file_from_google_drive(file_id, root, filename=None, - md5=None): + sha256=None): """Download a Google Drive file from and place it in root. Args: file_id (str): id of file to be downloaded root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the id of the file. - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check """ # Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa @@ -119,8 +149,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) - if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath, - md5): + if os.path.isfile(fpath) and RemoteFileHandler.check_integrity( + fpath, sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) else: session = requests.Session() diff --git a/common/openpype_common/distribution/tests/test_addon_distributtion.py b/common/openpype_common/distribution/tests/test_addon_distributtion.py new file mode 100644 index 0000000000..765ea0596a --- /dev/null +++ b/common/openpype_common/distribution/tests/test_addon_distributtion.py @@ -0,0 +1,167 @@ +import pytest +import attr +import tempfile + +from common.openpype_common.distribution.addon_distribution import ( + AddonDownloader, + OSAddonDownloader, + HTTPAddonDownloader, + AddonInfo, + update_addon_state, + UpdateState +) +from common.openpype_common.distribution.addon_info import UrlType + + +@pytest.fixture +def addon_downloader(): + addon_downloader = AddonDownloader() + addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader) + addon_downloader.register_format(UrlType.HTTP, HTTPAddonDownloader) + + yield addon_downloader + + +@pytest.fixture +def http_downloader(addon_downloader): + yield addon_downloader.get_downloader(UrlType.HTTP.value) + + +@pytest.fixture +def temp_folder(): + yield tempfile.mkdtemp() + + +@pytest.fixture +def sample_addon_info(): + addon_info = { + "versions": { + "1.0.0": { + "clientPyproject": { + "tool": { + "poetry": { + "dependencies": { + "nxtools": "^1.6", + "orjson": "^3.6.7", + "typer": "^0.4.1", + "email-validator": "^1.1.3", + "python": "^3.10", + "fastapi": "^0.73.0" + } + } + } + }, + "hasSettings": True, + "clientSourceInfo": [ + { + "type": "http", + "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" # noqa + }, + { + "type": "filesystem", + "path": { + "windows": ["P:/sources/some_file.zip", + "W:/sources/some_file.zip"], # noqa + "linux": ["/mnt/srv/sources/some_file.zip"], + "darwin": ["/Volumes/srv/sources/some_file.zip"] + } + } + ], + "frontendScopes": { + "project": { + "sidebar": "hierarchy" + } + } + } + }, + "description": "", + "title": "Slack addon", + "name": "openpype_slack", + "productionVersion": "1.0.0", + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa + } + yield addon_info + + +def test_register(printer): + addon_downloader = AddonDownloader() + + assert len(addon_downloader._downloaders) == 0, "Contains registered" + + addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader) + assert len(addon_downloader._downloaders) == 1, "Should contain one" + + +def test_get_downloader(printer, addon_downloader): + assert addon_downloader.get_downloader(UrlType.FILESYSTEM.value), "Should find" # noqa + + with pytest.raises(ValueError): + addon_downloader.get_downloader("unknown"), "Shouldn't find" + + +def test_addon_info(printer, sample_addon_info): + """Tests parsing of expected payload from v4 server into AadonInfo.""" + valid_minimum = { + "name": "openpype_slack", + "productionVersion": "1.0.0", + "versions": { + "1.0.0": { + "clientSourceInfo": [ + { + "type": "filesystem", + "path": { + "windows": [ + "P:/sources/some_file.zip", + "W:/sources/some_file.zip"], + "linux": [ + "/mnt/srv/sources/some_file.zip"], + "darwin": [ + "/Volumes/srv/sources/some_file.zip"] # noqa + } + } + ] + } + } + } + + assert AddonInfo.from_dict(valid_minimum), "Missing required fields" + + valid_minimum["versions"].pop("1.0.0") + with pytest.raises(KeyError): + assert not AddonInfo.from_dict(valid_minimum), "Must fail without version data" # noqa + + valid_minimum.pop("productionVersion") + assert not AddonInfo.from_dict( + valid_minimum), "none if not productionVersion" # noqa + + addon = AddonInfo.from_dict(sample_addon_info) + assert addon, "Should be created" + assert addon.name == "openpype_slack", "Incorrect name" + assert addon.version == "1.0.0", "Incorrect version" + + with pytest.raises(TypeError): + assert addon["name"], "Dict approach not implemented" + + addon_as_dict = attr.asdict(addon) + assert addon_as_dict["name"], "Dict approach should work" + + +def test_update_addon_state(printer, sample_addon_info, + temp_folder, addon_downloader): + """Tests possible cases of addon update.""" + addon_info = AddonInfo.from_dict(sample_addon_info) + orig_hash = addon_info.hash + + addon_info.hash = "brokenhash" + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \ + "Update should failed because of wrong hash" + + addon_info.hash = orig_hash + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \ + "Addon should have been updated" + + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \ + "Addon should already exist" diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index 64676f62f4..85121f0d73 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -10,7 +10,7 @@ import glob import platform from tests.lib.db_handler import DBHandler -from tests.lib.file_handler import RemoteFileHandler +from distribution.file_handler import RemoteFileHandler class BaseTest: