From c4854be5c090cf63f2044fc81b8a7e33ee8c642d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Aug 2022 18:48:54 +0200 Subject: [PATCH 01/30] OP-3682 - extracted sha256 method to lib --- openpype/client/addon_distribution.py | 149 ++++++++++++++++++++++++++ openpype/lib/path_tools.py | 20 ++++ openpype/tools/repack_version.py | 24 +---- 3 files changed, 172 insertions(+), 21 deletions(-) create mode 100644 openpype/client/addon_distribution.py diff --git a/openpype/client/addon_distribution.py b/openpype/client/addon_distribution.py new file mode 100644 index 0000000000..3246c5bb72 --- /dev/null +++ b/openpype/client/addon_distribution.py @@ -0,0 +1,149 @@ +import os +from enum import Enum +from zipfile import ZipFile +from abc import abstractmethod + +import attr + +from openpype.lib.path_tools import sha256sum +from openpype.lib import PypeLogger + +log = PypeLogger().get_logger(__name__) + + +class UrlType(Enum): + HTTP = {} + GIT = {} + OS = {} + + +@attr.s +class AddonInfo(object): + """Object matching json payload from Server""" + name = attr.ib(default=None) + version = attr.ib(default=None) + addon_url = attr.ib(default=None) + type = attr.ib(default=None) + hash = attr.ib(default=None) + + +class AddonDownloader: + + def __init__(self): + self._downloaders = {} + + def register_format(self, downloader_type, downloader): + self._downloaders[downloader_type] = downloader + + def get_downloader(self, downloader_type): + downloader = self._downloaders.get(downloader_type) + if not downloader: + raise ValueError(f"{downloader_type} not implemented") + return downloader() + + @classmethod + @abstractmethod + def download(cls, addon_url, destination): + """Returns url to downloaded addon zip file. + + Args: + addon_url (str): http or OS or any supported protocol url to addon + zip file + destination (str): local folder to unzip + Retursn: + (str) local path to addon zip file + """ + pass + + @classmethod + def check_hash(cls, addon_path, addon_hash): + """Compares 'hash' of downloaded 'addon_url' file. + + Args: + addon_path (str): local path to addon zip file + addon_hash (str): sha256 hash of zip file + Raises: + ValueError if hashes doesn't match + """ + if addon_hash != sha256sum(addon_path): + raise ValueError( + "{} doesn't match expected hash".format(addon_path)) + + @classmethod + def unzip(cls, addon_path, destination): + """Unzips local 'addon_path' to 'destination'. + + Args: + addon_path (str): local path to addon zip file + destination (str): local folder to unzip + """ + addon_file_name = os.path.basename(addon_path) + addon_base_file_name, _ = os.path.splitext(addon_file_name) + with ZipFile(addon_path, "r") as zip_ref: + log.debug(f"Unzipping {addon_path} to {destination}.") + zip_ref.extractall( + os.path.join(destination, addon_base_file_name)) + + @classmethod + def remove(cls, addon_url): + pass + + +class OSAddonDownloader(AddonDownloader): + + @classmethod + def download(cls, addon_url, destination): + # OS doesnt need to download, unzip directly + if not os.path.exists(addon_url): + raise ValueError("{} is not accessible".format(addon_url)) + return addon_url + + +def get_addons_info(): + """Returns list of addon information from Server""" + # TODO temp + addon_info = AddonInfo( + **{"name": "openpype_slack", + "version": "1.0.0", + "addon_url": "c:/projects/openpype_slack_1.0.0.zip", + "type": UrlType.OS, + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + + return [addon_info] + + +def update_addon_state(addon_infos, destination_folder, factory): + """Loops through all 'addon_infos', compares local version, unzips. + + Loops through server provided list of dictionaries with information about + available addons. Looks if each addon is already present and deployed. + If isn't, addon zip gets downloaded and unzipped into 'destination_folder'. + Args: + addon_infos (list of AddonInfo) + destination_folder (str): local path + factory (AddonDownloader): factory to get appropriate downloader per + addon type + """ + for addon in addon_infos: + full_name = "{}_{}".format(addon.name, addon.version) + addon_url = os.path.join(destination_folder, full_name) + + if os.path.isdir(addon_url): + log.debug(f"Addon version folder {addon_url} already exists.") + continue + + downloader = factory.get_downloader(addon.type) + downloader.download(addon.addon_url, destination_folder) + + +def cli(args): + addon_folder = "c:/Users/petrk/AppData/Local/pypeclub/openpype/addons" + + downloader_factory = AddonDownloader() + downloader_factory.register_format(UrlType.OS, OSAddonDownloader) + + print(update_addon_state(get_addons_info(), addon_folder, + downloader_factory)) + print(sha256sum("c:/projects/openpype_slack_1.0.0.zip")) + + diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 4f28be3302..2083dc48d1 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -5,6 +5,7 @@ import json import logging import six import platform +import hashlib from openpype.client import get_project from openpype.settings import get_project_settings @@ -478,3 +479,22 @@ class HostDirmap: log.debug("local sync mapping:: {}".format(mapping)) return mapping + + +def sha256sum(filename): + """Calculate sha256 for content of the file. + + Args: + filename (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() \ No newline at end of file diff --git a/openpype/tools/repack_version.py b/openpype/tools/repack_version.py index 0172264c79..414152970a 100644 --- a/openpype/tools/repack_version.py +++ b/openpype/tools/repack_version.py @@ -7,10 +7,11 @@ from pathlib import Path import platform from zipfile import ZipFile from typing import List -import hashlib import sys from igniter.bootstrap_repos import OpenPypeVersion +from openpype.lib.path_tools import sha256sum + class VersionRepacker: @@ -45,25 +46,6 @@ class VersionRepacker: print("{}{}".format(header, msg)) - @staticmethod - def sha256sum(filename): - """Calculate sha256 for content of the file. - - Args: - filename (str): Path to file. - - Returns: - str: hex encoded sha256 - - """ - h = hashlib.sha256() - b = bytearray(128 * 1024) - mv = memoryview(b) - with open(filename, 'rb', buffering=0) as f: - for n in iter(lambda: f.readinto(mv), 0): - h.update(mv[:n]) - return h.hexdigest() - @staticmethod def _filter_dir(path: Path, path_filter: List) -> List[Path]: """Recursively crawl over path and filter.""" @@ -104,7 +86,7 @@ class VersionRepacker: nits="%", color="green") for file in file_list: checksums.append(( - VersionRepacker.sha256sum(file.as_posix()), + sha256sum(file.as_posix()), file.resolve().relative_to(self.version_path), file )) From 66b280796e30ad89bfc5ef2e43f3f1b677d64a4f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Aug 2022 18:49:24 +0200 Subject: [PATCH 02/30] OP-3682 - implemented local disk downloader --- openpype/client/addon_distribution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/client/addon_distribution.py b/openpype/client/addon_distribution.py index 3246c5bb72..8fe9567688 100644 --- a/openpype/client/addon_distribution.py +++ b/openpype/client/addon_distribution.py @@ -144,6 +144,5 @@ def cli(args): print(update_addon_state(get_addons_info(), addon_folder, downloader_factory)) - print(sha256sum("c:/projects/openpype_slack_1.0.0.zip")) From 159052f8f9555ec1706d2c565b74133c785096ec Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 11:24:41 +0200 Subject: [PATCH 03/30] OP-3682 - Hound --- openpype/lib/path_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 2083dc48d1..0ae5e44d79 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -497,4 +497,4 @@ def sha256sum(filename): with open(filename, 'rb', buffering=0) as f: for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) - return h.hexdigest() \ No newline at end of file + return h.hexdigest() From 27125a1088786f004404302485b750fa1594462d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 12:02:55 +0200 Subject: [PATCH 04/30] OP-3682 - extract file_handler from tests Addon distribution could use already implemented methods for dowloading from HTTP (GDrive urls). --- {tests => openpype}/lib/file_handler.py | 0 tests/lib/testing_classes.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {tests => openpype}/lib/file_handler.py (100%) diff --git a/tests/lib/file_handler.py b/openpype/lib/file_handler.py similarity index 100% rename from tests/lib/file_handler.py rename to openpype/lib/file_handler.py diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index 2b4d7deb48..75f859de48 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -10,7 +10,7 @@ import glob import platform from tests.lib.db_handler import DBHandler -from tests.lib.file_handler import RemoteFileHandler +from openpype.lib.file_handler import RemoteFileHandler from openpype.lib.remote_publish import find_variant_key From 66899d9dd9b50c6bd9285d191fd1da116ab03f4f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 13:04:13 +0200 Subject: [PATCH 05/30] OP-3682 - implemented download from HTTP Handles shared links from GDrive. --- openpype/client/addon_distribution.py | 59 ++++++++++++++++++--------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/openpype/client/addon_distribution.py b/openpype/client/addon_distribution.py index 8fe9567688..de84c7301a 100644 --- a/openpype/client/addon_distribution.py +++ b/openpype/client/addon_distribution.py @@ -7,14 +7,15 @@ import attr from openpype.lib.path_tools import sha256sum from openpype.lib import PypeLogger +from openpype.lib.file_handler import RemoteFileHandler log = PypeLogger().get_logger(__name__) class UrlType(Enum): - HTTP = {} - GIT = {} - OS = {} + HTTP = "http" + GIT = "git" + OS = "os" @attr.s @@ -70,19 +71,15 @@ class AddonDownloader: "{} doesn't match expected hash".format(addon_path)) @classmethod - def unzip(cls, addon_path, destination): - """Unzips local 'addon_path' to 'destination'. + def unzip(cls, addon_zip_path, destination): + """Unzips local 'addon_zip_path' to 'destination'. Args: - addon_path (str): local path to addon zip file + addon_zip_path (str): local path to addon zip file destination (str): local folder to unzip """ - addon_file_name = os.path.basename(addon_path) - addon_base_file_name, _ = os.path.splitext(addon_file_name) - with ZipFile(addon_path, "r") as zip_ref: - log.debug(f"Unzipping {addon_path} to {destination}.") - zip_ref.extractall( - os.path.join(destination, addon_base_file_name)) + RemoteFileHandler.unzip(addon_zip_path, destination) + os.remove(addon_zip_path) @classmethod def remove(cls, addon_url): @@ -99,6 +96,23 @@ class OSAddonDownloader(AddonDownloader): return addon_url +class HTTPAddonDownloader(AddonDownloader): + CHUNK_SIZE = 100000 + + @classmethod + def download(cls, addon_url, destination): + log.debug(f"Downloading {addon_url} to {destination}") + file_name = os.path.basename(destination) + _, ext = os.path.splitext(file_name) + if (ext.replace(".", '') not + in set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)): + file_name += ".zip" + RemoteFileHandler.download_url(addon_url, + destination, + filename=file_name) + + return os.path.join(destination, file_name) + def get_addons_info(): """Returns list of addon information from Server""" # TODO temp @@ -109,7 +123,14 @@ def get_addons_info(): "type": UrlType.OS, "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa - return [addon_info] + http_addon = AddonInfo( + **{"name": "openpype_slack", + "version": "1.0.0", + "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa + "type": UrlType.HTTP, + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + + return [http_addon] def update_addon_state(addon_infos, destination_folder, factory): @@ -126,14 +147,15 @@ def update_addon_state(addon_infos, destination_folder, factory): """ for addon in addon_infos: full_name = "{}_{}".format(addon.name, addon.version) - addon_url = os.path.join(destination_folder, full_name) + addon_dest = os.path.join(destination_folder, full_name) - if os.path.isdir(addon_url): - log.debug(f"Addon version folder {addon_url} already exists.") + if os.path.isdir(addon_dest): + log.debug(f"Addon version folder {addon_dest} already exists.") continue downloader = factory.get_downloader(addon.type) - downloader.download(addon.addon_url, destination_folder) + zip_file_path = downloader.download(addon.addon_url, addon_dest) + downloader.unzip(zip_file_path, addon_dest) def cli(args): @@ -141,8 +163,7 @@ def cli(args): downloader_factory = AddonDownloader() downloader_factory.register_format(UrlType.OS, OSAddonDownloader) + downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader) print(update_addon_state(get_addons_info(), addon_folder, downloader_factory)) - - From bc33432a57bf16245f5bbc9ca22f1f26fbea9dd1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 16:05:56 +0200 Subject: [PATCH 06/30] OP-3682 - updated hash logic Currently only checking hash of zip file. --- openpype/client/addon_distribution.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/openpype/client/addon_distribution.py b/openpype/client/addon_distribution.py index de84c7301a..95c8e7d23f 100644 --- a/openpype/client/addon_distribution.py +++ b/openpype/client/addon_distribution.py @@ -1,6 +1,5 @@ import os from enum import Enum -from zipfile import ZipFile from abc import abstractmethod import attr @@ -66,9 +65,10 @@ class AddonDownloader: Raises: ValueError if hashes doesn't match """ + if not os.path.exists(addon_path): + raise ValueError(f"{addon_path} doesn't exist.") if addon_hash != sha256sum(addon_path): - raise ValueError( - "{} doesn't match expected hash".format(addon_path)) + raise ValueError(f"{addon_path} doesn't match expected hash.") @classmethod def unzip(cls, addon_zip_path, destination): @@ -153,9 +153,14 @@ def update_addon_state(addon_infos, destination_folder, factory): log.debug(f"Addon version folder {addon_dest} already exists.") continue - downloader = factory.get_downloader(addon.type) - zip_file_path = downloader.download(addon.addon_url, addon_dest) - downloader.unzip(zip_file_path, addon_dest) + try: + downloader = factory.get_downloader(addon.type) + zip_file_path = downloader.download(addon.addon_url, addon_dest) + downloader.check_hash(zip_file_path, addon.hash) + downloader.unzip(zip_file_path, addon_dest) + except Exception: + log.warning(f"Error happened during updating {addon.name}", + stack_info=True) def cli(args): From ceeb652699a4dd5a2ceb2ccba15ae84f57684e07 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 16:36:55 +0200 Subject: [PATCH 07/30] OP-3682 - changed logging method PypeLogger is obsolete --- openpype/client/addon_distribution.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/openpype/client/addon_distribution.py b/openpype/client/addon_distribution.py index 95c8e7d23f..46098cfa11 100644 --- a/openpype/client/addon_distribution.py +++ b/openpype/client/addon_distribution.py @@ -1,14 +1,11 @@ import os from enum import Enum from abc import abstractmethod - import attr from openpype.lib.path_tools import sha256sum -from openpype.lib import PypeLogger from openpype.lib.file_handler import RemoteFileHandler - -log = PypeLogger().get_logger(__name__) +from openpype.lib import Logger class UrlType(Enum): @@ -28,6 +25,7 @@ class AddonInfo(object): class AddonDownloader: + log = Logger.get_logger(__name__) def __init__(self): self._downloaders = {} @@ -101,7 +99,7 @@ class HTTPAddonDownloader(AddonDownloader): @classmethod def download(cls, addon_url, destination): - log.debug(f"Downloading {addon_url} to {destination}") + cls.log.debug(f"Downloading {addon_url} to {destination}") file_name = os.path.basename(destination) _, ext = os.path.splitext(file_name) if (ext.replace(".", '') not @@ -113,6 +111,7 @@ class HTTPAddonDownloader(AddonDownloader): return os.path.join(destination, file_name) + def get_addons_info(): """Returns list of addon information from Server""" # TODO temp @@ -145,6 +144,10 @@ def update_addon_state(addon_infos, destination_folder, factory): factory (AddonDownloader): factory to get appropriate downloader per addon type """ + from openpype.lib import Logger + + log = Logger.get_logger(__name__) + for addon in addon_infos: full_name = "{}_{}".format(addon.name, addon.version) addon_dest = os.path.join(destination_folder, full_name) From 542eedb4b299aeab0a6e74a361e72e3961c17bfb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 17:19:58 +0200 Subject: [PATCH 08/30] OP-3682 - moved file to distribution folder Needs to be separate from Openpype. Igniter and Openpype (and tests) could import from this if necessary. --- {openpype/client => distribution}/addon_distribution.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {openpype/client => distribution}/addon_distribution.py (100%) diff --git a/openpype/client/addon_distribution.py b/distribution/addon_distribution.py similarity index 100% rename from openpype/client/addon_distribution.py rename to distribution/addon_distribution.py From cfbc9b00777073b945b1ec25e18c32b89127ed7c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 17:24:38 +0200 Subject: [PATCH 09/30] OP-3682 - replaced Logger to logging Shouldn't import anything from Openpype --- distribution/addon_distribution.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 46098cfa11..b76cd8e3f8 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -2,10 +2,10 @@ import os from enum import Enum from abc import abstractmethod import attr +import logging from openpype.lib.path_tools import sha256sum from openpype.lib.file_handler import RemoteFileHandler -from openpype.lib import Logger class UrlType(Enum): @@ -25,7 +25,7 @@ class AddonInfo(object): class AddonDownloader: - log = Logger.get_logger(__name__) + log = logging.getLogger(__name__) def __init__(self): self._downloaders = {} @@ -132,7 +132,8 @@ def get_addons_info(): return [http_addon] -def update_addon_state(addon_infos, destination_folder, factory): +def update_addon_state(addon_infos, destination_folder, factory, + log=None): """Loops through all 'addon_infos', compares local version, unzips. Loops through server provided list of dictionaries with information about @@ -143,10 +144,10 @@ def update_addon_state(addon_infos, destination_folder, factory): destination_folder (str): local path factory (AddonDownloader): factory to get appropriate downloader per addon type + log (logging.Logger) """ - from openpype.lib import Logger - - log = Logger.get_logger(__name__) + if not log: + log = logging.getLogger(__name__) for addon in addon_infos: full_name = "{}_{}".format(addon.name, addon.version) From 98444762cd97da52e62370677762a82b25b850c8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 17:27:14 +0200 Subject: [PATCH 10/30] OP-3682 - moved file_handler --- distribution/addon_distribution.py | 5 ++--- {openpype/lib => distribution}/file_handler.py | 2 +- tests/lib/testing_classes.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) rename {openpype/lib => distribution}/file_handler.py (99%) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index b76cd8e3f8..e29e9bbf9b 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -4,8 +4,7 @@ from abc import abstractmethod import attr import logging -from openpype.lib.path_tools import sha256sum -from openpype.lib.file_handler import RemoteFileHandler +from distribution.file_handler import RemoteFileHandler class UrlType(Enum): @@ -65,7 +64,7 @@ class AddonDownloader: """ if not os.path.exists(addon_path): raise ValueError(f"{addon_path} doesn't exist.") - if addon_hash != sha256sum(addon_path): + if addon_hash != RemoteFileHandler.calculate_md5(addon_path): raise ValueError(f"{addon_path} doesn't match expected hash.") @classmethod diff --git a/openpype/lib/file_handler.py b/distribution/file_handler.py similarity index 99% rename from openpype/lib/file_handler.py rename to distribution/file_handler.py index ee3abc6ecb..8c8b4230ce 100644 --- a/openpype/lib/file_handler.py +++ b/distribution/file_handler.py @@ -21,7 +21,7 @@ class RemoteFileHandler: 'tar.gz', 'tar.xz', 'tar.bz2'] @staticmethod - def calculate_md5(fpath, chunk_size): + def calculate_md5(fpath, chunk_size=10000): md5 = hashlib.md5() with open(fpath, 'rb') as f: for chunk in iter(lambda: f.read(chunk_size), b''): diff --git a/tests/lib/testing_classes.py b/tests/lib/testing_classes.py index 75f859de48..e819ae80de 100644 --- a/tests/lib/testing_classes.py +++ b/tests/lib/testing_classes.py @@ -10,7 +10,7 @@ import glob import platform from tests.lib.db_handler import DBHandler -from openpype.lib.file_handler import RemoteFileHandler +from distribution.file_handler import RemoteFileHandler from openpype.lib.remote_publish import find_variant_key From b0c8a47f0f27a734f8ba9f201ae08dabe5d1271d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 17:32:22 +0200 Subject: [PATCH 11/30] Revert "OP-3682 - extracted sha256 method to lib" This reverts commit c4854be5 --- openpype/lib/path_tools.py | 19 ------------------- openpype/tools/repack_version.py | 24 +++++++++++++++++++++--- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 0ae5e44d79..11648f9969 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -5,7 +5,6 @@ import json import logging import six import platform -import hashlib from openpype.client import get_project from openpype.settings import get_project_settings @@ -480,21 +479,3 @@ class HostDirmap: log.debug("local sync mapping:: {}".format(mapping)) return mapping - -def sha256sum(filename): - """Calculate sha256 for content of the file. - - Args: - filename (str): Path to file. - - Returns: - str: hex encoded sha256 - - """ - h = hashlib.sha256() - b = bytearray(128 * 1024) - mv = memoryview(b) - with open(filename, 'rb', buffering=0) as f: - for n in iter(lambda: f.readinto(mv), 0): - h.update(mv[:n]) - return h.hexdigest() diff --git a/openpype/tools/repack_version.py b/openpype/tools/repack_version.py index 414152970a..0172264c79 100644 --- a/openpype/tools/repack_version.py +++ b/openpype/tools/repack_version.py @@ -7,11 +7,10 @@ from pathlib import Path import platform from zipfile import ZipFile from typing import List +import hashlib import sys from igniter.bootstrap_repos import OpenPypeVersion -from openpype.lib.path_tools import sha256sum - class VersionRepacker: @@ -46,6 +45,25 @@ class VersionRepacker: print("{}{}".format(header, msg)) + @staticmethod + def sha256sum(filename): + """Calculate sha256 for content of the file. + + Args: + filename (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + @staticmethod def _filter_dir(path: Path, path_filter: List) -> List[Path]: """Recursively crawl over path and filter.""" @@ -86,7 +104,7 @@ class VersionRepacker: nits="%", color="green") for file in file_list: checksums.append(( - sha256sum(file.as_posix()), + VersionRepacker.sha256sum(file.as_posix()), file.resolve().relative_to(self.version_path), file )) From 0fb8988522a328afa33cc08960a8a2a678e2b26c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Aug 2022 17:35:12 +0200 Subject: [PATCH 12/30] OP-3682 - Hound --- openpype/lib/path_tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openpype/lib/path_tools.py b/openpype/lib/path_tools.py index 11648f9969..4f28be3302 100644 --- a/openpype/lib/path_tools.py +++ b/openpype/lib/path_tools.py @@ -478,4 +478,3 @@ class HostDirmap: log.debug("local sync mapping:: {}".format(mapping)) return mapping - From 3252c732e1c41460016bc8a17ab01ea5ade34f60 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Aug 2022 18:28:14 +0200 Subject: [PATCH 13/30] OP-3682 - added more fields to metadata Additional fields could be useful in the future for some addon Store or pulling information into customer's internal CRM. --- distribution/addon_distribution.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index e29e9bbf9b..465950f6e8 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -3,6 +3,7 @@ from enum import Enum from abc import abstractmethod import attr import logging +import requests from distribution.file_handler import RemoteFileHandler @@ -21,6 +22,9 @@ class AddonInfo(object): addon_url = attr.ib(default=None) type = attr.ib(default=None) hash = attr.ib(default=None) + description = attr.ib(default=None) + license = attr.ib(default=None) + authors = attr.ib(default=None) class AddonDownloader: From 6187cf18f50c43ca60f234bce439a4dc466263cf Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Aug 2022 19:08:30 +0200 Subject: [PATCH 14/30] OP-3682 - implemented basic GET Used publish Postman mock server for testing --- distribution/addon_distribution.py | 59 ++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 465950f6e8..86b6de3a74 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -34,7 +34,7 @@ class AddonDownloader: self._downloaders = {} def register_format(self, downloader_type, downloader): - self._downloaders[downloader_type] = downloader + self._downloaders[downloader_type.value] = downloader def get_downloader(self, downloader_type): downloader = self._downloaders.get(downloader_type) @@ -115,24 +115,31 @@ class HTTPAddonDownloader(AddonDownloader): return os.path.join(destination, file_name) -def get_addons_info(): +def get_addons_info(server_endpoint): """Returns list of addon information from Server""" # TODO temp - addon_info = AddonInfo( - **{"name": "openpype_slack", - "version": "1.0.0", - "addon_url": "c:/projects/openpype_slack_1.0.0.zip", - "type": UrlType.OS, - "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + # addon_info = AddonInfo( + # **{"name": "openpype_slack", + # "version": "1.0.0", + # "addon_url": "c:/projects/openpype_slack_1.0.0.zip", + # "type": UrlType.OS, + # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa + # + # http_addon = AddonInfo( + # **{"name": "openpype_slack", + # "version": "1.0.0", + # "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa + # "type": UrlType.HTTP, + # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa - http_addon = AddonInfo( - **{"name": "openpype_slack", - "version": "1.0.0", - "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa - "type": UrlType.HTTP, - "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa + response = requests.get(server_endpoint) + if not response.ok: + raise Exception(response.text) - return [http_addon] + addons_info = [] + for addon in response.json(): + addons_info.append(AddonInfo(**addon)) + return addons_info def update_addon_state(addon_infos, destination_folder, factory, @@ -167,15 +174,29 @@ def update_addon_state(addon_infos, destination_folder, factory, downloader.unzip(zip_file_path, addon_dest) except Exception: log.warning(f"Error happened during updating {addon.name}", - stack_info=True) + exc_info=True) + + +def check_addons(server_endpoint, addon_folder, downloaders): + """Main entry point to compare existing addons with those on server.""" + addons_info = get_addons_info(server_endpoint) + update_addon_state(addons_info, + addon_folder, + downloaders) def cli(args): - addon_folder = "c:/Users/petrk/AppData/Local/pypeclub/openpype/addons" + addon_folder = "c:/projects/testing_addons/pypeclub/openpype/addons" downloader_factory = AddonDownloader() downloader_factory.register_format(UrlType.OS, OSAddonDownloader) downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader) - print(update_addon_state(get_addons_info(), addon_folder, - downloader_factory)) + test_endpoint = "https://34e99f0f-f987-4715-95e6-d2d88caa7586.mock.pstmn.io/get_addons_info" # noqa + if os.environ.get("OPENPYPE_SERVER"): # TODO or from keychain + server_endpoint = os.environ.get("OPENPYPE_SERVER") + "get_addons_info" + else: + server_endpoint = test_endpoint + + check_addons(server_endpoint, addon_folder, downloader_factory) + From 385b6b97f02c2a384e3432fd8f204ee4f6810e18 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Aug 2022 19:09:06 +0200 Subject: [PATCH 15/30] OP-3682 - Hound --- distribution/addon_distribution.py | 1 - 1 file changed, 1 deletion(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 86b6de3a74..a0c48923df 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -199,4 +199,3 @@ def cli(args): server_endpoint = test_endpoint check_addons(server_endpoint, addon_folder, downloader_factory) - From 538513304e9b3ebcd433765881a620a0e00bc48c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 16 Aug 2022 13:20:25 +0200 Subject: [PATCH 16/30] OP-3682 - refactored OS to FILESYSTEM --- distribution/addon_distribution.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index a0c48923df..3cc2374b93 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -11,7 +11,7 @@ from distribution.file_handler import RemoteFileHandler class UrlType(Enum): HTTP = "http" GIT = "git" - OS = "os" + FILESYSTEM = "filesystem" @attr.s @@ -122,7 +122,7 @@ def get_addons_info(server_endpoint): # **{"name": "openpype_slack", # "version": "1.0.0", # "addon_url": "c:/projects/openpype_slack_1.0.0.zip", - # "type": UrlType.OS, + # "type": UrlType.FILESYSTEM, # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa # # http_addon = AddonInfo( @@ -189,7 +189,7 @@ def cli(args): addon_folder = "c:/projects/testing_addons/pypeclub/openpype/addons" downloader_factory = AddonDownloader() - downloader_factory.register_format(UrlType.OS, OSAddonDownloader) + downloader_factory.register_format(UrlType.FILESYSTEM, OSAddonDownloader) downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader) test_endpoint = "https://34e99f0f-f987-4715-95e6-d2d88caa7586.mock.pstmn.io/get_addons_info" # noqa From 1615f74af3e8daddb8c625329f67f0756deb9bc9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Aug 2022 15:40:02 +0200 Subject: [PATCH 17/30] OP-3214 - updated format of addon info response When downloading it should go through each source until it succeeds --- distribution/addon_distribution.py | 37 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 3cc2374b93..2efbb34274 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -4,6 +4,7 @@ from abc import abstractmethod import attr import logging import requests +import platform from distribution.file_handler import RemoteFileHandler @@ -14,13 +15,26 @@ class UrlType(Enum): FILESYSTEM = "filesystem" +@attr.s +class MultiPlatformPath(object): + windows = attr.ib(default=None) + linux = attr.ib(default=None) + darwin = attr.ib(default=None) + + +@attr.s +class AddonSource(object): + type = attr.ib() + url = attr.ib(default=None) + path = attr.ib(default=attr.Factory(MultiPlatformPath)) + + @attr.s class AddonInfo(object): """Object matching json payload from Server""" - name = attr.ib(default=None) - version = attr.ib(default=None) - addon_url = attr.ib(default=None) - type = attr.ib(default=None) + name = attr.ib() + version = attr.ib() + sources = attr.ib(default=attr.Factory(list), type=AddonSource) hash = attr.ib(default=None) description = attr.ib(default=None) license = attr.ib(default=None) @@ -44,12 +58,11 @@ class AddonDownloader: @classmethod @abstractmethod - def download(cls, addon_url, destination): + def download(cls, source, destination): """Returns url to downloaded addon zip file. Args: - addon_url (str): http or OS or any supported protocol url to addon - zip file + source (dict): {type:"http", "url":"https://} ...} destination (str): local folder to unzip Retursn: (str) local path to addon zip file @@ -90,8 +103,9 @@ class AddonDownloader: class OSAddonDownloader(AddonDownloader): @classmethod - def download(cls, addon_url, destination): + def download(cls, source, destination): # OS doesnt need to download, unzip directly + addon_url = source["path"].get(platform.system().lower()) if not os.path.exists(addon_url): raise ValueError("{} is not accessible".format(addon_url)) return addon_url @@ -101,14 +115,15 @@ class HTTPAddonDownloader(AddonDownloader): CHUNK_SIZE = 100000 @classmethod - def download(cls, addon_url, destination): - cls.log.debug(f"Downloading {addon_url} to {destination}") + def download(cls, source, destination): + source_url = source["url"] + cls.log.debug(f"Downloading {source_url} to {destination}") file_name = os.path.basename(destination) _, ext = os.path.splitext(file_name) if (ext.replace(".", '') not in set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)): file_name += ".zip" - RemoteFileHandler.download_url(addon_url, + RemoteFileHandler.download_url(source_url, destination, filename=file_name) From 34c15c24292a3ae434b509987acb9b28f8176106 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Aug 2022 15:41:03 +0200 Subject: [PATCH 18/30] OP-3214 - fixed update_addon_state Should be able to update whatever can. --- distribution/addon_distribution.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 2efbb34274..f5af0f77ed 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -170,26 +170,36 @@ def update_addon_state(addon_infos, destination_folder, factory, factory (AddonDownloader): factory to get appropriate downloader per addon type log (logging.Logger) + Returns: + (dict): {"addon_full_name":"exists"|"updated"|"failed" """ if not log: log = logging.getLogger(__name__) + download_states = {} for addon in addon_infos: full_name = "{}_{}".format(addon.name, addon.version) addon_dest = os.path.join(destination_folder, full_name) if os.path.isdir(addon_dest): log.debug(f"Addon version folder {addon_dest} already exists.") + download_states[full_name] = "exists" continue - try: - downloader = factory.get_downloader(addon.type) - zip_file_path = downloader.download(addon.addon_url, addon_dest) - downloader.check_hash(zip_file_path, addon.hash) - downloader.unzip(zip_file_path, addon_dest) - except Exception: - log.warning(f"Error happened during updating {addon.name}", - exc_info=True) + for source in addon.sources: + download_states[full_name] = "failed" + try: + downloader = factory.get_downloader(source["type"]) + zip_file_path = downloader.download(source, addon_dest) + downloader.check_hash(zip_file_path, addon.hash) + downloader.unzip(zip_file_path, addon_dest) + download_states[full_name] = "updated" + break + except Exception: + log.warning(f"Error happened during updating {addon.name}", + exc_info=True) + + return download_states def check_addons(server_endpoint, addon_folder, downloaders): From 437ead97762c861172f19901d0d83d8ea11b7b2b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Aug 2022 15:54:06 +0200 Subject: [PATCH 19/30] OP-3214 - introduced update state enum --- distribution/addon_distribution.py | 33 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index f5af0f77ed..4ca3f5687a 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -15,6 +15,11 @@ class UrlType(Enum): FILESYSTEM = "filesystem" +class UpdateState(Enum): + EXISTS = "exists" + UPDATED = "updated" + FAILED = "failed" + @attr.s class MultiPlatformPath(object): windows = attr.ib(default=None) @@ -171,7 +176,8 @@ def update_addon_state(addon_infos, destination_folder, factory, addon type log (logging.Logger) Returns: - (dict): {"addon_full_name":"exists"|"updated"|"failed" + (dict): {"addon_full_name": UpdateState.value + (eg. "exists"|"updated"|"failed") """ if not log: log = logging.getLogger(__name__) @@ -183,17 +189,17 @@ def update_addon_state(addon_infos, destination_folder, factory, if os.path.isdir(addon_dest): log.debug(f"Addon version folder {addon_dest} already exists.") - download_states[full_name] = "exists" + download_states[full_name] = UpdateState.EXISTS.value continue for source in addon.sources: - download_states[full_name] = "failed" + download_states[full_name] = UpdateState.FAILED.value try: downloader = factory.get_downloader(source["type"]) zip_file_path = downloader.download(source, addon_dest) downloader.check_hash(zip_file_path, addon.hash) downloader.unzip(zip_file_path, addon_dest) - download_states[full_name] = "updated" + download_states[full_name] = UpdateState.UPDATED.value break except Exception: log.warning(f"Error happened during updating {addon.name}", @@ -203,11 +209,22 @@ def update_addon_state(addon_infos, destination_folder, factory, def check_addons(server_endpoint, addon_folder, downloaders): - """Main entry point to compare existing addons with those on server.""" + """Main entry point to compare existing addons with those on server. + + Args: + server_endpoint (str): url to v4 server endpoint + addon_folder (str): local dir path for addons + downloaders (AddonDownloader): factory of downloaders + + Raises: + (RuntimeError) if any addon failed update + """ addons_info = get_addons_info(server_endpoint) - update_addon_state(addons_info, - addon_folder, - downloaders) + result = update_addon_state(addons_info, + addon_folder, + downloaders) + if UpdateState.FAILED.value in result.values(): + raise RuntimeError(f"Unable to update some addons {result}") def cli(args): From 0d495a36834d0e5aec062841c3f3820f68900c0a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Aug 2022 16:01:37 +0200 Subject: [PATCH 20/30] OP-3214 - added unit tests --- distribution/__init__.py | 0 .../tests/test_addon_distributtion.py | 121 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 distribution/__init__.py create mode 100644 distribution/tests/test_addon_distributtion.py diff --git a/distribution/__init__.py b/distribution/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/distribution/tests/test_addon_distributtion.py b/distribution/tests/test_addon_distributtion.py new file mode 100644 index 0000000000..2e81bc4ef9 --- /dev/null +++ b/distribution/tests/test_addon_distributtion.py @@ -0,0 +1,121 @@ +import pytest +import attr +import tempfile + +from distribution.addon_distribution import ( + AddonDownloader, + UrlType, + OSAddonDownloader, + HTTPAddonDownloader, + AddonInfo, + update_addon_state, + UpdateState +) + + +@pytest.fixture +def addon_downloader(): + addon_downloader = AddonDownloader() + addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader) + addon_downloader.register_format(UrlType.HTTP, HTTPAddonDownloader) + + yield addon_downloader + + +@pytest.fixture +def http_downloader(addon_downloader): + yield addon_downloader.get_downloader(UrlType.HTTP.value) + + +@pytest.fixture +def temp_folder(): + yield tempfile.mkdtemp() + + +@pytest.fixture +def sample_addon_info(): + addon_info = { + "name": "openpype_slack", + "version": "1.0.0", + "sources": [ + { + "type": "http", + "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" + }, + { + "type": "filesystem", + "path": { + "windows": ["P:/sources/some_file.zip", "W:/sources/some_file.zip"], + "linux": ["/mnt/srv/sources/some_file.zip"], + "darwin": ["/Volumes/srv/sources/some_file.zip"] + } + } + ], + "hash": "4f6b8568eb9dd6f510fd7c4dcb676788" + } + yield addon_info + + +def test_register(printer): + addon_downloader = AddonDownloader() + + assert len(addon_downloader._downloaders) == 0, "Contains registered" + + addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader) + assert len(addon_downloader._downloaders) == 1, "Should contain one" + + +def test_get_downloader(printer, addon_downloader): + assert addon_downloader.get_downloader(UrlType.FILESYSTEM.value), "Should find" # noqa + + with pytest.raises(ValueError): + addon_downloader.get_downloader("unknown"), "Shouldn't find" + + +def test_addon_info(printer, sample_addon_info): + valid_minimum = {"name": "openpype_slack", "version": "1.0.0"} + + assert AddonInfo(**valid_minimum), "Missing required fields" + assert AddonInfo(name=valid_minimum["name"], + version=valid_minimum["version"]), \ + "Missing required fields" + + with pytest.raises(TypeError): + # TODO should be probably implemented + assert AddonInfo(valid_minimum), "Wrong argument format" + + addon = AddonInfo(**sample_addon_info) + assert addon, "Should be created" + assert addon.name == "openpype_slack", "Incorrect name" + assert addon.version == "1.0.0", "Incorrect version" + + with pytest.raises(TypeError): + assert addon["name"], "Dict approach not implemented" + + addon_as_dict = attr.asdict(addon) + assert addon_as_dict["name"], "Dict approach should work" + + with pytest.raises(AttributeError): + # TODO should be probably implemented as . not dict + first_source = addon.sources[0] + assert first_source.type == "http", "Not implemented" + + +def test_update_addon_state(printer, sample_addon_info, + temp_folder, addon_downloader): + addon_info = AddonInfo(**sample_addon_info) + orig_hash = addon_info.hash + + addon_info.hash = "brokenhash" + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert (result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, + "Hashes not matching") + + addon_info.hash = orig_hash + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert (result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, + "Failed updating") + + result = update_addon_state([addon_info], temp_folder, addon_downloader) + assert (result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, + "Tried to update") From be5dbd6512362c58abb5ec9415414903e8badb20 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 24 Aug 2022 16:03:57 +0200 Subject: [PATCH 21/30] OP-3214 - Hound --- distribution/addon_distribution.py | 1 + distribution/tests/test_addon_distributtion.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 4ca3f5687a..95d0b5e397 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -20,6 +20,7 @@ class UpdateState(Enum): UPDATED = "updated" FAILED = "failed" + @attr.s class MultiPlatformPath(object): windows = attr.ib(default=None) diff --git a/distribution/tests/test_addon_distributtion.py b/distribution/tests/test_addon_distributtion.py index 2e81bc4ef9..e67ca3c479 100644 --- a/distribution/tests/test_addon_distributtion.py +++ b/distribution/tests/test_addon_distributtion.py @@ -40,12 +40,12 @@ def sample_addon_info(): "sources": [ { "type": "http", - "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" + "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" # noqa }, { "type": "filesystem", "path": { - "windows": ["P:/sources/some_file.zip", "W:/sources/some_file.zip"], + "windows": ["P:/sources/some_file.zip", "W:/sources/some_file.zip"], # noqa "linux": ["/mnt/srv/sources/some_file.zip"], "darwin": ["/Volumes/srv/sources/some_file.zip"] } @@ -108,14 +108,14 @@ def test_update_addon_state(printer, sample_addon_info, addon_info.hash = "brokenhash" result = update_addon_state([addon_info], temp_folder, addon_downloader) - assert (result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, - "Hashes not matching") + assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \ + "Hashes not matching" addon_info.hash = orig_hash result = update_addon_state([addon_info], temp_folder, addon_downloader) - assert (result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, - "Failed updating") + assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \ + "Failed updating" result = update_addon_state([addon_info], temp_folder, addon_downloader) - assert (result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, - "Tried to update") + assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \ + "Tried to update" From 390dbb6320f97ba1b05e1de895905b57c62ce1e3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 25 Aug 2022 14:51:21 +0200 Subject: [PATCH 22/30] OP-3682 - added readme to highlight it is for v4 --- distribution/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 distribution/README.md diff --git a/distribution/README.md b/distribution/README.md new file mode 100644 index 0000000000..212eb267b8 --- /dev/null +++ b/distribution/README.md @@ -0,0 +1,18 @@ +Addon distribution tool +------------------------ + +Code in this folder is backend portion of Addon distribution logic for v4 server. + +Each host, module will be separate Addon in the future. Each v4 server could run different set of Addons. + +Client (running on artist machine) will in the first step ask v4 for list of enabled addons. +(It expects list of json documents matching to `addon_distribution.py:AddonInfo` object.) +Next it will compare presence of enabled addon version in local folder. In the case of missing version of +an addon, client will use information in the addon to download (from http/shared local disk/git) zip file +and unzip it. + +Required part of addon distribution will be sharing of dependencies (python libraries, utilities) which is not part of this folder. + +Location of this folder might change in the future as it will be required for a clint to add this folder to sys.path reliably. + +This code needs to be independent on Openpype code as much as possible! \ No newline at end of file From ae88578dbda9eca89cc792cec498d19c7ef3d6af Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 7 Sep 2022 14:38:45 +0200 Subject: [PATCH 23/30] OP-3682 - changed md5 to sha256 Updated tests. Removed test cli method --- distribution/addon_distribution.py | 28 ++++------ distribution/file_handler.py | 54 ++++++++++++++----- .../tests/test_addon_distributtion.py | 8 +-- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 95d0b5e397..0e3c672915 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -5,6 +5,7 @@ import attr import logging import requests import platform +import shutil from distribution.file_handler import RemoteFileHandler @@ -87,7 +88,9 @@ class AddonDownloader: """ if not os.path.exists(addon_path): raise ValueError(f"{addon_path} doesn't exist.") - if addon_hash != RemoteFileHandler.calculate_md5(addon_path): + if not RemoteFileHandler.check_integrity(addon_path, + addon_hash, + hash_type="sha256"): raise ValueError(f"{addon_path} doesn't match expected hash.") @classmethod @@ -144,14 +147,14 @@ def get_addons_info(server_endpoint): # "version": "1.0.0", # "addon_url": "c:/projects/openpype_slack_1.0.0.zip", # "type": UrlType.FILESYSTEM, - # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa # # http_addon = AddonInfo( # **{"name": "openpype_slack", # "version": "1.0.0", # "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa # "type": UrlType.HTTP, - # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa response = requests.get(server_endpoint) if not response.ok: @@ -205,6 +208,9 @@ def update_addon_state(addon_infos, destination_folder, factory, except Exception: log.warning(f"Error happened during updating {addon.name}", exc_info=True) + if os.path.isdir(addon_dest): + log.debug(f"Cleaning {addon_dest}") + shutil.rmtree(addon_dest) return download_states @@ -228,17 +234,5 @@ def check_addons(server_endpoint, addon_folder, downloaders): raise RuntimeError(f"Unable to update some addons {result}") -def cli(args): - addon_folder = "c:/projects/testing_addons/pypeclub/openpype/addons" - - downloader_factory = AddonDownloader() - downloader_factory.register_format(UrlType.FILESYSTEM, OSAddonDownloader) - downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader) - - test_endpoint = "https://34e99f0f-f987-4715-95e6-d2d88caa7586.mock.pstmn.io/get_addons_info" # noqa - if os.environ.get("OPENPYPE_SERVER"): # TODO or from keychain - server_endpoint = os.environ.get("OPENPYPE_SERVER") + "get_addons_info" - else: - server_endpoint = test_endpoint - - check_addons(server_endpoint, addon_folder, downloader_factory) +def cli(*args): + raise NotImplemented \ No newline at end of file diff --git a/distribution/file_handler.py b/distribution/file_handler.py index 8c8b4230ce..f585c77632 100644 --- a/distribution/file_handler.py +++ b/distribution/file_handler.py @@ -33,17 +33,45 @@ class RemoteFileHandler: return md5 == RemoteFileHandler.calculate_md5(fpath, **kwargs) @staticmethod - def check_integrity(fpath, md5=None): + def calculate_sha256(fpath): + """Calculate sha256 for content of the file. + + Args: + fpath (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(fpath, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + + @staticmethod + def check_sha256(fpath, sha256, **kwargs): + return sha256 == RemoteFileHandler.calculate_sha256(fpath, **kwargs) + + @staticmethod + def check_integrity(fpath, hash_value=None, hash_type=None): if not os.path.isfile(fpath): return False - if md5 is None: + if hash_value is None: return True - return RemoteFileHandler.check_md5(fpath, md5) + if not hash_type: + raise ValueError("Provide hash type, md5 or sha256") + if hash_type == 'md5': + return RemoteFileHandler.check_md5(fpath, hash_value) + if hash_type == "sha256": + return RemoteFileHandler.check_sha256(fpath, hash_value) @staticmethod def download_url( url, root, filename=None, - md5=None, max_redirect_hops=3 + sha256=None, max_redirect_hops=3 ): """Download a file from a url and place it in root. Args: @@ -51,7 +79,7 @@ class RemoteFileHandler: root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the basename of the URL - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check max_redirect_hops (int, optional): Maximum number of redirect hops allowed @@ -64,7 +92,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) # check if file is already present locally - if RemoteFileHandler.check_integrity(fpath, md5): + if RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) return @@ -76,7 +105,7 @@ class RemoteFileHandler: file_id = RemoteFileHandler._get_google_drive_file_id(url) if file_id is not None: return RemoteFileHandler.download_file_from_google_drive( - file_id, root, filename, md5) + file_id, root, filename, sha256) # download the file try: @@ -92,20 +121,21 @@ class RemoteFileHandler: raise e # check integrity of downloaded file - if not RemoteFileHandler.check_integrity(fpath, md5): + if not RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): raise RuntimeError("File not found or corrupted.") @staticmethod def download_file_from_google_drive(file_id, root, filename=None, - md5=None): + sha256=None): """Download a Google Drive file from and place it in root. Args: file_id (str): id of file to be downloaded root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the id of the file. - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check """ # Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa @@ -119,8 +149,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) - if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath, - md5): + if os.path.isfile(fpath) and RemoteFileHandler.check_integrity( + fpath, sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) else: session = requests.Session() diff --git a/distribution/tests/test_addon_distributtion.py b/distribution/tests/test_addon_distributtion.py index e67ca3c479..717ef1330e 100644 --- a/distribution/tests/test_addon_distributtion.py +++ b/distribution/tests/test_addon_distributtion.py @@ -51,7 +51,7 @@ def sample_addon_info(): } } ], - "hash": "4f6b8568eb9dd6f510fd7c4dcb676788" + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" } yield addon_info @@ -109,13 +109,13 @@ def test_update_addon_state(printer, sample_addon_info, addon_info.hash = "brokenhash" result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \ - "Hashes not matching" + "Update should failed because of wrong hash" addon_info.hash = orig_hash result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \ - "Failed updating" + "Addon should have been updated" result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \ - "Tried to update" + "Addon should already exist" From b2999a7bbd402154570140fd1db72f6a62158d60 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 7 Sep 2022 14:46:12 +0200 Subject: [PATCH 24/30] OP-3682 - Hound --- distribution/addon_distribution.py | 2 +- distribution/tests/test_addon_distributtion.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 0e3c672915..389b92b10b 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -235,4 +235,4 @@ def check_addons(server_endpoint, addon_folder, downloaders): def cli(*args): - raise NotImplemented \ No newline at end of file + raise NotImplemented diff --git a/distribution/tests/test_addon_distributtion.py b/distribution/tests/test_addon_distributtion.py index 717ef1330e..c6ecaca3c8 100644 --- a/distribution/tests/test_addon_distributtion.py +++ b/distribution/tests/test_addon_distributtion.py @@ -51,7 +51,7 @@ def sample_addon_info(): } } ], - "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa } yield addon_info From 5fa019527b2868c010334a6c36852e32ebaa476e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 12 Sep 2022 10:26:23 +0200 Subject: [PATCH 25/30] OP-3682 - changed folder structure --- {distribution => common/openpype_common/distribution}/README.md | 0 .../openpype_common/distribution}/__init__.py | 0 .../openpype_common/distribution}/addon_distribution.py | 2 +- .../openpype_common/distribution}/file_handler.py | 0 .../distribution}/tests/test_addon_distributtion.py | 2 +- 5 files changed, 2 insertions(+), 2 deletions(-) rename {distribution => common/openpype_common/distribution}/README.md (100%) rename {distribution => common/openpype_common/distribution}/__init__.py (100%) rename {distribution => common/openpype_common/distribution}/addon_distribution.py (98%) rename {distribution => common/openpype_common/distribution}/file_handler.py (100%) rename {distribution => common/openpype_common/distribution}/tests/test_addon_distributtion.py (98%) diff --git a/distribution/README.md b/common/openpype_common/distribution/README.md similarity index 100% rename from distribution/README.md rename to common/openpype_common/distribution/README.md diff --git a/distribution/__init__.py b/common/openpype_common/distribution/__init__.py similarity index 100% rename from distribution/__init__.py rename to common/openpype_common/distribution/__init__.py diff --git a/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py similarity index 98% rename from distribution/addon_distribution.py rename to common/openpype_common/distribution/addon_distribution.py index 389b92b10b..e39ce66a0a 100644 --- a/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -7,7 +7,7 @@ import requests import platform import shutil -from distribution.file_handler import RemoteFileHandler +from common.openpype_common.distribution.file_handler import RemoteFileHandler class UrlType(Enum): diff --git a/distribution/file_handler.py b/common/openpype_common/distribution/file_handler.py similarity index 100% rename from distribution/file_handler.py rename to common/openpype_common/distribution/file_handler.py diff --git a/distribution/tests/test_addon_distributtion.py b/common/openpype_common/distribution/tests/test_addon_distributtion.py similarity index 98% rename from distribution/tests/test_addon_distributtion.py rename to common/openpype_common/distribution/tests/test_addon_distributtion.py index c6ecaca3c8..7dd27fd44f 100644 --- a/distribution/tests/test_addon_distributtion.py +++ b/common/openpype_common/distribution/tests/test_addon_distributtion.py @@ -2,7 +2,7 @@ import pytest import attr import tempfile -from distribution.addon_distribution import ( +from common.openpype_common.distribution.addon_distribution import ( AddonDownloader, UrlType, OSAddonDownloader, From 8a21fdfcf25a3294b53e742dff84eb82950768e7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 15 Sep 2022 12:48:35 +0200 Subject: [PATCH 26/30] OP-3682 - Hound --- common/openpype_common/distribution/addon_distribution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py index e39ce66a0a..ac9c69deca 100644 --- a/common/openpype_common/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -71,7 +71,7 @@ class AddonDownloader: Args: source (dict): {type:"http", "url":"https://} ...} destination (str): local folder to unzip - Retursn: + Returns: (str) local path to addon zip file """ pass @@ -235,4 +235,4 @@ def check_addons(server_endpoint, addon_folder, downloaders): def cli(*args): - raise NotImplemented + raise NotImplementedError From 0fb1b9be93de9fe690afc4b1ca6fca2b1f8ce2fd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 15 Sep 2022 17:59:02 +0200 Subject: [PATCH 27/30] OP-3682 - updated AddonSource --- .../distribution/addon_distribution.py | 33 +++++++++++++++++-- .../tests/test_addon_distributtion.py | 8 ++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py index ac9c69deca..be6faab3e6 100644 --- a/common/openpype_common/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -32,21 +32,50 @@ class MultiPlatformPath(object): @attr.s class AddonSource(object): type = attr.ib() - url = attr.ib(default=None) + + +@attr.s +class LocalAddonSource(AddonSource): path = attr.ib(default=attr.Factory(MultiPlatformPath)) +@attr.s +class WebAddonSource(AddonSource): + url = attr.ib(default=None) + + @attr.s class AddonInfo(object): """Object matching json payload from Server""" name = attr.ib() version = attr.ib() - sources = attr.ib(default=attr.Factory(list), type=AddonSource) + sources = attr.ib(default=attr.Factory(list)) hash = attr.ib(default=None) description = attr.ib(default=None) license = attr.ib(default=None) authors = attr.ib(default=None) + @classmethod + def from_dict(cls, data): + sources = [] + for source in data.get("sources", []): + if source.get("type") == UrlType.FILESYSTEM.value: + source_addon = LocalAddonSource(type=source["type"], + path=source["path"]) + if source.get("type") == UrlType.HTTP.value: + source_addon = WebAddonSource(type=source["type"], + url=source["url"]) + + sources.append(source_addon) + + return cls(name=data.get("name"), + version=data.get("version"), + hash=data.get("hash"), + description=data.get("description"), + sources=sources, + license=data.get("license"), + authors=data.get("authors")) + class AddonDownloader: log = logging.getLogger(__name__) diff --git a/common/openpype_common/distribution/tests/test_addon_distributtion.py b/common/openpype_common/distribution/tests/test_addon_distributtion.py index 7dd27fd44f..faf4e01e22 100644 --- a/common/openpype_common/distribution/tests/test_addon_distributtion.py +++ b/common/openpype_common/distribution/tests/test_addon_distributtion.py @@ -75,7 +75,7 @@ def test_get_downloader(printer, addon_downloader): def test_addon_info(printer, sample_addon_info): valid_minimum = {"name": "openpype_slack", "version": "1.0.0"} - assert AddonInfo(**valid_minimum), "Missing required fields" + assert AddonInfo.from_dict(valid_minimum), "Missing required fields" assert AddonInfo(name=valid_minimum["name"], version=valid_minimum["version"]), \ "Missing required fields" @@ -84,7 +84,7 @@ def test_addon_info(printer, sample_addon_info): # TODO should be probably implemented assert AddonInfo(valid_minimum), "Wrong argument format" - addon = AddonInfo(**sample_addon_info) + addon = AddonInfo.from_dict(sample_addon_info) assert addon, "Should be created" assert addon.name == "openpype_slack", "Incorrect name" assert addon.version == "1.0.0", "Incorrect version" @@ -95,10 +95,10 @@ def test_addon_info(printer, sample_addon_info): addon_as_dict = attr.asdict(addon) assert addon_as_dict["name"], "Dict approach should work" - with pytest.raises(AttributeError): + with pytest.raises(TypeError): # TODO should be probably implemented as . not dict first_source = addon.sources[0] - assert first_source.type == "http", "Not implemented" + assert first_source["type"] == "http", "Not implemented" def test_update_addon_state(printer, sample_addon_info, From 9c6d0b1d7e10a9d5d56832f8b0bb25952f25ad38 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 16 Sep 2022 11:20:52 +0200 Subject: [PATCH 28/30] OP-3682 - changed to relative import --- common/openpype_common/distribution/addon_distribution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py index be6faab3e6..ad17a831d8 100644 --- a/common/openpype_common/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -7,7 +7,7 @@ import requests import platform import shutil -from common.openpype_common.distribution.file_handler import RemoteFileHandler +from .file_handler import RemoteFileHandler class UrlType(Enum): From da7d4cb1d7d792ae4c2398d64bdd3e1d9036d81c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 23 Sep 2022 12:57:17 +0200 Subject: [PATCH 29/30] OP-3682 - updates to match to v4 payload Parsing should match payload from localhost:5000/api/addons?details=1 --- .../distribution/addon_distribution.py | 29 ++++- .../tests/test_addon_distributtion.py | 104 +++++++++++++----- 2 files changed, 98 insertions(+), 35 deletions(-) diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py index ad17a831d8..fec8cb762b 100644 --- a/common/openpype_common/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -44,12 +44,18 @@ class WebAddonSource(AddonSource): url = attr.ib(default=None) +@attr.s +class VersionData(object): + version_data = attr.ib(default=None) + + @attr.s class AddonInfo(object): """Object matching json payload from Server""" name = attr.ib() version = attr.ib() - sources = attr.ib(default=attr.Factory(list)) + title = attr.ib(default=None) + sources = attr.ib(default=attr.Factory(dict)) hash = attr.ib(default=None) description = attr.ib(default=None) license = attr.ib(default=None) @@ -58,7 +64,16 @@ class AddonInfo(object): @classmethod def from_dict(cls, data): sources = [] - for source in data.get("sources", []): + + production_version = data.get("productionVersion") + if not production_version: + return + + # server payload contains info about all versions + # active addon must have 'productionVersion' and matching version info + version_data = data.get("versions", {})[production_version] + + for source in version_data.get("clientSourceInfo", []): if source.get("type") == UrlType.FILESYSTEM.value: source_addon = LocalAddonSource(type=source["type"], path=source["path"]) @@ -69,10 +84,11 @@ class AddonInfo(object): sources.append(source_addon) return cls(name=data.get("name"), - version=data.get("version"), + version=production_version, + sources=sources, hash=data.get("hash"), description=data.get("description"), - sources=sources, + title=data.get("title"), license=data.get("license"), authors=data.get("authors")) @@ -228,8 +244,9 @@ def update_addon_state(addon_infos, destination_folder, factory, for source in addon.sources: download_states[full_name] = UpdateState.FAILED.value try: - downloader = factory.get_downloader(source["type"]) - zip_file_path = downloader.download(source, addon_dest) + downloader = factory.get_downloader(source.type) + zip_file_path = downloader.download(attr.asdict(source), + addon_dest) downloader.check_hash(zip_file_path, addon.hash) downloader.unzip(zip_file_path, addon_dest) download_states[full_name] = UpdateState.UPDATED.value diff --git a/common/openpype_common/distribution/tests/test_addon_distributtion.py b/common/openpype_common/distribution/tests/test_addon_distributtion.py index faf4e01e22..46bcd276cd 100644 --- a/common/openpype_common/distribution/tests/test_addon_distributtion.py +++ b/common/openpype_common/distribution/tests/test_addon_distributtion.py @@ -35,23 +35,50 @@ def temp_folder(): @pytest.fixture def sample_addon_info(): addon_info = { - "name": "openpype_slack", - "version": "1.0.0", - "sources": [ - { - "type": "http", - "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" # noqa - }, - { - "type": "filesystem", - "path": { - "windows": ["P:/sources/some_file.zip", "W:/sources/some_file.zip"], # noqa - "linux": ["/mnt/srv/sources/some_file.zip"], - "darwin": ["/Volumes/srv/sources/some_file.zip"] - } + "versions": { + "1.0.0": { + "clientPyproject": { + "tool": { + "poetry": { + "dependencies": { + "nxtools": "^1.6", + "orjson": "^3.6.7", + "typer": "^0.4.1", + "email-validator": "^1.1.3", + "python": "^3.10", + "fastapi": "^0.73.0" + } + } + } + }, + "hasSettings": True, + "clientSourceInfo": [ + { + "type": "http", + "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing" # noqa + }, + { + "type": "filesystem", + "path": { + "windows": ["P:/sources/some_file.zip", + "W:/sources/some_file.zip"], # noqa + "linux": ["/mnt/srv/sources/some_file.zip"], + "darwin": ["/Volumes/srv/sources/some_file.zip"] + } + } + ], + "frontendScopes": { + "project": { + "sidebar": "hierarchy" + } + } } - ], - "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa + }, + "description": "", + "title": "Slack addon", + "name": "openpype_slack", + "productionVersion": "1.0.0", + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" # noqa } yield addon_info @@ -73,16 +100,39 @@ def test_get_downloader(printer, addon_downloader): def test_addon_info(printer, sample_addon_info): - valid_minimum = {"name": "openpype_slack", "version": "1.0.0"} + """Tests parsing of expected payload from v4 server into AadonInfo.""" + valid_minimum = { + "name": "openpype_slack", + "productionVersion": "1.0.0", + "versions": { + "1.0.0": { + "clientSourceInfo": [ + { + "type": "filesystem", + "path": { + "windows": [ + "P:/sources/some_file.zip", + "W:/sources/some_file.zip"], + "linux": [ + "/mnt/srv/sources/some_file.zip"], + "darwin": [ + "/Volumes/srv/sources/some_file.zip"] # noqa + } + } + ] + } + } + } assert AddonInfo.from_dict(valid_minimum), "Missing required fields" - assert AddonInfo(name=valid_minimum["name"], - version=valid_minimum["version"]), \ - "Missing required fields" - with pytest.raises(TypeError): - # TODO should be probably implemented - assert AddonInfo(valid_minimum), "Wrong argument format" + valid_minimum["versions"].pop("1.0.0") + with pytest.raises(KeyError): + assert not AddonInfo.from_dict(valid_minimum), "Must fail without version data" # noqa + + valid_minimum.pop("productionVersion") + assert not AddonInfo.from_dict( + valid_minimum), "none if not productionVersion" # noqa addon = AddonInfo.from_dict(sample_addon_info) assert addon, "Should be created" @@ -95,15 +145,11 @@ def test_addon_info(printer, sample_addon_info): addon_as_dict = attr.asdict(addon) assert addon_as_dict["name"], "Dict approach should work" - with pytest.raises(TypeError): - # TODO should be probably implemented as . not dict - first_source = addon.sources[0] - assert first_source["type"] == "http", "Not implemented" - def test_update_addon_state(printer, sample_addon_info, temp_folder, addon_downloader): - addon_info = AddonInfo(**sample_addon_info) + """Tests possible cases of addon update.""" + addon_info = AddonInfo.from_dict(sample_addon_info) orig_hash = addon_info.hash addon_info.hash = "brokenhash" From c659dcfce6393972aa0443f03f67950b1e9fdc45 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 23 Sep 2022 13:05:50 +0200 Subject: [PATCH 30/30] OP-3682 - extracted AddonInfo to separate file Parsing of v4 payload info will be required in Dependencies tool also. --- .../distribution/addon_distribution.py | 78 +----------------- .../distribution/addon_info.py | 80 +++++++++++++++++++ .../tests/test_addon_distributtion.py | 2 +- 3 files changed, 82 insertions(+), 78 deletions(-) create mode 100644 common/openpype_common/distribution/addon_info.py diff --git a/common/openpype_common/distribution/addon_distribution.py b/common/openpype_common/distribution/addon_distribution.py index fec8cb762b..5e48639dec 100644 --- a/common/openpype_common/distribution/addon_distribution.py +++ b/common/openpype_common/distribution/addon_distribution.py @@ -8,12 +8,7 @@ import platform import shutil from .file_handler import RemoteFileHandler - - -class UrlType(Enum): - HTTP = "http" - GIT = "git" - FILESYSTEM = "filesystem" +from .addon_info import AddonInfo class UpdateState(Enum): @@ -22,77 +17,6 @@ class UpdateState(Enum): FAILED = "failed" -@attr.s -class MultiPlatformPath(object): - windows = attr.ib(default=None) - linux = attr.ib(default=None) - darwin = attr.ib(default=None) - - -@attr.s -class AddonSource(object): - type = attr.ib() - - -@attr.s -class LocalAddonSource(AddonSource): - path = attr.ib(default=attr.Factory(MultiPlatformPath)) - - -@attr.s -class WebAddonSource(AddonSource): - url = attr.ib(default=None) - - -@attr.s -class VersionData(object): - version_data = attr.ib(default=None) - - -@attr.s -class AddonInfo(object): - """Object matching json payload from Server""" - name = attr.ib() - version = attr.ib() - title = attr.ib(default=None) - sources = attr.ib(default=attr.Factory(dict)) - hash = attr.ib(default=None) - description = attr.ib(default=None) - license = attr.ib(default=None) - authors = attr.ib(default=None) - - @classmethod - def from_dict(cls, data): - sources = [] - - production_version = data.get("productionVersion") - if not production_version: - return - - # server payload contains info about all versions - # active addon must have 'productionVersion' and matching version info - version_data = data.get("versions", {})[production_version] - - for source in version_data.get("clientSourceInfo", []): - if source.get("type") == UrlType.FILESYSTEM.value: - source_addon = LocalAddonSource(type=source["type"], - path=source["path"]) - if source.get("type") == UrlType.HTTP.value: - source_addon = WebAddonSource(type=source["type"], - url=source["url"]) - - sources.append(source_addon) - - return cls(name=data.get("name"), - version=production_version, - sources=sources, - hash=data.get("hash"), - description=data.get("description"), - title=data.get("title"), - license=data.get("license"), - authors=data.get("authors")) - - class AddonDownloader: log = logging.getLogger(__name__) diff --git a/common/openpype_common/distribution/addon_info.py b/common/openpype_common/distribution/addon_info.py new file mode 100644 index 0000000000..00ece11f3b --- /dev/null +++ b/common/openpype_common/distribution/addon_info.py @@ -0,0 +1,80 @@ +import attr +from enum import Enum + + +class UrlType(Enum): + HTTP = "http" + GIT = "git" + FILESYSTEM = "filesystem" + + +@attr.s +class MultiPlatformPath(object): + windows = attr.ib(default=None) + linux = attr.ib(default=None) + darwin = attr.ib(default=None) + + +@attr.s +class AddonSource(object): + type = attr.ib() + + +@attr.s +class LocalAddonSource(AddonSource): + path = attr.ib(default=attr.Factory(MultiPlatformPath)) + + +@attr.s +class WebAddonSource(AddonSource): + url = attr.ib(default=None) + + +@attr.s +class VersionData(object): + version_data = attr.ib(default=None) + + +@attr.s +class AddonInfo(object): + """Object matching json payload from Server""" + name = attr.ib() + version = attr.ib() + title = attr.ib(default=None) + sources = attr.ib(default=attr.Factory(dict)) + hash = attr.ib(default=None) + description = attr.ib(default=None) + license = attr.ib(default=None) + authors = attr.ib(default=None) + + @classmethod + def from_dict(cls, data): + sources = [] + + production_version = data.get("productionVersion") + if not production_version: + return + + # server payload contains info about all versions + # active addon must have 'productionVersion' and matching version info + version_data = data.get("versions", {})[production_version] + + for source in version_data.get("clientSourceInfo", []): + if source.get("type") == UrlType.FILESYSTEM.value: + source_addon = LocalAddonSource(type=source["type"], + path=source["path"]) + if source.get("type") == UrlType.HTTP.value: + source_addon = WebAddonSource(type=source["type"], + url=source["url"]) + + sources.append(source_addon) + + return cls(name=data.get("name"), + version=production_version, + sources=sources, + hash=data.get("hash"), + description=data.get("description"), + title=data.get("title"), + license=data.get("license"), + authors=data.get("authors")) + diff --git a/common/openpype_common/distribution/tests/test_addon_distributtion.py b/common/openpype_common/distribution/tests/test_addon_distributtion.py index 46bcd276cd..765ea0596a 100644 --- a/common/openpype_common/distribution/tests/test_addon_distributtion.py +++ b/common/openpype_common/distribution/tests/test_addon_distributtion.py @@ -4,13 +4,13 @@ import tempfile from common.openpype_common.distribution.addon_distribution import ( AddonDownloader, - UrlType, OSAddonDownloader, HTTPAddonDownloader, AddonInfo, update_addon_state, UpdateState ) +from common.openpype_common.distribution.addon_info import UrlType @pytest.fixture