Merge pull request #3646 from pypeclub/feature/OP-3682_Addons-distribution

Backend: Implementation of addon deployment logic

common/openpype_common/distribution/README.md

@@ -0,0 +1,18 @@
+Addon distribution tool
+------------------------
+
+Code in this folder is backend portion of Addon distribution logic for v4 server.
+
+Each host, module will be separate Addon in the future. Each v4 server could run different set of Addons.
+
+Client (running on artist machine) will in the first step ask v4 for list of enabled addons. 
+(It expects list of json documents matching to `addon_distribution.py:AddonInfo` object.)
+Next it will compare presence of enabled addon version in local folder. In the case of missing version of
+an addon, client will use information in the addon to download (from http/shared local disk/git) zip file
+and unzip it.
+
+Required part of addon distribution will be sharing of dependencies (python libraries, utilities) which is not part of this folder.
+
+Location of this folder might change in the future as it will be required for a clint to add this folder to sys.path reliably.
+
+This code needs to be independent on Openpype code as much as possible!

common/openpype_common/distribution/addon_distribution.py

@@ -0,0 +1,208 @@
+import os
+from enum import Enum
+from abc import abstractmethod
+import attr
+import logging
+import requests
+import platform
+import shutil
+
+from .file_handler import RemoteFileHandler
+from .addon_info import AddonInfo
+
+
+class UpdateState(Enum):
+    EXISTS = "exists"
+    UPDATED = "updated"
+    FAILED = "failed"
+
+
+class AddonDownloader:
+    log = logging.getLogger(__name__)
+
+    def __init__(self):
+        self._downloaders = {}
+
+    def register_format(self, downloader_type, downloader):
+        self._downloaders[downloader_type.value] = downloader
+
+    def get_downloader(self, downloader_type):
+        downloader = self._downloaders.get(downloader_type)
+        if not downloader:
+            raise ValueError(f"{downloader_type} not implemented")
+        return downloader()
+
+    @classmethod
+    @abstractmethod
+    def download(cls, source, destination):
+        """Returns url to downloaded addon zip file.
+
+        Args:
+            source (dict): {type:"http", "url":"https://} ...}
+            destination (str): local folder to unzip
+        Returns:
+            (str) local path to addon zip file
+        """
+        pass
+
+    @classmethod
+    def check_hash(cls, addon_path, addon_hash):
+        """Compares 'hash' of downloaded 'addon_url' file.
+
+        Args:
+            addon_path (str): local path to addon zip file
+            addon_hash (str): sha256 hash of zip file
+        Raises:
+            ValueError if hashes doesn't match
+        """
+        if not os.path.exists(addon_path):
+            raise ValueError(f"{addon_path} doesn't exist.")
+        if not RemoteFileHandler.check_integrity(addon_path,
+                                                 addon_hash,
+                                                 hash_type="sha256"):
+            raise ValueError(f"{addon_path} doesn't match expected hash.")
+
+    @classmethod
+    def unzip(cls, addon_zip_path, destination):
+        """Unzips local 'addon_zip_path' to 'destination'.
+
+        Args:
+            addon_zip_path (str): local path to addon zip file
+            destination (str): local folder to unzip
+        """
+        RemoteFileHandler.unzip(addon_zip_path, destination)
+        os.remove(addon_zip_path)
+
+    @classmethod
+    def remove(cls, addon_url):
+        pass
+
+
+class OSAddonDownloader(AddonDownloader):
+
+    @classmethod
+    def download(cls, source, destination):
+        # OS doesnt need to download, unzip directly
+        addon_url = source["path"].get(platform.system().lower())
+        if not os.path.exists(addon_url):
+            raise ValueError("{} is not accessible".format(addon_url))
+        return addon_url
+
+
+class HTTPAddonDownloader(AddonDownloader):
+    CHUNK_SIZE = 100000
+
+    @classmethod
+    def download(cls, source, destination):
+        source_url = source["url"]
+        cls.log.debug(f"Downloading {source_url} to {destination}")
+        file_name = os.path.basename(destination)
+        _, ext = os.path.splitext(file_name)
+        if (ext.replace(".", '') not
+                in set(RemoteFileHandler.IMPLEMENTED_ZIP_FORMATS)):
+            file_name += ".zip"
+        RemoteFileHandler.download_url(source_url,
+                                       destination,
+                                       filename=file_name)
+
+        return os.path.join(destination, file_name)
+
+
+def get_addons_info(server_endpoint):
+    """Returns list of addon information from Server"""
+    # TODO temp
+    # addon_info = AddonInfo(
+    #     **{"name": "openpype_slack",
+    #        "version": "1.0.0",
+    #        "addon_url": "c:/projects/openpype_slack_1.0.0.zip",
+    #        "type": UrlType.FILESYSTEM,
+    #        "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"})  # noqa
+    #
+    # http_addon = AddonInfo(
+    #     **{"name": "openpype_slack",
+    #        "version": "1.0.0",
+    #        "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing",  # noqa
+    #        "type": UrlType.HTTP,
+    #        "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"})  # noqa
+
+    response = requests.get(server_endpoint)
+    if not response.ok:
+        raise Exception(response.text)
+
+    addons_info = []
+    for addon in response.json():
+        addons_info.append(AddonInfo(**addon))
+    return addons_info
+
+
+def update_addon_state(addon_infos, destination_folder, factory,
+                       log=None):
+    """Loops through all 'addon_infos', compares local version, unzips.
+
+    Loops through server provided list of dictionaries with information about
+    available addons. Looks if each addon is already present and deployed.
+    If isn't, addon zip gets downloaded and unzipped into 'destination_folder'.
+    Args:
+        addon_infos (list of AddonInfo)
+        destination_folder (str): local path
+        factory (AddonDownloader): factory to get appropriate downloader per
+            addon type
+        log (logging.Logger)
+    Returns:
+        (dict): {"addon_full_name": UpdateState.value
+            (eg. "exists"|"updated"|"failed")
+    """
+    if not log:
+        log = logging.getLogger(__name__)
+
+    download_states = {}
+    for addon in addon_infos:
+        full_name = "{}_{}".format(addon.name, addon.version)
+        addon_dest = os.path.join(destination_folder, full_name)
+
+        if os.path.isdir(addon_dest):
+            log.debug(f"Addon version folder {addon_dest} already exists.")
+            download_states[full_name] = UpdateState.EXISTS.value
+            continue
+
+        for source in addon.sources:
+            download_states[full_name] = UpdateState.FAILED.value
+            try:
+                downloader = factory.get_downloader(source.type)
+                zip_file_path = downloader.download(attr.asdict(source),
+                                                    addon_dest)
+                downloader.check_hash(zip_file_path, addon.hash)
+                downloader.unzip(zip_file_path, addon_dest)
+                download_states[full_name] = UpdateState.UPDATED.value
+                break
+            except Exception:
+                log.warning(f"Error happened during updating {addon.name}",
+                            exc_info=True)
+                if os.path.isdir(addon_dest):
+                    log.debug(f"Cleaning {addon_dest}")
+                    shutil.rmtree(addon_dest)
+
+    return download_states
+
+
+def check_addons(server_endpoint, addon_folder, downloaders):
+    """Main entry point to compare existing addons with those on server.
+
+    Args:
+        server_endpoint (str): url to v4 server endpoint
+        addon_folder (str): local dir path for addons
+        downloaders (AddonDownloader): factory of downloaders
+
+    Raises:
+        (RuntimeError) if any addon failed update
+    """
+    addons_info = get_addons_info(server_endpoint)
+    result = update_addon_state(addons_info,
+                                addon_folder,
+                                downloaders)
+    if UpdateState.FAILED.value in result.values():
+        raise RuntimeError(f"Unable to update some addons {result}")
+
+
+def cli(*args):
+    raise NotImplementedError

common/openpype_common/distribution/addon_info.py

@@ -0,0 +1,80 @@
+import attr
+from enum import Enum
+
+
+class UrlType(Enum):
+    HTTP = "http"
+    GIT = "git"
+    FILESYSTEM = "filesystem"
+
+
+@attr.s
+class MultiPlatformPath(object):
+    windows = attr.ib(default=None)
+    linux = attr.ib(default=None)
+    darwin = attr.ib(default=None)
+
+
+@attr.s
+class AddonSource(object):
+    type = attr.ib()
+
+
+@attr.s
+class LocalAddonSource(AddonSource):
+    path = attr.ib(default=attr.Factory(MultiPlatformPath))
+
+
+@attr.s
+class WebAddonSource(AddonSource):
+    url = attr.ib(default=None)
+
+
+@attr.s
+class VersionData(object):
+    version_data = attr.ib(default=None)
+
+
+@attr.s
+class AddonInfo(object):
+    """Object matching json payload from Server"""
+    name = attr.ib()
+    version = attr.ib()
+    title = attr.ib(default=None)
+    sources = attr.ib(default=attr.Factory(dict))
+    hash = attr.ib(default=None)
+    description = attr.ib(default=None)
+    license = attr.ib(default=None)
+    authors = attr.ib(default=None)
+
+    @classmethod
+    def from_dict(cls, data):
+        sources = []
+
+        production_version = data.get("productionVersion")
+        if not production_version:
+            return
+
+        # server payload contains info about all versions
+        # active addon must have 'productionVersion' and matching version info
+        version_data = data.get("versions", {})[production_version]
+
+        for source in version_data.get("clientSourceInfo", []):
+            if source.get("type") == UrlType.FILESYSTEM.value:
+                source_addon = LocalAddonSource(type=source["type"],
+                                                path=source["path"])
+            if source.get("type") == UrlType.HTTP.value:
+                source_addon = WebAddonSource(type=source["type"],
+                                              url=source["url"])
+
+            sources.append(source_addon)
+
+        return cls(name=data.get("name"),
+                   version=production_version,
+                   sources=sources,
+                   hash=data.get("hash"),
+                   description=data.get("description"),
+                   title=data.get("title"),
+                   license=data.get("license"),
+                   authors=data.get("authors"))
+

common/openpype_common/distribution/file_handler.py

@@ -21,7 +21,7 @@ class RemoteFileHandler:
                                'tar.gz', 'tar.xz', 'tar.bz2']
 
     @staticmethod
-    def calculate_md5(fpath, chunk_size):
+    def calculate_md5(fpath, chunk_size=10000):
         md5 = hashlib.md5()
         with open(fpath, 'rb') as f:
             for chunk in iter(lambda: f.read(chunk_size), b''):
@@ -33,17 +33,45 @@ class RemoteFileHandler:
         return md5 == RemoteFileHandler.calculate_md5(fpath, **kwargs)
 
     @staticmethod
-    def check_integrity(fpath, md5=None):
+    def calculate_sha256(fpath):
+        """Calculate sha256 for content of the file.
+
+        Args:
+             fpath (str): Path to file.
+
+        Returns:
+            str: hex encoded sha256
+
+        """
+        h = hashlib.sha256()
+        b = bytearray(128 * 1024)
+        mv = memoryview(b)
+        with open(fpath, 'rb', buffering=0) as f:
+            for n in iter(lambda: f.readinto(mv), 0):
+                h.update(mv[:n])
+        return h.hexdigest()
+
+    @staticmethod
+    def check_sha256(fpath, sha256, **kwargs):
+        return sha256 == RemoteFileHandler.calculate_sha256(fpath, **kwargs)
+
+    @staticmethod
+    def check_integrity(fpath, hash_value=None, hash_type=None):
         if not os.path.isfile(fpath):
             return False
-        if md5 is None:
+        if hash_value is None:
             return True
-        return RemoteFileHandler.check_md5(fpath, md5)
+        if not hash_type:
+            raise ValueError("Provide hash type, md5 or sha256")
+        if hash_type == 'md5':
+            return RemoteFileHandler.check_md5(fpath, hash_value)
+        if hash_type == "sha256":
+            return RemoteFileHandler.check_sha256(fpath, hash_value)
 
     @staticmethod
     def download_url(
         url, root, filename=None,
-        md5=None, max_redirect_hops=3
+        sha256=None, max_redirect_hops=3
     ):
         """Download a file from a url and place it in root.
         Args:
@@ -51,7 +79,7 @@ class RemoteFileHandler:
             root (str): Directory to place downloaded file in
             filename (str, optional): Name to save the file under.
                 If None, use the basename of the URL
-            md5 (str, optional): MD5 checksum of the download.
+            sha256 (str, optional): sha256 checksum of the download.
                 If None, do not check
             max_redirect_hops (int, optional): Maximum number of redirect
                 hops allowed
@@ -64,7 +92,8 @@ class RemoteFileHandler:
         os.makedirs(root, exist_ok=True)
 
         # check if file is already present locally
-        if RemoteFileHandler.check_integrity(fpath, md5):
+        if RemoteFileHandler.check_integrity(fpath,
+                                             sha256, hash_type="sha256"):
             print('Using downloaded and verified file: ' + fpath)
             return
 
@@ -76,7 +105,7 @@ class RemoteFileHandler:
         file_id = RemoteFileHandler._get_google_drive_file_id(url)
         if file_id is not None:
             return RemoteFileHandler.download_file_from_google_drive(
-                file_id, root, filename, md5)
+                file_id, root, filename, sha256)
 
         # download the file
         try:
@@ -92,20 +121,21 @@ class RemoteFileHandler:
                 raise e
 
         # check integrity of downloaded file
-        if not RemoteFileHandler.check_integrity(fpath, md5):
+        if not RemoteFileHandler.check_integrity(fpath,
+                                                 sha256, hash_type="sha256"):
             raise RuntimeError("File not found or corrupted.")
 
     @staticmethod
     def download_file_from_google_drive(file_id, root,
                                         filename=None,
-                                        md5=None):
+                                        sha256=None):
         """Download a Google Drive file from  and place it in root.
         Args:
             file_id (str): id of file to be downloaded
             root (str): Directory to place downloaded file in
             filename (str, optional): Name to save the file under.
                 If None, use the id of the file.
-            md5 (str, optional): MD5 checksum of the download.
+            sha256 (str, optional): sha256 checksum of the download.
                 If None, do not check
         """
         # Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa
@@ -119,8 +149,8 @@ class RemoteFileHandler:
 
         os.makedirs(root, exist_ok=True)
 
-        if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath,
-                                                                       md5):
+        if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(
+                fpath, sha256, hash_type="sha256"):
             print('Using downloaded and verified file: ' + fpath)
         else:
             session = requests.Session()

common/openpype_common/distribution/tests/test_addon_distributtion.py

@@ -0,0 +1,167 @@
+import pytest
+import attr
+import tempfile
+
+from common.openpype_common.distribution.addon_distribution import (
+    AddonDownloader,
+    OSAddonDownloader,
+    HTTPAddonDownloader,
+    AddonInfo,
+    update_addon_state,
+    UpdateState
+)
+from common.openpype_common.distribution.addon_info import UrlType
+
+
+@pytest.fixture
+def addon_downloader():
+    addon_downloader = AddonDownloader()
+    addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader)
+    addon_downloader.register_format(UrlType.HTTP, HTTPAddonDownloader)
+
+    yield addon_downloader
+
+
+@pytest.fixture
+def http_downloader(addon_downloader):
+    yield addon_downloader.get_downloader(UrlType.HTTP.value)
+
+
+@pytest.fixture
+def temp_folder():
+    yield tempfile.mkdtemp()
+
+
+@pytest.fixture
+def sample_addon_info():
+    addon_info = {
+       "versions": {
+            "1.0.0": {
+                 "clientPyproject": {
+                      "tool": {
+                           "poetry": {
+                            "dependencies": {
+                             "nxtools": "^1.6",
+                             "orjson": "^3.6.7",
+                             "typer": "^0.4.1",
+                             "email-validator": "^1.1.3",
+                             "python": "^3.10",
+                             "fastapi": "^0.73.0"
+                            }
+                       }
+                  }
+                 },
+                 "hasSettings": True,
+                 "clientSourceInfo": [
+                     {
+                         "type": "http",
+                         "url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing"  # noqa
+                     },
+                     {
+                         "type": "filesystem",
+                         "path": {
+                             "windows": ["P:/sources/some_file.zip",
+                                         "W:/sources/some_file.zip"],  # noqa
+                             "linux": ["/mnt/srv/sources/some_file.zip"],
+                             "darwin": ["/Volumes/srv/sources/some_file.zip"]
+                         }
+                     }
+                 ],
+                 "frontendScopes": {
+                      "project": {
+                       "sidebar": "hierarchy"
+                      }
+                 }
+            }
+       },
+       "description": "",
+       "title": "Slack addon",
+       "name": "openpype_slack",
+       "productionVersion": "1.0.0",
+       "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"  # noqa
+    }
+    yield addon_info
+
+
+def test_register(printer):
+    addon_downloader = AddonDownloader()
+
+    assert len(addon_downloader._downloaders) == 0, "Contains registered"
+
+    addon_downloader.register_format(UrlType.FILESYSTEM, OSAddonDownloader)
+    assert len(addon_downloader._downloaders) == 1, "Should contain one"
+
+
+def test_get_downloader(printer, addon_downloader):
+    assert addon_downloader.get_downloader(UrlType.FILESYSTEM.value), "Should find"  # noqa
+
+    with pytest.raises(ValueError):
+        addon_downloader.get_downloader("unknown"), "Shouldn't find"
+
+
+def test_addon_info(printer, sample_addon_info):
+    """Tests parsing of expected payload from v4 server into AadonInfo."""
+    valid_minimum = {
+        "name": "openpype_slack",
+         "productionVersion": "1.0.0",
+         "versions": {
+             "1.0.0": {
+                 "clientSourceInfo": [
+                     {
+                         "type": "filesystem",
+                         "path": {
+                             "windows": [
+                                 "P:/sources/some_file.zip",
+                                 "W:/sources/some_file.zip"],
+                             "linux": [
+                                 "/mnt/srv/sources/some_file.zip"],
+                             "darwin": [
+                                 "/Volumes/srv/sources/some_file.zip"]  # noqa
+                         }
+                     }
+                 ]
+             }
+         }
+    }
+
+    assert AddonInfo.from_dict(valid_minimum), "Missing required fields"
+
+    valid_minimum["versions"].pop("1.0.0")
+    with pytest.raises(KeyError):
+        assert not AddonInfo.from_dict(valid_minimum), "Must fail without version data"  # noqa
+
+    valid_minimum.pop("productionVersion")
+    assert not AddonInfo.from_dict(
+        valid_minimum), "none if not productionVersion"  # noqa
+
+    addon = AddonInfo.from_dict(sample_addon_info)
+    assert addon, "Should be created"
+    assert addon.name == "openpype_slack", "Incorrect name"
+    assert addon.version == "1.0.0", "Incorrect version"
+
+    with pytest.raises(TypeError):
+        assert addon["name"], "Dict approach not implemented"
+
+    addon_as_dict = attr.asdict(addon)
+    assert addon_as_dict["name"], "Dict approach should work"
+
+
+def test_update_addon_state(printer, sample_addon_info,
+                            temp_folder, addon_downloader):
+    """Tests possible cases of addon update."""
+    addon_info = AddonInfo.from_dict(sample_addon_info)
+    orig_hash = addon_info.hash
+
+    addon_info.hash = "brokenhash"
+    result = update_addon_state([addon_info], temp_folder, addon_downloader)
+    assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \
+        "Update should failed because of wrong hash"
+
+    addon_info.hash = orig_hash
+    result = update_addon_state([addon_info], temp_folder, addon_downloader)
+    assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \
+        "Addon should have been updated"
+
+    result = update_addon_state([addon_info], temp_folder, addon_downloader)
+    assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \
+        "Addon should already exist"

tests/lib/testing_classes.py

@@ -10,7 +10,7 @@ import glob
 import platform
 
 from tests.lib.db_handler import DBHandler
-from tests.lib.file_handler import RemoteFileHandler
+from distribution.file_handler import RemoteFileHandler
 
 
 class BaseTest: