From ae88578dbda9eca89cc792cec498d19c7ef3d6af Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 7 Sep 2022 14:38:45 +0200 Subject: [PATCH] OP-3682 - changed md5 to sha256 Updated tests. Removed test cli method --- distribution/addon_distribution.py | 28 ++++------ distribution/file_handler.py | 54 ++++++++++++++----- .../tests/test_addon_distributtion.py | 8 +-- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/distribution/addon_distribution.py b/distribution/addon_distribution.py index 95d0b5e397..0e3c672915 100644 --- a/distribution/addon_distribution.py +++ b/distribution/addon_distribution.py @@ -5,6 +5,7 @@ import attr import logging import requests import platform +import shutil from distribution.file_handler import RemoteFileHandler @@ -87,7 +88,9 @@ class AddonDownloader: """ if not os.path.exists(addon_path): raise ValueError(f"{addon_path} doesn't exist.") - if addon_hash != RemoteFileHandler.calculate_md5(addon_path): + if not RemoteFileHandler.check_integrity(addon_path, + addon_hash, + hash_type="sha256"): raise ValueError(f"{addon_path} doesn't match expected hash.") @classmethod @@ -144,14 +147,14 @@ def get_addons_info(server_endpoint): # "version": "1.0.0", # "addon_url": "c:/projects/openpype_slack_1.0.0.zip", # "type": UrlType.FILESYSTEM, - # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa # # http_addon = AddonInfo( # **{"name": "openpype_slack", # "version": "1.0.0", # "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa # "type": UrlType.HTTP, - # "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa + # "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa response = requests.get(server_endpoint) if not response.ok: @@ -205,6 +208,9 @@ def update_addon_state(addon_infos, destination_folder, factory, except Exception: log.warning(f"Error happened during updating {addon.name}", exc_info=True) + if os.path.isdir(addon_dest): + log.debug(f"Cleaning {addon_dest}") + shutil.rmtree(addon_dest) return download_states @@ -228,17 +234,5 @@ def check_addons(server_endpoint, addon_folder, downloaders): raise RuntimeError(f"Unable to update some addons {result}") -def cli(args): - addon_folder = "c:/projects/testing_addons/pypeclub/openpype/addons" - - downloader_factory = AddonDownloader() - downloader_factory.register_format(UrlType.FILESYSTEM, OSAddonDownloader) - downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader) - - test_endpoint = "https://34e99f0f-f987-4715-95e6-d2d88caa7586.mock.pstmn.io/get_addons_info" # noqa - if os.environ.get("OPENPYPE_SERVER"): # TODO or from keychain - server_endpoint = os.environ.get("OPENPYPE_SERVER") + "get_addons_info" - else: - server_endpoint = test_endpoint - - check_addons(server_endpoint, addon_folder, downloader_factory) +def cli(*args): + raise NotImplemented \ No newline at end of file diff --git a/distribution/file_handler.py b/distribution/file_handler.py index 8c8b4230ce..f585c77632 100644 --- a/distribution/file_handler.py +++ b/distribution/file_handler.py @@ -33,17 +33,45 @@ class RemoteFileHandler: return md5 == RemoteFileHandler.calculate_md5(fpath, **kwargs) @staticmethod - def check_integrity(fpath, md5=None): + def calculate_sha256(fpath): + """Calculate sha256 for content of the file. + + Args: + fpath (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(fpath, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + + @staticmethod + def check_sha256(fpath, sha256, **kwargs): + return sha256 == RemoteFileHandler.calculate_sha256(fpath, **kwargs) + + @staticmethod + def check_integrity(fpath, hash_value=None, hash_type=None): if not os.path.isfile(fpath): return False - if md5 is None: + if hash_value is None: return True - return RemoteFileHandler.check_md5(fpath, md5) + if not hash_type: + raise ValueError("Provide hash type, md5 or sha256") + if hash_type == 'md5': + return RemoteFileHandler.check_md5(fpath, hash_value) + if hash_type == "sha256": + return RemoteFileHandler.check_sha256(fpath, hash_value) @staticmethod def download_url( url, root, filename=None, - md5=None, max_redirect_hops=3 + sha256=None, max_redirect_hops=3 ): """Download a file from a url and place it in root. Args: @@ -51,7 +79,7 @@ class RemoteFileHandler: root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the basename of the URL - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check max_redirect_hops (int, optional): Maximum number of redirect hops allowed @@ -64,7 +92,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) # check if file is already present locally - if RemoteFileHandler.check_integrity(fpath, md5): + if RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) return @@ -76,7 +105,7 @@ class RemoteFileHandler: file_id = RemoteFileHandler._get_google_drive_file_id(url) if file_id is not None: return RemoteFileHandler.download_file_from_google_drive( - file_id, root, filename, md5) + file_id, root, filename, sha256) # download the file try: @@ -92,20 +121,21 @@ class RemoteFileHandler: raise e # check integrity of downloaded file - if not RemoteFileHandler.check_integrity(fpath, md5): + if not RemoteFileHandler.check_integrity(fpath, + sha256, hash_type="sha256"): raise RuntimeError("File not found or corrupted.") @staticmethod def download_file_from_google_drive(file_id, root, filename=None, - md5=None): + sha256=None): """Download a Google Drive file from and place it in root. Args: file_id (str): id of file to be downloaded root (str): Directory to place downloaded file in filename (str, optional): Name to save the file under. If None, use the id of the file. - md5 (str, optional): MD5 checksum of the download. + sha256 (str, optional): sha256 checksum of the download. If None, do not check """ # Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa @@ -119,8 +149,8 @@ class RemoteFileHandler: os.makedirs(root, exist_ok=True) - if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath, - md5): + if os.path.isfile(fpath) and RemoteFileHandler.check_integrity( + fpath, sha256, hash_type="sha256"): print('Using downloaded and verified file: ' + fpath) else: session = requests.Session() diff --git a/distribution/tests/test_addon_distributtion.py b/distribution/tests/test_addon_distributtion.py index e67ca3c479..717ef1330e 100644 --- a/distribution/tests/test_addon_distributtion.py +++ b/distribution/tests/test_addon_distributtion.py @@ -51,7 +51,7 @@ def sample_addon_info(): } } ], - "hash": "4f6b8568eb9dd6f510fd7c4dcb676788" + "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658" } yield addon_info @@ -109,13 +109,13 @@ def test_update_addon_state(printer, sample_addon_info, addon_info.hash = "brokenhash" result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \ - "Hashes not matching" + "Update should failed because of wrong hash" addon_info.hash = orig_hash result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \ - "Failed updating" + "Addon should have been updated" result = update_addon_state([addon_info], temp_folder, addon_downloader) assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \ - "Tried to update" + "Addon should already exist"