Frame13/ayon-core
1
0
Fork 1
mirror of https://github.com/ynput/ayon-core.git synced 2025-12-24 21:04:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

OP-3682 - changed md5 to sha256

Browse source 
Updated tests.
Removed test cli method
This commit is contained in:
Petr Kalis 2022-09-07 14:38:45 +02:00
parent 390dbb6320
commit ae88578dbd
3 changed files with 57 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

28
distribution/addon_distribution.py
View file

@ -5,6 +5,7 @@ import attr
import logging
import requests
import platform
import shutil
from distribution.file_handler import RemoteFileHandler
@ -87,7 +88,9 @@ class AddonDownloader:
"""
if not os.path.exists(addon_path):
raise ValueError(f"{addon_path} doesn't exist.")
if addon_hash != RemoteFileHandler.calculate_md5(addon_path):
if not RemoteFileHandler.check_integrity(addon_path,
addon_hash,
hash_type="sha256"):
raise ValueError(f"{addon_path} doesn't match expected hash.")
@classmethod
@ -144,14 +147,14 @@ def get_addons_info(server_endpoint):
# "version": "1.0.0",
# "addon_url": "c:/projects/openpype_slack_1.0.0.zip",
# "type": UrlType.FILESYSTEM,
# "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa
# "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa
#
# http_addon = AddonInfo(
# **{"name": "openpype_slack",
# "version": "1.0.0",
# "addon_url": "https://drive.google.com/file/d/1TcuV8c2OV8CcbPeWi7lxOdqWsEqQNPYy/view?usp=sharing", # noqa
# "type": UrlType.HTTP,
# "hash": "4f6b8568eb9dd6f510fd7c4dcb676788"}) # noqa
# "hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"}) # noqa
response = requests.get(server_endpoint)
if not response.ok:
@ -205,6 +208,9 @@ def update_addon_state(addon_infos, destination_folder, factory,
except Exception:
log.warning(f"Error happened during updating {addon.name}",
exc_info=True)
if os.path.isdir(addon_dest):
log.debug(f"Cleaning {addon_dest}")
shutil.rmtree(addon_dest)
return download_states
@ -228,17 +234,5 @@ def check_addons(server_endpoint, addon_folder, downloaders):
raise RuntimeError(f"Unable to update some addons {result}")
def cli(args):
addon_folder = "c:/projects/testing_addons/pypeclub/openpype/addons"
downloader_factory = AddonDownloader()
downloader_factory.register_format(UrlType.FILESYSTEM, OSAddonDownloader)
downloader_factory.register_format(UrlType.HTTP, HTTPAddonDownloader)
test_endpoint = "https://34e99f0f-f987-4715-95e6-d2d88caa7586.mock.pstmn.io/get_addons_info" # noqa
if os.environ.get("OPENPYPE_SERVER"): # TODO or from keychain
server_endpoint = os.environ.get("OPENPYPE_SERVER") + "get_addons_info"
else:
server_endpoint = test_endpoint
check_addons(server_endpoint, addon_folder, downloader_factory)
def cli(*args):
raise NotImplemented

54
distribution/file_handler.py
View file

@ -33,17 +33,45 @@ class RemoteFileHandler:
return md5 == RemoteFileHandler.calculate_md5(fpath, **kwargs)
@staticmethod
def check_integrity(fpath, md5=None):
def calculate_sha256(fpath):
"""Calculate sha256 for content of the file.
Args:
fpath (str): Path to file.
Returns:
str: hex encoded sha256
"""
h = hashlib.sha256()
b = bytearray(128 * 1024)
mv = memoryview(b)
with open(fpath, 'rb', buffering=0) as f:
for n in iter(lambda: f.readinto(mv), 0):
h.update(mv[:n])
return h.hexdigest()
@staticmethod
def check_sha256(fpath, sha256, **kwargs):
return sha256 == RemoteFileHandler.calculate_sha256(fpath, **kwargs)
@staticmethod
def check_integrity(fpath, hash_value=None, hash_type=None):
if not os.path.isfile(fpath):
return False
if md5 is None:
if hash_value is None:
return True
return RemoteFileHandler.check_md5(fpath, md5)
if not hash_type:
raise ValueError("Provide hash type, md5 or sha256")
if hash_type == 'md5':
return RemoteFileHandler.check_md5(fpath, hash_value)
if hash_type == "sha256":
return RemoteFileHandler.check_sha256(fpath, hash_value)
@staticmethod
def download_url(
url, root, filename=None,
md5=None, max_redirect_hops=3
sha256=None, max_redirect_hops=3
):
"""Download a file from a url and place it in root.
Args:
@ -51,7 +79,7 @@ class RemoteFileHandler:
root (str): Directory to place downloaded file in
filename (str, optional): Name to save the file under.
If None, use the basename of the URL
md5 (str, optional): MD5 checksum of the download.
sha256 (str, optional): sha256 checksum of the download.
If None, do not check
max_redirect_hops (int, optional): Maximum number of redirect
hops allowed
@ -64,7 +92,8 @@ class RemoteFileHandler:
os.makedirs(root, exist_ok=True)
# check if file is already present locally
if RemoteFileHandler.check_integrity(fpath, md5):
if RemoteFileHandler.check_integrity(fpath,
sha256, hash_type="sha256"):
print('Using downloaded and verified file: ' + fpath)
return
@ -76,7 +105,7 @@ class RemoteFileHandler:
file_id = RemoteFileHandler._get_google_drive_file_id(url)
if file_id is not None:
return RemoteFileHandler.download_file_from_google_drive(
file_id, root, filename, md5)
file_id, root, filename, sha256)
# download the file
try:
@ -92,20 +121,21 @@ class RemoteFileHandler:
raise e
# check integrity of downloaded file
if not RemoteFileHandler.check_integrity(fpath, md5):
if not RemoteFileHandler.check_integrity(fpath,
sha256, hash_type="sha256"):
raise RuntimeError("File not found or corrupted.")
@staticmethod
def download_file_from_google_drive(file_id, root,
filename=None,
md5=None):
sha256=None):
"""Download a Google Drive file from and place it in root.
Args:
file_id (str): id of file to be downloaded
root (str): Directory to place downloaded file in
filename (str, optional): Name to save the file under.
If None, use the id of the file.
md5 (str, optional): MD5 checksum of the download.
sha256 (str, optional): sha256 checksum of the download.
If None, do not check
"""
# Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url # noqa
@ -119,8 +149,8 @@ class RemoteFileHandler:
os.makedirs(root, exist_ok=True)
if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(fpath,
md5):
if os.path.isfile(fpath) and RemoteFileHandler.check_integrity(
fpath, sha256, hash_type="sha256"):
print('Using downloaded and verified file: ' + fpath)
else:
session = requests.Session()

8
distribution/tests/test_addon_distributtion.py
View file

@ -51,7 +51,7 @@ def sample_addon_info():
}
}
],
"hash": "4f6b8568eb9dd6f510fd7c4dcb676788"
"hash": "4be25eb6215e91e5894d3c5475aeb1e379d081d3f5b43b4ee15b0891cf5f5658"
}
yield addon_info
@ -109,13 +109,13 @@ def test_update_addon_state(printer, sample_addon_info,
addon_info.hash = "brokenhash"
result = update_addon_state([addon_info], temp_folder, addon_downloader)
assert result["openpype_slack_1.0.0"] == UpdateState.FAILED.value, \
"Hashes not matching"
"Update should failed because of wrong hash"
addon_info.hash = orig_hash
result = update_addon_state([addon_info], temp_folder, addon_downloader)
assert result["openpype_slack_1.0.0"] == UpdateState.UPDATED.value, \
"Failed updating"
"Addon should have been updated"
result = update_addon_state([addon_info], temp_folder, addon_downloader)
assert result["openpype_slack_1.0.0"] == UpdateState.EXISTS.value, \
"Tried to update"
"Addon should already exist"