From cc580b07f3112279f7635ed015c69ca17fd74991 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Tue, 16 Nov 2021 18:34:14 +0100 Subject: [PATCH] version repacker command --- igniter/bootstrap_repos.py | 35 ++-- openpype/cli.py | 12 ++ openpype/pype_commands.py | 7 + openpype/tools/repack_version.py | 185 +++++++++++++++------ tests/unit/igniter/test_bootstrap_repos.py | 2 +- 5 files changed, 174 insertions(+), 67 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index f7f35824c8..a7b79b2ac0 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -10,6 +10,7 @@ import tempfile from pathlib import Path from typing import Union, Callable, List, Tuple import hashlib +import platform from zipfile import ZipFile, BadZipFile @@ -196,21 +197,21 @@ class OpenPypeVersion(semver.VersionInfo): return str(self.finalize_version()) @staticmethod - def version_in_str(string: str) -> Tuple: + def version_in_str(string: str) -> Union[None, OpenPypeVersion]: """Find OpenPype version in given string. Args: string (str): string to search. Returns: - tuple: True/False and OpenPypeVersion if found. + OpenPypeVersion: of detected or None. """ m = re.search(OpenPypeVersion._VERSION_REGEX, string) if not m: - return False, None + return None version = OpenPypeVersion.parse(string[m.start():m.end()]) - return True, version + return version @classmethod def parse(cls, version): @@ -531,6 +532,7 @@ class BootstrapRepos: processed_path = file self._print(f"- processing {processed_path}") + checksums.append( ( sha256sum(file.as_posix()), @@ -542,7 +544,10 @@ class BootstrapRepos: checksums_str = "" for c in checksums: - checksums_str += "{}:{}\n".format(c[0], c[1]) + file_str = c[1] + if platform.system().lower() == "windows": + file_str = c[1].as_posix().replace("\\", "/") + checksums_str += "{}:{}\n".format(c[0], file_str) zip_file.writestr("checksums", checksums_str) # test if zip is ok zip_file.testzip() @@ -589,13 +594,16 @@ class BootstrapRepos: # calculate and compare checksums in the zip file for file in checksums: + file_name = file[1] + if platform.system().lower() == "windows": + file_name = file_name.replace("/", "\\") h = hashlib.sha256() try: - h.update(zip_file.read(file[1])) + h.update(zip_file.read(file_name)) except FileNotFoundError: - return False, f"Missing file [ {file[1]} ]" + return False, f"Missing file [ {file_name} ]" if h.hexdigest() != file[0]: - return False, f"Invalid checksum on {file[1]}" + return False, f"Invalid checksum on {file_name}" # get list of files in zip minus `checksums` file itself # and turn in to set to compare against list of files @@ -631,13 +639,16 @@ class BootstrapRepos: files_in_checksum = set([file[1] for file in checksums]) for file in checksums: + file_name = file[1] + if platform.system().lower() == "windows": + file_name = file_name.replace("/", "\\") try: - current = sha256sum((path / file[1]).as_posix()) + current = sha256sum((path / file_name).as_posix()) except FileNotFoundError: - return False, f"Missing file [ {file[1]} ]" + return False, f"Missing file [ {file_name} ]" if file[0] != current: - return False, f"Invalid checksum on {file[1]}" + return False, f"Invalid checksum on {file_name}" diff = files_in_dir.difference(files_in_checksum) if diff: return False, f"Missing files {diff}" @@ -1163,7 +1174,7 @@ class BootstrapRepos: if result[0]: detected_version: OpenPypeVersion - detected_version = result[1] + detected_version = result if item.is_dir() and not self._is_openpype_in_dir( item, detected_version diff --git a/openpype/cli.py b/openpype/cli.py index 3194723d4c..e5aaca3db1 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -384,3 +384,15 @@ def syncserver(debug, active_site): if debug: os.environ['OPENPYPE_DEBUG'] = '3' PypeCommands().syncserver(active_site) + + +@main.command() +@click.argument("directory", help="OpenPype version directory") +def repack_version(directory): + """Repack OpenPype version from directory. + + This command will re-create zip file from specified directory, + recalculating file checksums. It will try to use version detected in + directory name. + """ + PypeCommands().repack_version(directory) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index f72c7404f9..37430f879c 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -405,3 +405,10 @@ class PypeCommands: import time while True: time.sleep(1.0) + + def repack_version(self, directory): + """Repacking OpenPype version.""" + from openpype.tools.repack_version import VersionRepacker + + version_packer = VersionRepacker(directory) + version_packer.process() diff --git a/openpype/tools/repack_version.py b/openpype/tools/repack_version.py index ab1a095632..15e75e0469 100644 --- a/openpype/tools/repack_version.py +++ b/openpype/tools/repack_version.py @@ -1,66 +1,143 @@ # -*- coding: utf-8 -*- """Script to rehash and repack current version.""" -from igniter import bootstrap_repos + import enlighten import blessed from pathlib import Path import os +import platform from zipfile import ZipFile, BadZipFile -from igniter.bootstrap_repos import sha256sum +from typing import Union, Callable, List, Tuple +import hashlib +import sys +from igniter.bootstrap_repos import OpenPypeVersion +import re -term = blessed.Terminal() -manager = enlighten.get_manager() -last_increment = 0 +class VersionRepacker: -progress_bar = enlighten.Counter( - total=100, desc="OpenPype ZIP", units="%", color="green") + def __init__(self, directory: str): + self._term = blessed.Terminal() + self._manager = enlighten.get_manager() + self._last_increment = 0 + self.version_path = Path(directory) + self.zip_path = self.version_path.parent + _version = {} + with open(self.version_path / "openpype" / "version.py") as fp: + exec(fp.read(), _version) + self._version_py = _version["__version__"] + del _version + + def _print(self, msg: str, message_type: int = 0) -> None: + """Print message to console. + + Args: + msg (str): message to print + message_type (int): type of message (0 info, 1 error, 2 note) + + """ + if message_type == 0: + header = self._term.aquamarine3(">>> ") + elif message_type == 1: + header = self._term.orangered2("!!! ") + elif message_type == 2: + header = self._term.tan1("... ") + else: + header = self._term.darkolivegreen3("--- ") + + print("{}{}".format(header, msg)) + + @staticmethod + def sha256sum(filename): + """Calculate sha256 for content of the file. + + Args: + filename (str): Path to file. + + Returns: + str: hex encoded sha256 + + """ + h = hashlib.sha256() + b = bytearray(128 * 1024) + mv = memoryview(b) + with open(filename, 'rb', buffering=0) as f: + for n in iter(lambda: f.readinto(mv), 0): + h.update(mv[:n]) + return h.hexdigest() + + @staticmethod + def _filter_dir(path: Path, path_filter: List) -> List[Path]: + """Recursively crawl over path and filter.""" + result = [] + for item in path.iterdir(): + if item.name in path_filter: + continue + if item.name.startswith('.'): + continue + if item.is_dir(): + result.extend(VersionRepacker._filter_dir(item, path_filter)) + else: + result.append(item) + return result + + def process(self): + if (self.version_path / "pyproject.toml").exists(): + self._print( + ("This cannot run on OpenPype sources. " + "Please run it on extracted version."), 1) + return + self._print(f"Rehashing and zipping {self.version_path}") + version = OpenPypeVersion.version_in_str(self.version_path.name) + if not version: + self._print("Cannot get version from directory", 1) + return + + self._print(f"Detected version is {version}") + self._print(f"Recalculating checksums ...", 2) + + checksums = [] + + file_list = VersionRepacker._filter_dir(self.version_path, []) + progress_bar = enlighten.Counter( + total=len(file_list), desc="Calculating checksums", + nits="%", color="green") + for file in file_list: + checksums.append(( + VersionRepacker.sha256sum(file.as_posix()), + file.resolve().relative_to(self.version_path), + file + )) + progress_bar.update() + progress_bar.close() + + progress_bar = enlighten.Counter( + total=len(checksums), desc="Zipping directory", + nits="%", color=(56, 211, 159)) + + with ZipFile(self.zip_path, "w") as zip_file: + checksums = [] + + for item in checksums: + + processed_path = item[0] + self._print(f"- processing {processed_path}") + + zip_file.write(item[2], item[1]) + progress_bar.update() + + checksums_str = "" + for c in checksums: + file_str = c[1] + if platform.system().lower() == "windows": + file_str = c[1].as_posix().replace("\\", "/") + checksums_str += "{}:{}\n".format(c[0], file_str) + zip_file.writestr("checksums", checksums_str) + # test if zip is ok + zip_file.testzip() - -zip_path = Path(version_path).parent - - -with ZipFile(zip_path, "w") as zip_file: - progress = 0 - openpype_root = openpype_path.resolve() - # generate list of filtered paths - dir_filter = [openpype_root / f for f in self.openpype_filter] - checksums = [] - - file: Path - for file in openpype_list: - progress += openpype_inc - self._progress_callback(int(progress)) - - # if file resides in filtered path, skip it - is_inside = None - df: Path - for df in dir_filter: - try: - is_inside = file.resolve().relative_to(df) - except ValueError: - pass - - if not is_inside: - continue - - processed_path = file - self._print(f"- processing {processed_path}") - - checksums.append( - ( - sha256sum(file.as_posix()), - file.resolve().relative_to(openpype_root) - ) - ) - zip_file.write( - file, file.resolve().relative_to(openpype_root)) - - checksums_str = "" - for c in checksums: - checksums_str += "{}:{}\n".format(c[0], c[1]) - zip_file.writestr("checksums", checksums_str) - # test if zip is ok - zip_file.testzip() - self._progress_callback(100) +if __name__ == '__main__': + print(sys.argv[1]) + version_packer = VersionRepacker(sys.argv[1]) + version_packer.process() diff --git a/tests/unit/igniter/test_bootstrap_repos.py b/tests/unit/igniter/test_bootstrap_repos.py index 740a71a5ce..d6e861c262 100644 --- a/tests/unit/igniter/test_bootstrap_repos.py +++ b/tests/unit/igniter/test_bootstrap_repos.py @@ -140,7 +140,7 @@ def test_search_string_for_openpype_version(printer): ] for ver_string in strings: printer(f"testing {ver_string[0]} should be {ver_string[1]}") - assert OpenPypeVersion.version_in_str(ver_string[0])[0] == \ + assert OpenPypeVersion.version_in_str(ver_string[0]) == \ ver_string[1]