From 8d61529fb001c6aecdbb3b9c4903cf139ec8b721 Mon Sep 17 00:00:00 2001 From: Ondrej Samohel Date: Mon, 29 Mar 2021 13:55:43 +0200 Subject: [PATCH] mimic source hierarchy in zips for frozen and live code --- igniter/bootstrap_repos.py | 139 ++++++++++---------------- igniter/tools.py | 5 + start.py | 14 +-- tests/igniter/test_bootstrap_repos.py | 13 ++- 4 files changed, 71 insertions(+), 100 deletions(-) diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 58d59afe88..2ef22ec4fa 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -242,7 +242,7 @@ class BootstrapRepos: self.registry = PypeSettingsRegistry() self.zip_filter = [".pyc", "__pycache__"] self.pype_filter = [ - "build", "docs", "tests", "repos", "tools", "venv" + "build", "docs", "tests", "tools", "venv", "coverage" ] self._message = message @@ -351,7 +351,7 @@ class BootstrapRepos: Path(temp_dir) / f"pype-v{version}.zip" self._print(f"creating zip: {temp_zip}") - self._create_pype_zip(temp_zip, repo_dir) + self._create_pype_zip(temp_zip, repo_dir.parent) if not os.path.exists(temp_zip): self._print("make archive failed.", LOG_ERROR) return None @@ -417,15 +417,14 @@ class BootstrapRepos: """ frozen_root = Path(sys.executable).parent - repo_dir = frozen_root / "repos" - repo_list = self._filter_dir( - repo_dir, self.zip_filter) # from frozen code we need igniter, pype, schema vendor pype_list = self._filter_dir( frozen_root / "pype", self.zip_filter) pype_list += self._filter_dir( frozen_root / "igniter", self.zip_filter) + pype_list += self._filter_dir( + frozen_root / "repos", self.zip_filter) pype_list += self._filter_dir( frozen_root / "schema", self.zip_filter) pype_list += self._filter_dir( @@ -443,17 +442,7 @@ class BootstrapRepos: with ZipFile(temp_zip, "w") as zip_file: progress = 0 - repo_inc = 48.0 / float(len(repo_list)) - file: Path - for file in repo_list: - progress += repo_inc - self._progress_callback(int(progress)) - - # archive name is relative to repos dir - arc_name = file.relative_to(repo_dir) - zip_file.write(file, arc_name) - - pype_inc = 48.0 / float(len(pype_list)) + pype_inc = 98.0 / float(len(pype_list)) file: Path for file in pype_list: progress += pype_inc @@ -463,17 +452,14 @@ class BootstrapRepos: # we need to replace first part of path which starts with # something like `exe.win/linux....` with `pype` as this # is expected by Pype in zip archive. - arc_name = Path("pype").joinpath(*arc_name.parts[1:]) + arc_name = Path().joinpath(*arc_name.parts[1:]) zip_file.write(file, arc_name) destination = self._move_zip_to_data_dir(temp_zip) return PypeVersion(version=version, path=destination) - def _create_pype_zip( - self, - zip_path: Path, include_dir: Path, - include_pype: bool = True) -> None: + def _create_pype_zip(self, zip_path: Path, pype_path: Path) -> None: """Pack repositories and Pype into zip. We are using :mod:`zipfile` instead :meth:`shutil.make_archive` @@ -482,70 +468,46 @@ class BootstrapRepos: and :attr:`pype_filter` on top level directory in Pype repository. Args: - zip_path (str): path to zip file. - include_dir (Path): repo directories to include. - include_pype (bool): add Pype module itself. + zip_path (Path): Path to zip file. + pype_path (Path): Path to Pype sources. """ - include_dir = include_dir.resolve() - pype_list = [] - # get filtered list of files in repositories (repos directory) - repo_list = self._filter_dir(include_dir, self.zip_filter) - # count them - repo_files = len(repo_list) - - # there must be some files, otherwise `include_dir` path is wrong - assert repo_files != 0, f"No repositories to include in {include_dir}" pype_inc = 0 - if include_pype: - # get filtered list of file in Pype repository - pype_list = self._filter_dir(include_dir.parent, self.zip_filter) - pype_files = len(pype_list) - repo_inc = 48.0 / float(repo_files) - pype_inc = 48.0 / float(pype_files) - else: - repo_inc = 98.0 / float(repo_files) + + # get filtered list of file in Pype repository + pype_list = self._filter_dir(pype_path, self.zip_filter) + pype_files = len(pype_list) + + pype_inc = 98.0 / float(pype_files) with ZipFile(zip_path, "w") as zip_file: progress = 0 + pype_root = pype_path.resolve() + # generate list of filtered paths + dir_filter = [pype_root / f for f in self.pype_filter] + file: Path - for file in repo_list: - progress += repo_inc + for file in pype_list: + progress += pype_inc self._progress_callback(int(progress)) - # archive name is relative to repos dir - arc_name = file.relative_to(include_dir) - zip_file.write(file, arc_name) + # if file resides in filtered path, skip it + is_inside = None + df: Path + for df in dir_filter: + try: + is_inside = file.resolve().relative_to(df) + except ValueError: + pass - # add pype itself - if include_pype: - pype_root = include_dir.parent.resolve() - # generate list of filtered paths - dir_filter = [pype_root / f for f in self.pype_filter] + if is_inside: + continue - file: Path - for file in pype_list: - progress += pype_inc - self._progress_callback(int(progress)) + processed_path = file + self._print(f"- processing {processed_path}") - # if file resides in filtered path, skip it - is_inside = None - df: Path - for df in dir_filter: - try: - is_inside = file.resolve().relative_to(df) - except ValueError: - pass - - if is_inside: - continue - - processed_path = file - self._print(f"- processing {processed_path}") - - zip_file.write(file, - "pype" / file.relative_to(pype_root)) + zip_file.write(file, file.relative_to(pype_root)) # test if zip is ok zip_file.testzip() @@ -553,10 +515,10 @@ class BootstrapRepos: @staticmethod def add_paths_from_archive(archive: Path) -> None: - """Add first-level directories as paths to :mod:`sys.path`. + """Add first-level directory and 'repos' as paths to :mod:`sys.path`. - This will enable Python to import modules is second-level directories - in zip file. + This will enable Python to import Pype and modules in `repos` + submodule directory in zip file. Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. @@ -574,21 +536,29 @@ class BootstrapRepos: name_list = zip_file.namelist() roots = [] + paths = [] for item in name_list: - root = item.split("/")[0] + if not item.startswith("repos/"): + continue + + root = item.split("/")[1] + if root not in roots: roots.append(root) - sys.path.insert(0, f"{archive}{os.path.sep}{root}") + paths.append( + f"{archive}{os.path.sep}repos{os.path.sep}{root}") + sys.path.insert(0, paths[-1]) + sys.path.insert(0, f"{archive}") pythonpath = os.getenv("PYTHONPATH", "") - paths = pythonpath.split(os.pathsep) - paths += roots + python_paths = pythonpath.split(os.pathsep) + python_paths += paths - os.environ["PYTHONPATH"] = os.pathsep.join(paths) + os.environ["PYTHONPATH"] = os.pathsep.join(python_paths) @staticmethod def add_paths_from_directory(directory: Path) -> None: - """Add first level directories as paths to :mod:`sys.path`. + """Add repos first level directories as paths to :mod:`sys.path`. This works the same as :meth:`add_paths_from_archive` but in specified directory. @@ -599,6 +569,8 @@ class BootstrapRepos: directory (Path): path to directory. """ + sys.path.insert(0, directory.as_posix()) + directory = directory / "repos" if not directory.exists() and not directory.is_dir(): raise ValueError("directory is invalid") @@ -937,8 +909,7 @@ class BootstrapRepos: try: # add one 'pype' level as inside dir there should # be many other repositories. - version_str = BootstrapRepos.get_version( - dir_item / "pype") + version_str = BootstrapRepos.get_version(dir_item) version_check = PypeVersion(version=version_str) except ValueError: self._print( @@ -979,7 +950,7 @@ class BootstrapRepos: try: with ZipFile(zip_item, "r") as zip_file: with zip_file.open( - "pype/pype/version.py") as version_file: + "pype/version.py") as version_file: zip_version = {} exec(version_file.read(), zip_version) version_check = PypeVersion( diff --git a/igniter/tools.py b/igniter/tools.py index 4ed4ae67f4..17e53a2b07 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -6,6 +6,7 @@ Functions ``compose_url()`` and ``decompose_url()`` are the same as in version is decided. """ +import sys from typing import Dict, Union from urllib.parse import urlparse, parse_qs from pathlib import Path @@ -244,6 +245,10 @@ def get_pype_path_from_db(url: str) -> Union[str, None]: try: client = MongoClient(**mongo_args) + except ServerSelectionTimeoutError as e: + print("!!! Can't connect to mongodb database server.") + print("!!! {}".format(e)) + return None except Exception: return None diff --git a/start.py b/start.py index c231786797..8553449489 100644 --- a/start.py +++ b/start.py @@ -100,7 +100,6 @@ import subprocess import site from pathlib import Path -from pymongo.errors import ServerSelectionTimeoutError # add dependencies folder to sys.pat for frozen code if getattr(sys, 'frozen', False): @@ -327,13 +326,12 @@ def _initialize_environment(pype_version: PypeVersion) -> None: # to same hierarchy from code and from frozen pype additional_paths = [ # add pype tools - os.path.join(os.environ["PYPE_ROOT"], "pype", "pype", "tools"), + os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"), # add common pype vendor # (common for multiple Python interpreter versions) os.path.join( os.environ["PYPE_ROOT"], "pype", - "pype", "vendor", "python", "common" @@ -571,12 +569,10 @@ def boot(): # Get Pype path from database and set it to environment so Pype can # find its versions there and bootstrap them. - try: - pype_path = get_pype_path_from_db(pype_mongo) - except ServerSelectionTimeoutError as e: - print("!!! Can't connect to mongodb database server.") - print("!!! {}".format(e)) - sys.exit(1) + + pype_path = get_pype_path_from_db(pype_mongo) + if not pype_path: + print("*** Cannot get Pype path from database.") if not os.getenv("PYPE_PATH") and pype_path: os.environ["PYPE_PATH"] = pype_path diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index e1bdbffb30..75996b4026 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -150,9 +150,9 @@ def test_install_live_repos(fix_bootstrap, printer): pype_version = fix_bootstrap.create_version_from_live_code() sep = os.path.sep expected_paths = [ - f"{pype_version.path}{sep}avalon-core", - f"{pype_version.path}{sep}avalon-unreal-integration", - f"{pype_version.path}{sep}pype" + f"{pype_version.path}{sep}repos{sep}avalon-core", + f"{pype_version.path}{sep}repos{sep}avalon-unreal-integration", + f"{pype_version.path}" ] printer("testing zip creation") assert os.path.exists(pype_version.path), "zip archive was not created" @@ -165,10 +165,9 @@ def test_install_live_repos(fix_bootstrap, printer): import pype # noqa: F401 # test if pype is imported from specific location in zip - print(pype.__file__) assert "pype" in sys.modules.keys(), "Pype not imported" assert sys.modules["pype"].__file__ == \ - f"{pype_version.path}{sep}pype{sep}pype{sep}__init__.py" + f"{pype_version.path}{sep}pype{sep}__init__.py" def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): @@ -252,7 +251,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): def _create_valid_zip(path: Path, version: str): with ZipFile(path, "w") as zf: zf.writestr( - "pype/pype/version.py", f"__version__ = '{version}'\n\n") + "pype/version.py", f"__version__ = '{version}'\n\n") def _create_invalid_dir(path: Path): path.mkdir(parents=True, exist_ok=True) @@ -260,7 +259,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): fp.write("invalid") def _create_valid_dir(path: Path, version: str): - pype_path = path / "pype" / "pype" + pype_path = path / "pype" version_path = pype_path / "version.py" pype_path.mkdir(parents=True, exist_ok=True) with open(version_path, "w") as fp: