diff --git a/igniter/bootstrap_repos.py b/igniter/bootstrap_repos.py index 29fc26173e..16e0f212e9 100644 --- a/igniter/bootstrap_repos.py +++ b/igniter/bootstrap_repos.py @@ -242,7 +242,7 @@ class BootstrapRepos: self.registry = OpenPypeSettingsRegistry() self.zip_filter = [".pyc", "__pycache__"] self.openpype_filter = [ - "build", "docs", "tests", "repos", "tools", "venv" + "build", "docs", "tests", "tools", "venv", "coverage" ] self._message = message @@ -352,7 +352,7 @@ class BootstrapRepos: Path(temp_dir) / f"openpype-v{version}.zip" self._print(f"creating zip: {temp_zip}") - self._create_openpype_zip(temp_zip, repo_dir) + self._create_openpype_zip(temp_zip, repo_dir.parent) if not os.path.exists(temp_zip): self._print("make archive failed.", LOG_ERROR) return None @@ -418,20 +418,18 @@ class BootstrapRepos: """ frozen_root = Path(sys.executable).parent - repo_dir = frozen_root / "repos" - repo_list = self._filter_dir( - repo_dir, self.zip_filter) # from frozen code we need igniter, openpype, schema vendor openpype_list = self._filter_dir( frozen_root / "openpype", self.zip_filter) openpype_list += self._filter_dir( frozen_root / "igniter", self.zip_filter) + openpype_list += self._filter_dir( + frozen_root / "repos", self.zip_filter) openpype_list += self._filter_dir( frozen_root / "schema", self.zip_filter) openpype_list += self._filter_dir( frozen_root / "vendor", self.zip_filter) - openpype_list.append(frozen_root / "README.md") openpype_list.append(frozen_root / "LICENSE") version = self.get_version(frozen_root) @@ -444,17 +442,7 @@ class BootstrapRepos: with ZipFile(temp_zip, "w") as zip_file: progress = 0 - repo_inc = 48.0 / float(len(repo_list)) - file: Path - for file in repo_list: - progress += repo_inc - self._progress_callback(int(progress)) - - # archive name is relative to repos dir - arc_name = file.relative_to(repo_dir) - zip_file.write(file, arc_name) - - openpype_inc = 48.0 / float(len(openpype_list)) + openpype_inc = 98.0 / float(len(openpype_list)) file: Path for file in openpype_list: progress += openpype_inc @@ -462,19 +450,16 @@ class BootstrapRepos: arc_name = file.relative_to(frozen_root.parent) # we need to replace first part of path which starts with - # something like `exe.win/linux....` with `openpype` as this - # is expected by OpenPype in zip archive. - arc_name = Path("openpype").joinpath(*arc_name.parts[1:]) + # something like `exe.win/linux....` with `openpype` as + # this is expected by OpenPype in zip archive. + arc_name = Path().joinpath(*arc_name.parts[1:]) zip_file.write(file, arc_name) destination = self._move_zip_to_data_dir(temp_zip) return OpenPypeVersion(version=version, path=destination) - def _create_poenpype_zip( - self, - zip_path: Path, include_dir: Path, - include_openpype: bool = True) -> None: + def _create_openpype_zip(self, zip_path: Path, openpype_path: Path) -> None: """Pack repositories and OpenPype into zip. We are using :mod:`zipfile` instead :meth:`shutil.make_archive` @@ -484,71 +469,46 @@ class BootstrapRepos: repository. Args: - zip_path (str): path to zip file. - include_dir (Path): repo directories to include. - include_openpype (bool): add OpenPype module itself. + zip_path (Path): Path to zip file. + openpype_path (Path): Path to OpenPype sources. """ - include_dir = include_dir.resolve() - openpype_list = [] - # get filtered list of files in repositories (repos directory) - repo_list = self._filter_dir(include_dir, self.zip_filter) - # count them - repo_files = len(repo_list) - - # there must be some files, otherwise `include_dir` path is wrong - assert repo_files != 0, f"No repositories to include in {include_dir}" openpype_inc = 0 - if include_openpype: - # get filtered list of file in Pype repository - openpype_list = self._filter_dir(include_dir.parent, self.zip_filter) - openpype_files = len(openpype_list) - repo_inc = 48.0 / float(repo_files) - openpype_inc = 48.0 / float(openpype_files) - else: - repo_inc = 98.0 / float(repo_files) + + # get filtered list of file in Pype repository + openpype_list = self._filter_dir(openpype_path, self.zip_filter) + openpype_files = len(openpype_list) + + openpype_inc = 98.0 / float(openpype_files) with ZipFile(zip_path, "w") as zip_file: progress = 0 + openpype_root = openpype_path.resolve() + # generate list of filtered paths + dir_filter = [openpype_root / f for f in self.openpype_filter] + file: Path - for file in repo_list: - progress += repo_inc + for file in openpype_list: + progress += openpype_inc self._progress_callback(int(progress)) - # archive name is relative to repos dir - arc_name = file.relative_to(include_dir) - zip_file.write(file, arc_name) + # if file resides in filtered path, skip it + is_inside = None + df: Path + for df in dir_filter: + try: + is_inside = file.resolve().relative_to(df) + except ValueError: + pass - # add openpype itself - if include_openpype: - openpype_root = include_dir.parent.resolve() - # generate list of filtered paths - dir_filter = [openpype_root / f for f in self.openpype_filter] + if is_inside: + continue - file: Path - for file in openpype_list: - progress += openpype_inc - self._progress_callback(int(progress)) + processed_path = file + self._print(f"- processing {processed_path}") - # if file resides in filtered path, skip it - is_inside = None - df: Path - for df in dir_filter: - try: - is_inside = file.resolve().relative_to(df) - except ValueError: - pass - - if is_inside: - continue - - processed_path = file - self._print(f"- processing {processed_path}") - - zip_file.write(file, - "openpype" / file.relative_to( - openpype_root)) + zip_file.write(file, file.relative_to(openpype_root)) # test if zip is ok zip_file.testzip() @@ -556,10 +516,10 @@ class BootstrapRepos: @staticmethod def add_paths_from_archive(archive: Path) -> None: - """Add first-level directories as paths to :mod:`sys.path`. + """Add first-level directory and 'repos' as paths to :mod:`sys.path`. - This will enable Python to import modules is second-level directories - in zip file. + This will enable Python to import OpenPype and modules in `repos` + submodule directory in zip file. Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates. @@ -577,21 +537,29 @@ class BootstrapRepos: name_list = zip_file.namelist() roots = [] + paths = [] for item in name_list: - root = item.split("/")[0] + if not item.startswith("repos/"): + continue + + root = item.split("/")[1] + if root not in roots: roots.append(root) - sys.path.insert(0, f"{archive}{os.path.sep}{root}") + paths.append( + f"{archive}{os.path.sep}repos{os.path.sep}{root}") + sys.path.insert(0, paths[-1]) + sys.path.insert(0, f"{archive}") pythonpath = os.getenv("PYTHONPATH", "") - paths = pythonpath.split(os.pathsep) - paths += roots + python_paths = pythonpath.split(os.pathsep) + python_paths += paths - os.environ["PYTHONPATH"] = os.pathsep.join(paths) + os.environ["PYTHONPATH"] = os.pathsep.join(python_paths) @staticmethod def add_paths_from_directory(directory: Path) -> None: - """Add first level directories as paths to :mod:`sys.path`. + """Add repos first level directories as paths to :mod:`sys.path`. This works the same as :meth:`add_paths_from_archive` but in specified directory. @@ -602,6 +570,8 @@ class BootstrapRepos: directory (Path): path to directory. """ + sys.path.insert(0, directory.as_posix()) + directory = directory / "repos" if not directory.exists() and not directory.is_dir(): raise ValueError("directory is invalid") @@ -943,8 +913,7 @@ class BootstrapRepos: try: # add one 'openpype' level as inside dir there should # be many other repositories. - version_str = BootstrapRepos.get_version( - dir_item / "openpype") + version_str = BootstrapRepos.get_version(dir_item) version_check = OpenPypeVersion(version=version_str) except ValueError: self._print( diff --git a/igniter/tools.py b/igniter/tools.py index f25fcb3122..ff2db6bc7e 100644 --- a/igniter/tools.py +++ b/igniter/tools.py @@ -6,6 +6,7 @@ Functions ``compose_url()`` and ``decompose_url()`` are the same as in version is decided. """ +import sys import os from typing import Dict, Union from urllib.parse import urlparse, parse_qs diff --git a/start.py b/start.py index e066182c96..f3b2b337cc 100644 --- a/start.py +++ b/start.py @@ -100,6 +100,7 @@ import subprocess import site from pathlib import Path + # add dependencies folder to sys.pat for frozen code if getattr(sys, 'frozen', False): frozen_libs = os.path.normpath( @@ -568,9 +569,13 @@ def boot(): else: os.environ["PYPE_ROOT"] = os.path.dirname(__file__) - # Get OpenPype path from database and set it to environment so OpenPype - # can find its versions there and bootstrap them. + # Get Pype path from database and set it to environment so Pype can + # find its versions there and bootstrap them. + openpype_path = get_pype_path_from_db(openpype_mongo) + if not openpype_path: + print("*** Cannot get Pype path from database.") + if not os.getenv("PYPE_PATH") and openpype_path: os.environ["PYPE_PATH"] = openpype_path diff --git a/tests/igniter/test_bootstrap_repos.py b/tests/igniter/test_bootstrap_repos.py index 70edc5b89c..75996b4026 100644 --- a/tests/igniter/test_bootstrap_repos.py +++ b/tests/igniter/test_bootstrap_repos.py @@ -145,18 +145,18 @@ def test_search_string_for_pype_version(printer): assert PypeVersion.version_in_str(ver_string[0])[0] == ver_string[1] +@pytest.mark.slow def test_install_live_repos(fix_bootstrap, printer): - rf = fix_bootstrap.create_version_from_live_code() + pype_version = fix_bootstrap.create_version_from_live_code() sep = os.path.sep expected_paths = [ - f"{rf}{sep}avalon-core", - f"{rf}{sep}avalon-unreal-integration", - f"{rf}{sep}maya-look-assigner", - f"{rf}{sep}pype" + f"{pype_version.path}{sep}repos{sep}avalon-core", + f"{pype_version.path}{sep}repos{sep}avalon-unreal-integration", + f"{pype_version.path}" ] printer("testing zip creation") - assert os.path.exists(rf), "zip archive was not created" - fix_bootstrap.add_paths_from_archive(rf) + assert os.path.exists(pype_version.path), "zip archive was not created" + fix_bootstrap.add_paths_from_archive(pype_version.path) for ep in expected_paths: assert ep in sys.path, f"{ep} not set correctly" @@ -165,10 +165,9 @@ def test_install_live_repos(fix_bootstrap, printer): import pype # noqa: F401 # test if pype is imported from specific location in zip - print(pype.__file__) assert "pype" in sys.modules.keys(), "Pype not imported" assert sys.modules["pype"].__file__ == \ - f"{rf}{sep}pype{sep}pype{sep}__init__.py" + f"{pype_version.path}{sep}pype{sep}__init__.py" def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): @@ -252,7 +251,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): def _create_valid_zip(path: Path, version: str): with ZipFile(path, "w") as zf: zf.writestr( - "pype/pype/version.py", f"__version__ = '{version}'\n\n") + "pype/version.py", f"__version__ = '{version}'\n\n") def _create_invalid_dir(path: Path): path.mkdir(parents=True, exist_ok=True) @@ -260,7 +259,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer): fp.write("invalid") def _create_valid_dir(path: Path, version: str): - pype_path = path / "pype" / "pype" + pype_path = path / "pype" version_path = pype_path / "version.py" pype_path.mkdir(parents=True, exist_ok=True) with open(version_path, "w") as fp: diff --git a/tests/igniter/test_tools.py b/tests/igniter/test_tools.py index b87900311a..0c7104f698 100644 --- a/tests/igniter/test_tools.py +++ b/tests/igniter/test_tools.py @@ -10,7 +10,3 @@ def test_validate_path_string(tmp_path): status2, _ = validate_path_string("booo" + str(uuid4())) assert status2 is False - # todo: change when Pype token is implemented - status3, reason = validate_path_string(str(uuid4())) - assert status3 is False - assert reason == "Not implemented yet" diff --git a/tools/run_mongo.ps1 b/tools/run_mongo.ps1 index da5f4aaa26..7e43a355e5 100644 --- a/tools/run_mongo.ps1 +++ b/tools/run_mongo.ps1 @@ -34,37 +34,39 @@ function Exit-WithCode($exitcode) { function Find-Mongo { - Write-Host ">>> " -NoNewLine -ForegroundColor Green - Write-Host "Detecting MongoDB ... " -NoNewline - if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) { - if(Test-Path 'C:\Program Files\MongoDB\Server\*\bin\mongod.exe' -PathType Leaf) { - # we have mongo server installed on standard Windows location - # so we can inject it to the PATH. We'll use latest version available. - $mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]} - if(Test-Path "$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) { - $env:PATH="$($env:PATH);$($mongoVersions[-1])\bin\" - Write-Host "OK" -ForegroundColor Green - Write-Host " - auto-added from [ " -NoNewline - Write-Host "$($mongoVersions[-1])\bin\" -NoNewLine -ForegroundColor Cyan - Write-Host " ]" - } else { - Write-Host "FAILED " -NoNewLine -ForegroundColor Red - Write-Host "MongoDB not detected" -ForegroundColor Yellow - Write-Host "Tried to find it on standard location " -NoNewline -ForegroundColor Gray - Write-Host " [ " -NoNewline -ForegroundColor Cyan - Write-Host "$($mongoVersions[-1])\bin\" -NoNewline -ForegroundColor White - Write-Host " ] " -NonNewLine -ForegroundColor Cyan - Write-Host "but failed." -ForegroundColor Gray - Exit-WithCode 1 - } + $defaultPath = "C:\Program Files\MongoDB\Server" + Write-Host ">>> " -NoNewLine -ForegroundColor Green + Write-Host "Detecting MongoDB ... " -NoNewline + if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) { + if(Test-Path "$($defaultPath)\*\bin\mongod.exe" -PathType Leaf) { + # we have mongo server installed on standard Windows location + # so we can inject it to the PATH. We'll use latest version available. + $mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]} + if(Test-Path "$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) { + $env:PATH = "$($env:PATH);$($mongoVersions[-1])\bin\" + Write-Host "OK" -ForegroundColor Green + Write-Host " - auto-added from [ " -NoNewline + Write-Host "$($mongoVersions[-1])\bin\mongod.exe" -NoNewLine -ForegroundColor Cyan + Write-Host " ]" + return "$($mongoVersions[-1])\bin\mongod.exe" + } else { + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected" -ForegroundColor Yellow + Write-Host "Tried to find it on standard location " -NoNewline -ForegroundColor Gray + Write-Host " [ " -NoNewline -ForegroundColor Cyan + Write-Host "$($mongoVersions[-1])\bin\mongod.exe" -NoNewline -ForegroundColor White + Write-Host " ] " -NoNewLine -ForegroundColor Cyan + Write-Host "but failed." -ForegroundColor Gray + Exit-WithCode 1 + } } else { - Write-Host "FAILED " -NoNewLine -ForegroundColor Red - Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow - Exit-WithCode 1 + Write-Host "FAILED " -NoNewLine -ForegroundColor Red + Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow + Exit-WithCode 1 } - } else { - Write-Host "OK" -ForegroundColor Green + Write-Host "OK" -ForegroundColor Green + return Get-Command "mongod" -ErrorAction SilentlyContinue } <# .SYNOPSIS @@ -85,6 +87,7 @@ $port = 2707 # path to database $dbpath = (Get-Item $openpype_root).parent.FullName + "\mongo_db_data" -Find-Mongo -$mongo = Get-Command "mongod" | Select-Object -ExpandProperty Definition -Start-Process -FilePath $mongo "--dbpath $($dbpath) --port $($port)" +$mongoPath = Find-Mongo +Start-Process -FilePath $mongopath "--dbpath $($dbpath) --port $($port)" -PassThru + +