mirror of
https://github.com/ynput/ayon-core.git
synced 2025-12-24 12:54:40 +01:00
Merge remote-tracking branch 'openpype/bugfix/pype-modules-path-unification' into openpype-infrastructure
# Conflicts: # igniter/bootstrap_repos.py # igniter/tools.py # start.py
This commit is contained in:
Ondrej Samohel
commit
2a5c620087
6 changed files with 108 additions and 135 deletions
|
|
@ -242,7 +242,7 @@ class BootstrapRepos:
|
|||
self.registry = OpenPypeSettingsRegistry()
|
||||
self.zip_filter = [".pyc", "__pycache__"]
|
||||
self.openpype_filter = [
|
||||
"build", "docs", "tests", "repos", "tools", "venv"
|
||||
"build", "docs", "tests", "tools", "venv", "coverage"
|
||||
]
|
||||
self._message = message
|
||||
|
||||
|
|
@ -352,7 +352,7 @@ class BootstrapRepos:
|
|||
Path(temp_dir) / f"openpype-v{version}.zip"
|
||||
self._print(f"creating zip: {temp_zip}")
|
||||
|
||||
self._create_openpype_zip(temp_zip, repo_dir)
|
||||
self._create_openpype_zip(temp_zip, repo_dir.parent)
|
||||
if not os.path.exists(temp_zip):
|
||||
self._print("make archive failed.", LOG_ERROR)
|
||||
return None
|
||||
|
|
@ -418,20 +418,18 @@ class BootstrapRepos:
|
|||
|
||||
"""
|
||||
frozen_root = Path(sys.executable).parent
|
||||
repo_dir = frozen_root / "repos"
|
||||
repo_list = self._filter_dir(
|
||||
repo_dir, self.zip_filter)
|
||||
|
||||
# from frozen code we need igniter, openpype, schema vendor
|
||||
openpype_list = self._filter_dir(
|
||||
frozen_root / "openpype", self.zip_filter)
|
||||
openpype_list += self._filter_dir(
|
||||
frozen_root / "igniter", self.zip_filter)
|
||||
openpype_list += self._filter_dir(
|
||||
frozen_root / "repos", self.zip_filter)
|
||||
openpype_list += self._filter_dir(
|
||||
frozen_root / "schema", self.zip_filter)
|
||||
openpype_list += self._filter_dir(
|
||||
frozen_root / "vendor", self.zip_filter)
|
||||
openpype_list.append(frozen_root / "README.md")
|
||||
openpype_list.append(frozen_root / "LICENSE")
|
||||
|
||||
version = self.get_version(frozen_root)
|
||||
|
|
@ -444,17 +442,7 @@ class BootstrapRepos:
|
|||
|
||||
with ZipFile(temp_zip, "w") as zip_file:
|
||||
progress = 0
|
||||
repo_inc = 48.0 / float(len(repo_list))
|
||||
file: Path
|
||||
for file in repo_list:
|
||||
progress += repo_inc
|
||||
self._progress_callback(int(progress))
|
||||
|
||||
# archive name is relative to repos dir
|
||||
arc_name = file.relative_to(repo_dir)
|
||||
zip_file.write(file, arc_name)
|
||||
|
||||
openpype_inc = 48.0 / float(len(openpype_list))
|
||||
openpype_inc = 98.0 / float(len(openpype_list))
|
||||
file: Path
|
||||
for file in openpype_list:
|
||||
progress += openpype_inc
|
||||
|
|
@ -462,19 +450,16 @@ class BootstrapRepos:
|
|||
|
||||
arc_name = file.relative_to(frozen_root.parent)
|
||||
# we need to replace first part of path which starts with
|
||||
# something like `exe.win/linux....` with `openpype` as this
|
||||
# is expected by OpenPype in zip archive.
|
||||
arc_name = Path("openpype").joinpath(*arc_name.parts[1:])
|
||||
# something like `exe.win/linux....` with `openpype` as
|
||||
# this is expected by OpenPype in zip archive.
|
||||
arc_name = Path().joinpath(*arc_name.parts[1:])
|
||||
zip_file.write(file, arc_name)
|
||||
|
||||
destination = self._move_zip_to_data_dir(temp_zip)
|
||||
|
||||
return OpenPypeVersion(version=version, path=destination)
|
||||
|
||||
def _create_poenpype_zip(
|
||||
self,
|
||||
zip_path: Path, include_dir: Path,
|
||||
include_openpype: bool = True) -> None:
|
||||
def _create_openpype_zip(self, zip_path: Path, openpype_path: Path) -> None:
|
||||
"""Pack repositories and OpenPype into zip.
|
||||
|
||||
We are using :mod:`zipfile` instead :meth:`shutil.make_archive`
|
||||
|
|
@ -484,71 +469,46 @@ class BootstrapRepos:
|
|||
repository.
|
||||
|
||||
Args:
|
||||
zip_path (str): path to zip file.
|
||||
include_dir (Path): repo directories to include.
|
||||
include_openpype (bool): add OpenPype module itself.
|
||||
zip_path (Path): Path to zip file.
|
||||
openpype_path (Path): Path to OpenPype sources.
|
||||
|
||||
"""
|
||||
include_dir = include_dir.resolve()
|
||||
|
||||
openpype_list = []
|
||||
# get filtered list of files in repositories (repos directory)
|
||||
repo_list = self._filter_dir(include_dir, self.zip_filter)
|
||||
# count them
|
||||
repo_files = len(repo_list)
|
||||
|
||||
# there must be some files, otherwise `include_dir` path is wrong
|
||||
assert repo_files != 0, f"No repositories to include in {include_dir}"
|
||||
openpype_inc = 0
|
||||
if include_openpype:
|
||||
# get filtered list of file in Pype repository
|
||||
openpype_list = self._filter_dir(include_dir.parent, self.zip_filter)
|
||||
openpype_files = len(openpype_list)
|
||||
repo_inc = 48.0 / float(repo_files)
|
||||
openpype_inc = 48.0 / float(openpype_files)
|
||||
else:
|
||||
repo_inc = 98.0 / float(repo_files)
|
||||
|
||||
# get filtered list of file in Pype repository
|
||||
openpype_list = self._filter_dir(openpype_path, self.zip_filter)
|
||||
openpype_files = len(openpype_list)
|
||||
|
||||
openpype_inc = 98.0 / float(openpype_files)
|
||||
|
||||
with ZipFile(zip_path, "w") as zip_file:
|
||||
progress = 0
|
||||
openpype_root = openpype_path.resolve()
|
||||
# generate list of filtered paths
|
||||
dir_filter = [openpype_root / f for f in self.openpype_filter]
|
||||
|
||||
file: Path
|
||||
for file in repo_list:
|
||||
progress += repo_inc
|
||||
for file in openpype_list:
|
||||
progress += openpype_inc
|
||||
self._progress_callback(int(progress))
|
||||
|
||||
# archive name is relative to repos dir
|
||||
arc_name = file.relative_to(include_dir)
|
||||
zip_file.write(file, arc_name)
|
||||
# if file resides in filtered path, skip it
|
||||
is_inside = None
|
||||
df: Path
|
||||
for df in dir_filter:
|
||||
try:
|
||||
is_inside = file.resolve().relative_to(df)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# add openpype itself
|
||||
if include_openpype:
|
||||
openpype_root = include_dir.parent.resolve()
|
||||
# generate list of filtered paths
|
||||
dir_filter = [openpype_root / f for f in self.openpype_filter]
|
||||
if is_inside:
|
||||
continue
|
||||
|
||||
file: Path
|
||||
for file in openpype_list:
|
||||
progress += openpype_inc
|
||||
self._progress_callback(int(progress))
|
||||
processed_path = file
|
||||
self._print(f"- processing {processed_path}")
|
||||
|
||||
# if file resides in filtered path, skip it
|
||||
is_inside = None
|
||||
df: Path
|
||||
for df in dir_filter:
|
||||
try:
|
||||
is_inside = file.resolve().relative_to(df)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if is_inside:
|
||||
continue
|
||||
|
||||
processed_path = file
|
||||
self._print(f"- processing {processed_path}")
|
||||
|
||||
zip_file.write(file,
|
||||
"openpype" / file.relative_to(
|
||||
openpype_root))
|
||||
zip_file.write(file, file.relative_to(openpype_root))
|
||||
|
||||
# test if zip is ok
|
||||
zip_file.testzip()
|
||||
|
|
@ -556,10 +516,10 @@ class BootstrapRepos:
|
|||
|
||||
@staticmethod
|
||||
def add_paths_from_archive(archive: Path) -> None:
|
||||
"""Add first-level directories as paths to :mod:`sys.path`.
|
||||
"""Add first-level directory and 'repos' as paths to :mod:`sys.path`.
|
||||
|
||||
This will enable Python to import modules is second-level directories
|
||||
in zip file.
|
||||
This will enable Python to import OpenPype and modules in `repos`
|
||||
submodule directory in zip file.
|
||||
|
||||
Adding to both `sys.path` and `PYTHONPATH`, skipping duplicates.
|
||||
|
||||
|
|
@ -577,21 +537,29 @@ class BootstrapRepos:
|
|||
name_list = zip_file.namelist()
|
||||
|
||||
roots = []
|
||||
paths = []
|
||||
for item in name_list:
|
||||
root = item.split("/")[0]
|
||||
if not item.startswith("repos/"):
|
||||
continue
|
||||
|
||||
root = item.split("/")[1]
|
||||
|
||||
if root not in roots:
|
||||
roots.append(root)
|
||||
sys.path.insert(0, f"{archive}{os.path.sep}{root}")
|
||||
paths.append(
|
||||
f"{archive}{os.path.sep}repos{os.path.sep}{root}")
|
||||
sys.path.insert(0, paths[-1])
|
||||
|
||||
sys.path.insert(0, f"{archive}")
|
||||
pythonpath = os.getenv("PYTHONPATH", "")
|
||||
paths = pythonpath.split(os.pathsep)
|
||||
paths += roots
|
||||
python_paths = pythonpath.split(os.pathsep)
|
||||
python_paths += paths
|
||||
|
||||
os.environ["PYTHONPATH"] = os.pathsep.join(paths)
|
||||
os.environ["PYTHONPATH"] = os.pathsep.join(python_paths)
|
||||
|
||||
@staticmethod
|
||||
def add_paths_from_directory(directory: Path) -> None:
|
||||
"""Add first level directories as paths to :mod:`sys.path`.
|
||||
"""Add repos first level directories as paths to :mod:`sys.path`.
|
||||
|
||||
This works the same as :meth:`add_paths_from_archive` but in
|
||||
specified directory.
|
||||
|
|
@ -602,6 +570,8 @@ class BootstrapRepos:
|
|||
directory (Path): path to directory.
|
||||
|
||||
"""
|
||||
sys.path.insert(0, directory.as_posix())
|
||||
directory = directory / "repos"
|
||||
if not directory.exists() and not directory.is_dir():
|
||||
raise ValueError("directory is invalid")
|
||||
|
||||
|
|
@ -943,8 +913,7 @@ class BootstrapRepos:
|
|||
try:
|
||||
# add one 'openpype' level as inside dir there should
|
||||
# be many other repositories.
|
||||
version_str = BootstrapRepos.get_version(
|
||||
dir_item / "openpype")
|
||||
version_str = BootstrapRepos.get_version(dir_item)
|
||||
version_check = OpenPypeVersion(version=version_str)
|
||||
except ValueError:
|
||||
self._print(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Functions ``compose_url()`` and ``decompose_url()`` are the same as in
|
|||
version is decided.
|
||||
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Union
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
|
|
|||
9
start.py
9
start.py
|
|
@ -100,6 +100,7 @@ import subprocess
|
|||
import site
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# add dependencies folder to sys.pat for frozen code
|
||||
if getattr(sys, 'frozen', False):
|
||||
frozen_libs = os.path.normpath(
|
||||
|
|
@ -568,9 +569,13 @@ def boot():
|
|||
else:
|
||||
os.environ["PYPE_ROOT"] = os.path.dirname(__file__)
|
||||
|
||||
# Get OpenPype path from database and set it to environment so OpenPype
|
||||
# can find its versions there and bootstrap them.
|
||||
# Get Pype path from database and set it to environment so Pype can
|
||||
# find its versions there and bootstrap them.
|
||||
|
||||
openpype_path = get_pype_path_from_db(openpype_mongo)
|
||||
if not openpype_path:
|
||||
print("*** Cannot get Pype path from database.")
|
||||
|
||||
if not os.getenv("PYPE_PATH") and openpype_path:
|
||||
os.environ["PYPE_PATH"] = openpype_path
|
||||
|
||||
|
|
|
|||
|
|
@ -145,18 +145,18 @@ def test_search_string_for_pype_version(printer):
|
|||
assert PypeVersion.version_in_str(ver_string[0])[0] == ver_string[1]
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
def test_install_live_repos(fix_bootstrap, printer):
|
||||
rf = fix_bootstrap.create_version_from_live_code()
|
||||
pype_version = fix_bootstrap.create_version_from_live_code()
|
||||
sep = os.path.sep
|
||||
expected_paths = [
|
||||
f"{rf}{sep}avalon-core",
|
||||
f"{rf}{sep}avalon-unreal-integration",
|
||||
f"{rf}{sep}maya-look-assigner",
|
||||
f"{rf}{sep}pype"
|
||||
f"{pype_version.path}{sep}repos{sep}avalon-core",
|
||||
f"{pype_version.path}{sep}repos{sep}avalon-unreal-integration",
|
||||
f"{pype_version.path}"
|
||||
]
|
||||
printer("testing zip creation")
|
||||
assert os.path.exists(rf), "zip archive was not created"
|
||||
fix_bootstrap.add_paths_from_archive(rf)
|
||||
assert os.path.exists(pype_version.path), "zip archive was not created"
|
||||
fix_bootstrap.add_paths_from_archive(pype_version.path)
|
||||
for ep in expected_paths:
|
||||
assert ep in sys.path, f"{ep} not set correctly"
|
||||
|
||||
|
|
@ -165,10 +165,9 @@ def test_install_live_repos(fix_bootstrap, printer):
|
|||
import pype # noqa: F401
|
||||
|
||||
# test if pype is imported from specific location in zip
|
||||
print(pype.__file__)
|
||||
assert "pype" in sys.modules.keys(), "Pype not imported"
|
||||
assert sys.modules["pype"].__file__ == \
|
||||
f"{rf}{sep}pype{sep}pype{sep}__init__.py"
|
||||
f"{pype_version.path}{sep}pype{sep}__init__.py"
|
||||
|
||||
|
||||
def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer):
|
||||
|
|
@ -252,7 +251,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer):
|
|||
def _create_valid_zip(path: Path, version: str):
|
||||
with ZipFile(path, "w") as zf:
|
||||
zf.writestr(
|
||||
"pype/pype/version.py", f"__version__ = '{version}'\n\n")
|
||||
"pype/version.py", f"__version__ = '{version}'\n\n")
|
||||
|
||||
def _create_invalid_dir(path: Path):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -260,7 +259,7 @@ def test_find_pype(fix_bootstrap, tmp_path_factory, monkeypatch, printer):
|
|||
fp.write("invalid")
|
||||
|
||||
def _create_valid_dir(path: Path, version: str):
|
||||
pype_path = path / "pype" / "pype"
|
||||
pype_path = path / "pype"
|
||||
version_path = pype_path / "version.py"
|
||||
pype_path.mkdir(parents=True, exist_ok=True)
|
||||
with open(version_path, "w") as fp:
|
||||
|
|
|
|||
|
|
@ -10,7 +10,3 @@ def test_validate_path_string(tmp_path):
|
|||
status2, _ = validate_path_string("booo" + str(uuid4()))
|
||||
assert status2 is False
|
||||
|
||||
# todo: change when Pype token is implemented
|
||||
status3, reason = validate_path_string(str(uuid4()))
|
||||
assert status3 is False
|
||||
assert reason == "Not implemented yet"
|
||||
|
|
|
|||
|
|
@ -34,37 +34,39 @@ function Exit-WithCode($exitcode) {
|
|||
|
||||
|
||||
function Find-Mongo {
|
||||
Write-Host ">>> " -NoNewLine -ForegroundColor Green
|
||||
Write-Host "Detecting MongoDB ... " -NoNewline
|
||||
if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) {
|
||||
if(Test-Path 'C:\Program Files\MongoDB\Server\*\bin\mongod.exe' -PathType Leaf) {
|
||||
# we have mongo server installed on standard Windows location
|
||||
# so we can inject it to the PATH. We'll use latest version available.
|
||||
$mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]}
|
||||
if(Test-Path "$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) {
|
||||
$env:PATH="$($env:PATH);$($mongoVersions[-1])\bin\"
|
||||
Write-Host "OK" -ForegroundColor Green
|
||||
Write-Host " - auto-added from [ " -NoNewline
|
||||
Write-Host "$($mongoVersions[-1])\bin\" -NoNewLine -ForegroundColor Cyan
|
||||
Write-Host " ]"
|
||||
} else {
|
||||
Write-Host "FAILED " -NoNewLine -ForegroundColor Red
|
||||
Write-Host "MongoDB not detected" -ForegroundColor Yellow
|
||||
Write-Host "Tried to find it on standard location " -NoNewline -ForegroundColor Gray
|
||||
Write-Host " [ " -NoNewline -ForegroundColor Cyan
|
||||
Write-Host "$($mongoVersions[-1])\bin\" -NoNewline -ForegroundColor White
|
||||
Write-Host " ] " -NonNewLine -ForegroundColor Cyan
|
||||
Write-Host "but failed." -ForegroundColor Gray
|
||||
Exit-WithCode 1
|
||||
}
|
||||
$defaultPath = "C:\Program Files\MongoDB\Server"
|
||||
Write-Host ">>> " -NoNewLine -ForegroundColor Green
|
||||
Write-Host "Detecting MongoDB ... " -NoNewline
|
||||
if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) {
|
||||
if(Test-Path "$($defaultPath)\*\bin\mongod.exe" -PathType Leaf) {
|
||||
# we have mongo server installed on standard Windows location
|
||||
# so we can inject it to the PATH. We'll use latest version available.
|
||||
$mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]}
|
||||
if(Test-Path "$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) {
|
||||
$env:PATH = "$($env:PATH);$($mongoVersions[-1])\bin\"
|
||||
Write-Host "OK" -ForegroundColor Green
|
||||
Write-Host " - auto-added from [ " -NoNewline
|
||||
Write-Host "$($mongoVersions[-1])\bin\mongod.exe" -NoNewLine -ForegroundColor Cyan
|
||||
Write-Host " ]"
|
||||
return "$($mongoVersions[-1])\bin\mongod.exe"
|
||||
} else {
|
||||
Write-Host "FAILED " -NoNewLine -ForegroundColor Red
|
||||
Write-Host "MongoDB not detected" -ForegroundColor Yellow
|
||||
Write-Host "Tried to find it on standard location " -NoNewline -ForegroundColor Gray
|
||||
Write-Host " [ " -NoNewline -ForegroundColor Cyan
|
||||
Write-Host "$($mongoVersions[-1])\bin\mongod.exe" -NoNewline -ForegroundColor White
|
||||
Write-Host " ] " -NoNewLine -ForegroundColor Cyan
|
||||
Write-Host "but failed." -ForegroundColor Gray
|
||||
Exit-WithCode 1
|
||||
}
|
||||
} else {
|
||||
Write-Host "FAILED " -NoNewLine -ForegroundColor Red
|
||||
Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow
|
||||
Exit-WithCode 1
|
||||
Write-Host "FAILED " -NoNewLine -ForegroundColor Red
|
||||
Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow
|
||||
Exit-WithCode 1
|
||||
}
|
||||
|
||||
} else {
|
||||
Write-Host "OK" -ForegroundColor Green
|
||||
Write-Host "OK" -ForegroundColor Green
|
||||
return Get-Command "mongod" -ErrorAction SilentlyContinue
|
||||
}
|
||||
<#
|
||||
.SYNOPSIS
|
||||
|
|
@ -85,6 +87,7 @@ $port = 2707
|
|||
# path to database
|
||||
$dbpath = (Get-Item $openpype_root).parent.FullName + "\mongo_db_data"
|
||||
|
||||
Find-Mongo
|
||||
$mongo = Get-Command "mongod" | Select-Object -ExpandProperty Definition
|
||||
Start-Process -FilePath $mongo "--dbpath $($dbpath) --port $($port)"
|
||||
$mongoPath = Find-Mongo
|
||||
Start-Process -FilePath $mongopath "--dbpath $($dbpath) --port $($port)" -PassThru
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue