proper python environments

pype/lib/__init__.py

@@ -1,5 +1,16 @@
 # -*- coding: utf-8 -*-
+# flake8: noqa E402
 """Pype module API."""
+# add vendor to sys path based on Python version
+import sys
+import os
+import site
+
+# add Python version specific vendor folder
+site.addsitedir(
+    os.path.join(
+        os.getenv("PYPE_ROOT", ""),
+        "vendor", "python", "python_{}".format(sys.version[0])))
 
 from .terminal import Terminal
 from .execute import (

pype/vendor/python/common/README.md

@@ -0,0 +1,2 @@
+### Info
+Here are modules that can run under both Python 2 and Python 3 hosts.

pype/vendor/python/python_2/README.md

@@ -0,0 +1,3 @@
+## Info
+
+Only **Python 2** specific modules are here.

pype/vendor/python/python_3/README.md

@@ -0,0 +1,2 @@
+## Info
+Only **Python 3** modules are here.

setup.py

@@ -51,7 +51,6 @@ include_files = [
     "pype",
     "repos",
     "schema",
-    "setup",
     "vendor",
     "LICENSE",
     "README.md",

start.py

@@ -46,7 +46,7 @@ So, bootstrapping Pype looks like this::
                  YES             NO
                   |              |
                   |       +------v--------------+
-                  |       | Fire up Igniter GUI |<---------\
+                  |       | Fire up Igniter GUI |<---------+
                   |       | and ask User        |          |
                   |       +---------------------+          |
                   |                                        |
@@ -72,9 +72,9 @@ So, bootstrapping Pype looks like this::
               |      | to user data dir.               |   |
               |      +--------------|------------------+   |
               |           .-- Is Pype found? --.           |
-              |         YES                    NO ---------/
+              |         YES                    NO ---------+
               |          |
-              |<--------/
+              |<---------+
               |
 +-------------v------------+
 |         Run Pype         |
@@ -97,6 +97,7 @@ import re
 import sys
 import traceback
 import subprocess
+import site
 from pathlib import Path
 
 # add dependencies folder to sys.pat for frozen code
@@ -112,7 +113,7 @@ if getattr(sys, 'frozen', False):
 
 from igniter import BootstrapRepos  # noqa: E402
 from igniter.tools import load_environments  # noqa: E402
-
+from igniter.bootstrap_repos import PypeVersion  # noqa: E402
 
 bootstrap = BootstrapRepos()
 silent_commands = ["run", "igniter"]
@@ -214,7 +215,6 @@ def _process_arguments() -> tuple:
         tuple: Return tuple with specific version to use (if any) and flag
             to prioritize staging (if set)
     """
-
     # check for `--use-version=3.0.0` argument and `--use-staging`
     use_version = None
     use_staging = False
@@ -268,10 +268,35 @@ def _determine_mongodb() -> str:
     return pype_mongo
 
 
+def _initialize_environment(pype_version: PypeVersion) -> None:
+    version_path = pype_version.path
+    os.environ["PYPE_VERSION"] = pype_version.version
+    # inject version to Python environment (sys.path, ...)
+    print(">>> Injecting Pype version to running environment  ...")
+    bootstrap.add_paths_from_directory(version_path)
+
+    # add venv 'site-packages' to PYTHONPATH
+    python_path = os.getenv("PYTHONPATH", "")
+    split_paths = python_path.split(os.pathsep)
+    # add pype tools
+    split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"))
+    # add common pype vendor
+    # (common for multiple Python interpreter versions)
+    split_paths.append(os.path.join(
+        os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common"))
+    os.environ["PYTHONPATH"] = os.pathsep.join(split_paths)
+
+    # set PYPE_ROOT to point to currently used Pype version.
+    os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix())
+
+
 def _find_frozen_pype(use_version: str = None,
                       use_staging: bool = False) -> Path:
     """Find Pype to run from frozen code.
 
+    This will process and modify environment variables:
+    ``PYTHONPATH``, ``PYPE_VERSION``, ``PYPE_ROOT``
+
     Args:
         use_version (str, optional): Try to use specified version.
         use_staging (bool, optional): Prefer *staging* flavor over production.
@@ -284,7 +309,6 @@ def _find_frozen_pype(use_version: str = None,
             (if requested).
 
     """
-
     pype_version = None
     pype_versions = bootstrap.find_pype(include_zips=True,
                                         staging=use_staging)
@@ -299,15 +323,26 @@ def _find_frozen_pype(use_version: str = None,
         pype_versions = bootstrap.find_pype()
 
     if not pype_versions:
-        raise RuntimeError("No Pype versions found.")
+        # no Pype versions found anyway, lets use then the one
+        # shipped with frozen Pype
+        version_path = _bootstrap_from_code(use_version)
+        pype_version = PypeVersion(
+            version=BootstrapRepos.get_version(version_path),
+            path=version_path)
+        _initialize_environment(pype_version)
+        return version_path
 
     # get path of version specified in `--use-version`
     version_path = BootstrapRepos.get_version_path_from_list(
         use_version, pype_versions)
+
     if not version_path:
         if use_version is not None:
-            print(("!!! Specified version was not found, using "
+            if not pype_version:
-                   "latest available"))
+                ...
+            else:
+                print(("!!! Specified version was not found, using "
+                       "latest available"))
         # specified version was not found so use latest detected.
         version_path = pype_version.path
         print(f">>> Using version [ {pype_version} ]")
@@ -331,20 +366,76 @@ def _find_frozen_pype(use_version: str = None,
     if pype_version.path.is_file():
         print(">>> Extracting zip file ...")
         version_path = bootstrap.extract_pype(pype_version)
+        pype_version.path = version_path
 
-    os.environ["PYPE_VERSION"] = pype_version.version
+    _initialize_environment(pype_version)
-    # inject version to Python environment (sys.path, ...)
-    print(">>> Injecting Pype version to running environment  ...")
-    bootstrap.add_paths_from_directory(version_path)
-
-    # set PYPE_ROOT to point to currently used Pype version.
-    os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix())
-
     return version_path
 
 
+def _bootstrap_from_code(use_version):
+    """Bootstrap live code (or the one coming with frozen Pype.
+
+    Args:
+        use_version: (str): specific version to use.
+
+    Returns:
+        Path: path to sourced version.
+
+    """
+    # run through repos and add them to `sys.path` and `PYTHONPATH`
+    # set root
+    if getattr(sys, 'frozen', False):
+        pype_root = os.path.normpath(
+            os.path.dirname(sys.executable))
+        local_version = bootstrap.get_version(Path(pype_root))
+    else:
+        pype_root = os.path.normpath(
+            os.path.dirname(os.path.realpath(__file__)))
+        # get current version of Pype
+        local_version = bootstrap.get_local_live_version()
+
+    os.environ["PYPE_VERSION"] = local_version
+    if use_version and use_version != local_version:
+        pype_versions = bootstrap.find_pype(include_zips=True)
+        version_path = BootstrapRepos.get_version_path_from_list(
+            use_version, pype_versions)
+        if version_path:
+            # use specified
+            bootstrap.add_paths_from_directory(version_path)
+            os.environ["PYPE_VERSION"] = use_version
+    else:
+        version_path = pype_root
+    os.environ["PYPE_ROOT"] = pype_root
+    repos = os.listdir(os.path.join(pype_root, "repos"))
+    repos = [os.path.join(pype_root, "repos", repo) for repo in repos]
+    # add self to python paths
+    repos.insert(0, pype_root)
+    for repo in repos:
+        sys.path.append(repo)
+
+    # add venv 'site-packages' to PYTHONPATH
+    python_path = os.getenv("PYTHONPATH", "")
+    split_paths = python_path.split(os.pathsep)
+    split_paths += repos
+    # add pype tools
+    split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"))
+    # last one should be venv site-packages
+    # this is slightly convoluted as we can get here from frozen code too
+    # in case when we are running without any version installed.
+    if not getattr(sys, 'frozen', False):
+        split_paths.append(site.getsitepackages()[-1])
+    # add common pype vendor
+    # (common for multiple Python interpreter versions)
+    split_paths.append(os.path.join(
+        os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common"))
+    os.environ["PYTHONPATH"] = os.pathsep.join(split_paths)
+
+    return Path(version_path)
+
+
 def boot():
     """Bootstrap Pype."""
+    version_path = None
 
     # ------------------------------------------------------------------------
     # Play animation
@@ -378,7 +469,12 @@ def boot():
     # ------------------------------------------------------------------------
     # Load environments from database
     # ------------------------------------------------------------------------
-
+    # set PYPE_ROOT to running location until proper version can be
+    # determined.
+    if getattr(sys, 'frozen', False):
+        os.environ["PYPE_ROOT"] = os.path.dirname(sys.executable)
+    else:
+        os.environ["PYPE_ROOT"] = os.path.dirname(__file__)
     set_environments()
 
     # ------------------------------------------------------------------------
@@ -394,37 +490,7 @@ def boot():
             print(f"!!! {e}")
             sys.exit(1)
     else:
-        # run through repos and add them to sys.path and PYTHONPATH
+        version_path = _bootstrap_from_code(use_version)
-        # set root
-        pype_root = os.path.normpath(
-            os.path.dirname(os.path.realpath(__file__)))
-        # get current version of Pype
-        local_version = bootstrap.get_local_live_version()
-        os.environ["PYPE_VERSION"] = local_version
-        if use_version and use_version != local_version:
-            pype_versions = bootstrap.find_pype(include_zips=True)
-            version_path = BootstrapRepos.get_version_path_from_list(
-                use_version, pype_versions)
-            if version_path:
-                # use specified
-                bootstrap.add_paths_from_directory(version_path)
-                os.environ["PYPE_VERSION"] = use_version
-        else:
-            version_path = pype_root
-        os.environ["PYPE_ROOT"] = pype_root
-        repos = os.listdir(os.path.join(pype_root, "repos"))
-        repos = [os.path.join(pype_root, "repos", repo) for repo in repos]
-        # add self to python paths
-        repos.insert(0, pype_root)
-        for repo in repos:
-            sys.path.append(repo)
-
-        pythonpath = os.getenv("PYTHONPATH", "")
-        paths = pythonpath.split(os.pathsep)
-        paths += repos
-        os.environ["PYTHONPATH"] = os.pathsep.join(paths)
-
-        # TODO: add venv when running from source
 
     # set this to point either to `python` from venv in case of live code
     # or to `pype` or `pype_console` in case of frozen code
@@ -433,14 +499,16 @@ def boot():
     # TODO: DEPRECATE remove when `pype-config` dissolves into Pype for good.
     # PYPE_MODULE_ROOT should be changed to PYPE_REPOS_ROOT
     # This needs to replace environment building in hosts
+
     # .-=-----------------------=-=. ^ .=-=--------------------------=-.
     os.environ["PYPE_MODULE_ROOT"] = os.environ["PYPE_ROOT"]
+    # -=------------------------=-=. v .=-=--------------------------=-.
 
-    # TODO: add pype tools and vendor to environment
+    if getattr(sys, 'frozen', False):
-    os.environ["PYTHONPATH"] = os.pathsep.join(
+        os.environ["PYPE_REPOS_ROOT"] = os.environ["PYPE_ROOT"]
-        [os.environ["PYTHONPATH"],
+    else:
-        os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"),
+        os.environ["PYPE_REPOS_ROOT"] = os.path.join(
-        os.path.join(os.environ["PYPE_ROOT"], "pype", "vendor")])
+            os.environ["PYPE_ROOT"], "repos")
 
     # delete Pype module from cache so it is used from specific version
     try:
@@ -457,6 +525,7 @@ def boot():
     print(">>> loading environments ...")
     set_modules_environments()
 
+    assert version_path, "Version path not defined."
     info = get_info()
     info.insert(0, f">>> Using Pype from [ {version_path} ]")
 
@@ -485,42 +554,42 @@ def get_info() -> list:
 
     components = get_default_components()
 
-    infos = []
+    inf = []
     if not getattr(sys, 'frozen', False):
-        infos.append(("Pype variant", "staging"))
+        inf.append(("Pype variant", "staging"))
     else:
-        infos.append(("Pype variant", "production"))
+        inf.append(("Pype variant", "production"))
-    infos.append(("Running pype from", os.environ.get('PYPE_ROOT')))
+    inf.append(("Running pype from", os.environ.get('PYPE_ROOT')))
-    infos.append(("Using mongodb", components["host"]))
+    inf.append(("Using mongodb", components["host"]))
 
     if os.environ.get("FTRACK_SERVER"):
-        infos.append(("Using FTrack at",
+        inf.append(("Using FTrack at",
-                      os.environ.get("FTRACK_SERVER")))
+                    os.environ.get("FTRACK_SERVER")))
 
     if os.environ.get('DEADLINE_REST_URL'):
-        infos.append(("Using Deadline webservice at",
+        inf.append(("Using Deadline webservice at",
-                      os.environ.get("DEADLINE_REST_URL")))
+                    os.environ.get("DEADLINE_REST_URL")))
 
     if os.environ.get('MUSTER_REST_URL'):
-        infos.append(("Using Muster at",
+        inf.append(("Using Muster at",
-                      os.environ.get("MUSTER_REST_URL")))
+                    os.environ.get("MUSTER_REST_URL")))
 
     # Reinitialize
     PypeLogger.initialize()
 
     log_components = PypeLogger.log_mongo_url_components
     if log_components["host"]:
-        infos.append(("Logging to MongoDB", log_components["host"]))
+        inf.append(("Logging to MongoDB", log_components["host"]))
-        infos.append(("  - port", log_components["port"] or "<N/A>"))
+        inf.append(("  - port", log_components["port"] or "<N/A>"))
-        infos.append(("  - database", PypeLogger.log_database_name))
+        inf.append(("  - database", PypeLogger.log_database_name))
-        infos.append(("  - collection", PypeLogger.log_collection_name))
+        inf.append(("  - collection", PypeLogger.log_collection_name))
-        infos.append(("  - user", log_components["username"] or "<N/A>"))
+        inf.append(("  - user", log_components["username"] or "<N/A>"))
         if log_components["auth_db"]:
-            infos.append(("  - auth source", log_components["auth_db"]))
+            inf.append(("  - auth source", log_components["auth_db"]))
 
-    maximum = max(len(i[0]) for i in infos)
+    maximum = max(len(i[0]) for i in inf)
     formatted = []
-    for info in infos:
+    for info in inf:
         padding = (maximum - len(info[0])) + 1
         formatted.append(
             "... {}:{}[ {} ]".format(info[0], " " * padding, info[1]))

tools/build.ps1

@@ -175,6 +175,14 @@ Write-Host "OK" -ForegroundColor green
 Write-Host ">>> " -NoNewline -ForegroundColor green
 Write-Host "Building Pype ..."
 $out = & python setup.py build 2>&1
+if ($LASTEXITCODE -ne 0)
+{
+    Write-Host "!!! " -NoNewLine -ForegroundColor Red
+    Write-Host "Build failed. Check the log: " -NoNewline
+    Write-Host ".\build\build.log" -ForegroundColor Yellow
+    deactivate
+    Exit-WithCode $LASTEXITCODE
+}
 
 Set-Content -Path "$($pype_root)\build\build.log" -Value $out
 & python -B "$($pype_root)\tools\build_dependencies.py"

tools/build_dependencies.py

@@ -88,7 +88,10 @@ elif sys.platform == "win32":
 build_dir = Path(os.path.dirname(__file__)).parent / "build" / build_dir
 
 _print(f"Using build at {build_dir}", 2)
-assert build_dir.exists(), "Build directory doesn't exist"
+if not build_dir.exists():
+    _print("Build directory doesn't exist", 1)
+    _print("Probably freezing of code failed. Check ./build/build.log", 3)
+    sys.exit(1)
 
 deps_dir = build_dir / "dependencies"
 

tools/create_env.ps1

@@ -9,7 +9,7 @@
 
 .EXAMPLE
 
-PS> .\build.ps1
+PS> .\create_env.ps1
 
 #>
 

tools/create_zip.ps1

@@ -1,6 +1,6 @@
 <#
 .SYNOPSIS
-  Helper script create virtual env.
+  Helper script create distributable Pype zip.
 
 .DESCRIPTION
   This script will detect Python installation, create venv and install
@@ -9,7 +9,7 @@
 
 .EXAMPLE
 
-PS> .\build.ps1
+PS> .\create_zip.ps1
 
 #>
 

tools/run_mongo.ps1

@@ -1,7 +1,91 @@
+<#
+.SYNOPSIS
+  Helper script to run mongodb.
+
+.DESCRIPTION
+  This script will detect mongodb, add it to the PATH and launch it on specified port and db location.
+
+.EXAMPLE
+
+PS> .\run_mongo.ps1
+
+#>
+
+$art = @"
+
+
+        ____________
+       /\      ___  \
+       \ \     \/_\  \
+        \ \     _____/ ______   ___ ___ ___
+         \ \    \___/ /\     \  \  \\  \\  \
+          \ \____\    \ \_____\  \__\\__\\__\
+           \/____/     \/_____/  . PYPE Club .
+
+"@
+
+Write-Host $art -ForegroundColor DarkGreen
+
+function Exit-WithCode($exitcode) {
+   # Only exit this host process if it's a child of another PowerShell parent process...
+   $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId
+   $parentProcName = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$parentPID" | Select-Object -Property Name).Name
+   if ('powershell.exe' -eq $parentProcName) { $host.SetShouldExit($exitcode) }
+
+   exit $exitcode
+}
+
+
+function Find-Mongo {
+  Write-Host ">>> " -NoNewLine -ForegroundColor Green
+  Write-Host "Detecting MongoDB ... " -NoNewline
+  if (-not (Get-Command "mongod" -ErrorAction SilentlyContinue)) {
+    if(Test-Path 'C:\Program Files\MongoDB\Server\*\bin\mongod.exe' -PathType Leaf) {
+      # we have mongo server installed on standard Windows location
+      # so we can inject it to the PATH. We'll use latest version available.
+      $mongoVersions = Get-ChildItem -Directory 'C:\Program Files\MongoDB\Server' | Sort-Object -Property {$_.Name -as [int]}
+      if(Test-Path "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\mongod.exe" -PathType Leaf) {
+        $env:PATH="$($env:PATH);C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\"
+        Write-Host "OK" -ForegroundColor Green
+        Write-Host "  - auto-added from [ " -NoNewline
+        Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewLine -ForegroundColor Cyan
+        Write-Host " ]"
+      } else {
+          Write-Host "FAILED " -NoNewLine -ForegroundColor Red
+          Write-Host "MongoDB not detected" -ForegroundColor Yellow
+          Write-Host "Tried to find it on standard location [ " -NoNewline -ForegroundColor Gray
+          Write-Host "C:\Program Files\MongoDB\Server\$($mongoVersions[-1])\bin\" -NoNewline -ForegroundColor White
+          Write-Host " ] but failed." -ForegroundColor Gray
+          Exit-WithCode 1
+      }
+    } else {
+      Write-Host "FAILED " -NoNewLine -ForegroundColor Red
+      Write-Host "MongoDB not detected in PATH" -ForegroundColor Yellow
+      Exit-WithCode 1
+    }
+
+  } else {
+    Write-Host "OK" -ForegroundColor Green
+  }
+  <#
+  .SYNOPSIS
+  Function to detect mongod in path.
+  .DESCRIPTION
+  This will test presence of mongod in PATH. If it's not there, it will try
+  to find it in default install location. It support different mongo versions
+  (using latest if found). When mongod is found, path to it is added to PATH
+  #>
+}
+
 $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
 $pype_root = (Get-Item $script_dir).parent.FullName
 
-& "$($pype_root)\venv\Scripts\Activate.ps1"
+# mongodb port
+$port = 2707
 
-python "$($pype_root)\start.py" mongodb
+# path to database
-deactivate
+$dbpath = (Get-Item $pype_root).parent.FullName + "\mongo_db_data"
+
+Find-Mongo
+$mongo = Get-Command "mongod" | Select-Object -ExpandProperty Definition
+Start-Process -FilePath $mongo "--dbpath $($dbpath) --port $($port)"

tools/run_settings.ps1

@@ -1,3 +1,16 @@
+<#
+.SYNOPSIS
+  Helper script to Pype Settings UI
+
+.DESCRIPTION
+  This script will run Pype and open Settings UI.
+
+.EXAMPLE
+
+PS> .\run_settings.ps1
+
+#>
+
 $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
 $pype_root = (Get-Item $script_dir).parent.FullName
 

tools/run_tests.ps1

@@ -1,3 +1,18 @@
+<#
+.SYNOPSIS
+  Helper script to build Pype.
+
+.DESCRIPTION
+  This script will detect Python installation, create venv and install
+  all necessary packages from `requirements.txt` needed by Pype to be
+  included during application freeze on Windows.
+
+.EXAMPLE
+
+PS> .\run_test.ps1
+
+#>
+
 function Exit-WithCode($exitcode) {
    # Only exit this host process if it's a child of another PowerShell parent process...
    $parentPID = (Get-CimInstance -ClassName Win32_Process -Filter "ProcessId=$PID" | Select-Object -Property ParentProcessId).ParentProcessId

tools/run_tray.ps1

@@ -1,3 +1,15 @@
+<#
+.SYNOPSIS
+  Helper script Pype Tray.
+
+.DESCRIPTION
+
+
+.EXAMPLE
+
+PS> .\run_tray.ps1
+
+#>
 $script_dir = Split-Path -Path $MyInvocation.MyCommand.Definition -Parent
 $pype_root = (Get-Item $script_dir).parent.FullName