fix create_env, add icon, igniter as subprocess, refactored start

igniter/__init__.py

@@ -9,10 +9,12 @@ from .bootstrap_repos import BootstrapRepos
 
 
 def run():
-    app = QtWidgets.QApplication(sys.argv)
+    """Show Igniter dialog."""
+    # app = QtWidgets.QApplication(sys.argv)
     d = InstallDialog()
-    d.show()
+    d.exec_()
-    sys.exit(app.exec_())
+    #d.show()
+    #sys.exit(app.exec_())
 
 
 __all__ = [

igniter/bootstrap_repos.py

@@ -140,6 +140,22 @@ class PypeVersion:
 
         return False
 
+    def is_staging(self) -> bool:
+        """Test if current version is staging one."""
+        return self.variant == "staging"
+
+    def get_main_version(self) -> str:
+        """Return main version component.
+
+        This returns x.x.x part of version from possibly more complex one
+        like x.x.x-foo-bar.
+
+        Returns:
+            str: main version component
+
+        """
+        return "{}.{}.{}".format(self.major, self.minor, self.subversion)
+
     @staticmethod
     def version_in_str(string: str) -> Tuple:
         """Find Pype version in given string.
@@ -227,7 +243,7 @@ class BootstrapRepos:
                 return v.path
 
     @staticmethod
-    def get_local_version() -> str:
+    def get_local_live_version() -> str:
         """Get version of local Pype."""
         return __version__
 
@@ -273,7 +289,7 @@ class BootstrapRepos:
         # version and use it as a source. Otherwise repo_dir is user
         # entered location.
         if not repo_dir:
-            version = self.get_local_version()
+            version = self.get_local_live_version()
             repo_dir = self.live_repo_dir
         else:
             version = self.get_version(repo_dir)
@@ -516,11 +532,7 @@ class BootstrapRepos:
         for file in dir_to_search.iterdir():
 
             # if file, strip extension, in case of dir not.
-            if file.is_dir():
+            name = file.name if file.is_dir() else file.stem
-                name = file.name
-            else:
-                name = file.stem
-
             result = PypeVersion.version_in_str(name)
 
             if result[0]:
@@ -540,7 +552,10 @@ class BootstrapRepos:
                         self._log.error(
                             f"cannot determine version from {file}")
                         continue
-                    if version_check != detected_version:
+
+                    version_main = version_check.get_main_version()
+                    detected_main = detected_version.get_main_version()
+                    if version_main != detected_main:
                         self._log.error(
                             (f"dir version ({detected_version}) and "
                              f"its content version ({version_check}) "
@@ -568,7 +583,10 @@ class BootstrapRepos:
                                 version_check = PypeVersion(
                                     version=zip_version["__version__"])
 
-                                if version_check != detected_version:
+                                version_main = version_check.get_main_version()
+                                detected_main = detected_version.get_main_version()
+
+                                if version_main != detected_main:
                                     self._log.error(
                                         (f"zip version ({detected_version}) "
                                          f"and its content version "
@@ -769,3 +787,110 @@ class BootstrapRepos:
             zip_ref.extractall(destination)
 
         self._print(f"Installed as {version.path.stem}")
+
+    def install_version(self, pype_version: PypeVersion, force: bool = False):
+        """Install Pype version to user data directory.
+
+        Args:
+            pype_version (PypeVersion): Pype version to install.
+            force (bool, optional): Force overwrite existing version.
+
+        Returns:
+            Path: Path to installed Pype.
+
+        Raises:
+            PypeVersionExists: If not forced and this version already exist
+                in user data directory.
+            PypeVersionInvalid: If version to install is invalid.
+            PypeVersionIOError: If copying or zipping fail.
+
+        """
+
+        # test if version is located (in user data dir)
+        is_inside = False
+        try:
+            is_inside = pype_version.path.resolve().relative_to(
+                self.data_dir)
+        except ValueError:
+            # if relative path cannot be calculated, Pype version is not
+            # inside user data dir
+            pass
+
+        if is_inside:
+            raise PypeVersionExists("Pype already inside user data dir")
+
+        # determine destination directory name
+        # for zip file strip suffix
+        destination = self.data_dir / pype_version.path.stem
+
+        # test if destination file already exist, if so lets delete it.
+        # we consider path on location as authoritative place.
+        if destination.exists() and force:
+            try:
+                destination.unlink()
+            except OSError:
+                self._log.error(
+                    f"cannot remove already existing {destination}",
+                    exc_info=True)
+                return None
+        else:
+            raise PypeVersionExists(f"{destination} already exist.")
+
+        # create destination parent directories even if they don't exist.
+        if not destination.exists():
+            destination.mkdir(parents=True)
+
+        # version is directory
+        if pype_version.path.is_dir():
+            # create zip inside temporary directory.
+            self._log.info("Creating zip from directory ...")
+            with tempfile.TemporaryDirectory() as temp_dir:
+                temp_zip = \
+                    Path(temp_dir) / f"pype-v{pype_version}.zip"
+                self._log.info(f"creating zip: {temp_zip}")
+
+                self._create_pype_zip(temp_zip, pype_version.path)
+                if not os.path.exists(temp_zip):
+                    self._log.error("make archive failed.")
+                    raise PypeVersionIOError("Zip creation failed.")
+
+                # set zip as version source
+                pype_version.path = temp_zip
+
+        elif pype_version.path.is_file():
+            # check if file is zip (by extension)
+            if pype_version.path.suffix.lower() != ".zip":
+                raise PypeVersionInvalid("Invalid file format")
+
+        try:
+            # copy file to destination
+            self._log.info("Copying zip to destination ...")
+            copyfile(pype_version.path.as_posix(), destination.as_posix())
+        except OSError as e:
+            self._log.error(
+                "cannot copy version to user data directory",
+                exc_info=True)
+            raise PypeVersionIOError(
+                "can't copy version to destination") from e
+
+        # extract zip there
+        self._log.info("extracting zip to destination ...")
+        with ZipFile(pype_version.path, "r") as zip:
+            zip.extractall(destination)
+
+        return destination
+
+
+class PypeVersionExists(Exception):
+    """Exception for handling existing Pype version."""
+    pass
+
+
+class PypeVersionInvalid(Exception):
+    """Exception for handling invalid Pype version."""
+    pass
+
+
+class PypeVersionIOError(Exception):
+    """Exception for handling IO errors in Pype version."""
+    pass

igniter/install_thread.py

@@ -48,7 +48,7 @@ class InstallThread(QThread):
         # find local version of Pype
         bs = BootstrapRepos(
             progress_callback=self.set_progress, message=self.message)
-        local_version = bs.get_local_version()
+        local_version = bs.get_local_live_version()
 
         # if user did entered nothing, we install Pype from local version.
         # zip content of `repos`, copy it to user data dir and append
@@ -93,8 +93,6 @@ class InstallThread(QThread):
                         f"currently running {local_version}"
                     ), False)
                     self.message.emit("Skipping Pype install ...", False)
-                    if detected[-1].path.suffix.lower() == ".zip":
-                        bs.extract_pype(detected[-1])
                     return
 
                 self.message.emit((
@@ -150,6 +148,9 @@ class InstallThread(QThread):
                 bs.registry.set_secure_item("pypeMongo", self._mongo)
                 os.environ["PYPE_MONGO"] = self._mongo
 
+            if os.getenv("PYPE_PATH") == self._path:
+                ...
+
             self.message.emit(f"processing {self._path}", True)
             repo_file = bs.process_entered_location(self._path)
 

setup.py

@@ -2,12 +2,16 @@
 """Setup info for building Pype 3.0."""
 import os
 import sys
+from pathlib import Path
 
 from cx_Freeze import setup, Executable
 from sphinx.setup_command import BuildDoc
 
 version = {}
-with open(os.path.join("pype", "version.py")) as fp:
+
+pype_root = Path(os.path.dirname(__file__))
+
+with open(pype_root / "pype" / "version.py") as fp:
     exec(fp.read(), version)
 __version__ = version["__version__"]
 
@@ -65,10 +69,13 @@ build_options = dict(
     include_files=include_files
 )
 
+icon_path = pype_root / "igniter" / "pype.ico"
 
 executables = [
-    Executable("start.py", base=None, target_name="pype_console"),
+    Executable("start.py", base=None,
-    Executable("start.py", base=base, target_name="pype")
+               target_name="pype_console", icon=icon_path.as_posix()),
+    Executable("start.py", base=base,
+               target_name="pype", icon=icon_path.as_posix())
 ]
 
 setup(
@@ -82,8 +89,8 @@ setup(
             "project": "Pype",
             "version": __version__,
             "release": __version__,
-            "source_dir": "./docs/source",
+            "source_dir": (pype_root / "docs" / "source").as_posix(),
-            "build_dir": "./docs/build"
+            "build_dir": (pype_root / "docs" / "build").as_posix()
         }
     },
     executables=executables

start.py

@@ -84,6 +84,10 @@ So, bootstrapping Pype looks like this::
 Todo:
     Move or remove bootstrapping environments out of the code.
 
+Attributes:
+    silent_commands (list): list of commands for which we won't print Pype
+        logo and info header.
+
 .. _MongoDB:
    https://www.mongodb.com/
 
@@ -92,6 +96,7 @@ import os
 import re
 import sys
 import traceback
+import subprocess
 
 import acre
 
@@ -99,6 +104,9 @@ from igniter import BootstrapRepos
 from igniter.tools import load_environments
 
 
+silent_commands = ["run", "igniter"]
+
+
 def set_environments() -> None:
     """Set loaded environments.
 
@@ -106,14 +114,49 @@ def set_environments() -> None:
         better handling of environments
 
     """
-    env = load_environments(["global"])
+    env = {}
+    try:
+        env = load_environments(["global"])
+    except OSError as e:
+        print(f"!!! {e}")
+        exit()
+
     env = acre.merge(env, dict(os.environ))
     os.environ.clear()
     os.environ.update(env)
 
 
+def run(arguments: list, env: dict = None) -> int:
+    """Use correct executable to run stuff.
+
+    This passing arguments to correct Pype executable. If Pype is run from
+    live sources, executable will be `python` in virtual environment.
+    If running from frozen code, executable will be `pype`. Its equivalent in
+    live code is `python start.py`.
+
+    Args:
+        arguments (list): Argument list to pass Pype.
+        env (dict, optional): Dictionary containing environment.
+
+    Returns:
+        int: Process return code.
+
+    """
+    if getattr(sys, 'frozen', False):
+        interpreter = [sys.executable]
+    else:
+        interpreter = [sys.executable, __file__]
+
+    interpreter.extend(arguments)
+
+    p = subprocess.Popen(interpreter, env=env)
+    p.wait()
+    print(f">>> done [{p.returncode}]")
+    return p.returncode
+
+
 def set_modules_environments():
-    """Set global environments for pype's modules.
+    """Set global environments for pype modules.
 
     This requires to have pype in `sys.path`.
     """
@@ -129,10 +172,9 @@ def set_modules_environments():
     if publish_plugin_dirs:
         publish_paths_str = os.environ.get("PYBLISHPLUGINPATH") or ""
         publish_paths = publish_paths_str.split(os.pathsep)
-        _publish_paths = set()
+        _publish_paths = {
-        for path in publish_paths:
+            os.path.normpath(path) for path in publish_paths if path
-            if path:
+        }
-                _publish_paths.add(os.path.normpath(path))
         for path in publish_plugin_dirs:
             _publish_paths.add(os.path.normpath(path))
         module_envs["PYBLISHPLUGINPATH"] = os.pathsep.join(_publish_paths)
@@ -149,61 +191,133 @@ def boot():
     """Bootstrap Pype."""
 
     from pype.lib.terminal_splash import play_animation
-    play_animation()
-
-    # find pype versions
     bootstrap = BootstrapRepos()
-    pype_versions = bootstrap.find_pype()
 
-    # check for `--use-version=3.0.0` argument.
+    # ------------------------------------------------------------------------
+    # Process arguments
+    # ------------------------------------------------------------------------
+
+    # don't play for silenced commands
+    if all(item not in sys.argv for item in silent_commands):
+        play_animation()
+
+    # check for `--use-version=3.0.0` argument and `--use-staging`
     use_version = None
-
+    use_staging = False
     for arg in sys.argv:
         m = re.search(r"--use-version=(?P<version>\d+\.\d+\.\d*.+?)", arg)
         if m and m.group('version'):
             use_version = m.group('version')
             sys.argv.remove(arg)
             break
+    if "--use-staging" in sys.argv:
+        use_staging = True
+        sys.argv.remove("--use-staging")
 
+    # handle igniter
+    # this is helper to run igniter before anything else
+    if "igniter" in sys.argv:
+        import igniter
+        igniter.run()
+        return
+
+    # ------------------------------------------------------------------------
+    # Determine mongodb connection
+    # ------------------------------------------------------------------------
+
+    # try env variable
     if not os.getenv("PYPE_MONGO"):
+        # try system keyring
+        pype_mongo = ""
         try:
             pype_mongo = bootstrap.registry.get_secure_item("pypeMongo")
         except ValueError:
             print("*** No DB connection string specified.")
             print("--- launching setup UI ...")
-            import igniter
+            run(["igniter"])
-            igniter.run()
+            try:
-            return
+                pype_mongo = bootstrap.registry.get_secure_item("pypeMongo")
-        else:
+            except ValueError:
+                print("!!! Still no DB connection string.")
+                exit()
+        finally:
             os.environ["PYPE_MONGO"] = pype_mongo
 
+    # ------------------------------------------------------------------------
+    # Load environments from database
+    # ------------------------------------------------------------------------
+
     set_environments()
+
+    # ------------------------------------------------------------------------
+    # Find Pype versions
+    # ------------------------------------------------------------------------
+
+    pype_versions = bootstrap.find_pype(include_zips=True)
+    pype_version = pype_versions[-1]
+
     if getattr(sys, 'frozen', False):
         if not pype_versions:
-            import igniter
+            print('*** No Pype versions found.')
-            igniter.run()
+            print("--- launching setup UI ...")
+            run(["igniter"])
+            pype_versions = bootstrap.find_pype()
+        if not pype_versions:
+            print('!!! Still no Pype versions found.')
+            return
 
+        # find only staging versions
+        if use_staging:
+            staging_versions = [v for v in pype_versions if v.is_staging()]
+            if not staging_versions:
+                print("!!! No staging versions detected.")
+                return
+            staging_versions.sort()
+            # get latest
+            pype_version = staging_versions[-1]
+
+        # get path of version specified in `--use-version`
         version_path = BootstrapRepos.get_version_path_from_list(
             use_version, pype_versions)
-        if version_path:
+        if not version_path:
-            # use specified
-            bootstrap.add_paths_from_directory(version_path)
-
-        else:
             if use_version is not None:
                 print(("!!! Specified version was not found, using "
                        "latest available"))
-            # use latest
+            # specified version was not found so use latest detected.
-            version_path = pype_versions[-1].path
+            version_path = pype_version.path
-            bootstrap.add_paths_from_directory(version_path)
-            use_version = str(pype_versions[-1])
 
+        # test if latest detected is installed (in user data dir)
+        is_inside = False
+        try:
+            is_inside = pype_version.path.resolve().relative_to(
+                bootstrap.data_dir)
+        except ValueError:
+            # if relative path cannot be calculated, Pype version is not
+            # inside user data dir
+            pass
+
+        if not is_inside:
+            # install latest version to user data dir
+            version_path = bootstrap.install_version(
+                pype_version, force=True)
+
+        # inject version to Python environment (sys.path, ...)
+        bootstrap.add_paths_from_directory(version_path)
+
+        # add stuff from `<frozen>/lib` to PYTHONPATH.
+        os.environ["PYTHONPATH"] += os.pathsep + os.path.normpath(
+            os.path.join(os.path.dirname(sys.executable), "lib")
+        )
+
+        # set PYPE_ROOT to point to currently used Pype version.
         os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix())
     else:
         # run through repos and add them to sys.path and PYTHONPATH
+        # set root
         pype_root = os.path.normpath(
             os.path.dirname(os.path.realpath(__file__)))
-        local_version = bootstrap.get_local_version()
+        # get current version of Pype
+        local_version = bootstrap.get_local_live_version()
         if use_version and use_version != local_version:
             version_path = BootstrapRepos.get_version_path_from_list(
                 use_version, pype_versions)
@@ -248,10 +362,14 @@ def boot():
     info.insert(0, ">>> Using Pype from [ {} ]".format(
         os.path.dirname(cli.__file__)))
 
-    info_length = len(max(info, key=len))
+    t_width = os.get_terminal_size().columns
-    info.insert(0, f"*** Pype [{__version__}] " + "-" * info_length)
+    _header = f"*** Pype [{__version__}] "
+
+    info.insert(0, _header + "-" * (t_width - len(_header)))
     for i in info:
-        t.echo(i)
+        # don't show for running scripts
+        if all(item not in sys.argv for item in silent_commands):
+            t.echo(i)
 
     try:
         cli.main(obj={}, prog_name="pype")
@@ -302,7 +420,7 @@ def get_info() -> list:
         if log_components["auth_db"]:
             infos.append(("  - auth source", log_components["auth_db"]))
 
-    maximum = max([len(i[0]) for i in infos])
+    maximum = max(len(i[0]) for i in infos)
     formatted = []
     for info in infos:
         padding = (maximum - len(info[0])) + 1

tests/igniter/test_bootstrap_repos.py

@@ -108,6 +108,11 @@ def test_pype_version():
     assert v11.client == "client"
 
 
+def test_get_main_version():
+    ver = PypeVersion(1, 2, 3, variant="staging", client="foo")
+    assert ver.get_main_version() == "1.2.3"
+
+
 def test_get_version_path_from_list():
     versions = [
         PypeVersion(1, 2, 3, path=Path('/foo/bar')),

tools/create_env.ps1

@@ -105,14 +105,13 @@ catch {
     Exit-WithCode 1
 }
 
-
 Write-Host ">>> " -NoNewline -ForegroundColor green
 Write-Host "Creating virtual env ..."
 & python -m venv venv
 Write-Host ">>> " -NoNewline -ForegroundColor green
 Write-Host "Entering venv ..."
 try {
-  . (".\venv\Scripts\Activate.ps1")
+  . ("$($pype_root)\venv\Scripts\Activate.ps1")
 }
 catch {
     Write-Host "!!! Failed to activate" -ForegroundColor red