Merge pull request #947 from pypeclub/feature/python_executable_replacement

Python executable replacement

pype/cli.py

@@ -31,6 +31,12 @@ def settings(dev=False):
     PypeCommands().launch_settings_gui(dev)
 
 
+@main.command()
+def standalonepublisher():
+    """Show Pype Standalone publisher UI."""
+    PypeCommands().launch_standalone_publisher()
+
+
 @main.command()
 @click.option("-d", "--debug",
               is_flag=True, help=("Run pype tray in debug mode"))
@@ -88,43 +94,18 @@ def eventserver(debug,
     """
     if debug:
         os.environ['PYPE_DEBUG'] = "3"
-    # map eventserver options
-    # TODO: switch eventserver to click, normalize option names
-    args = []
-    if ftrack_url:
-        args.append('-ftrackurl')
-        args.append(ftrack_url)
 
-    if ftrack_user:
-        args.append('-ftrackuser')
-        args.append(ftrack_user)
-
-    if ftrack_api_key:
-        args.append('-ftrackapikey')
-        args.append(ftrack_api_key)
-
-    if ftrack_events_path:
-        args.append('-ftrackeventpaths')
-        args.append(ftrack_events_path)
-
-    if no_stored_credentials:
-        args.append('-noloadcred')
-
-    if store_credentials:
-        args.append('-storecred')
-
-    if legacy:
-        args.append('-legacy')
-
-    if clockify_api_key:
-        args.append('-clockifyapikey')
-        args.append(clockify_api_key)
-
-    if clockify_workspace:
-        args.append('-clockifyworkspace')
-        args.append(clockify_workspace)
-
-    PypeCommands().launch_eventservercli(args)
+    PypeCommands().launch_eventservercli(
+        ftrack_url,
+        ftrack_user,
+        ftrack_api_key,
+        ftrack_events_path,
+        no_stored_credentials,
+        store_credentials,
+        legacy,
+        clockify_api_key,
+        clockify_workspace
+    )
 
 
 @main.command()

pype/lib/__init__.py

@@ -14,6 +14,7 @@ site.addsitedir(
 
 from .terminal import Terminal
 from .execute import (
+    get_pype_execute_args,
     execute,
     run_subprocess
 )
@@ -112,6 +113,7 @@ from .editorial import (
 terminal = Terminal
 
 __all__ = [
+    "get_pype_execute_args",
     "execute",
     "run_subprocess",
 

pype/lib/execute.py

@@ -133,3 +133,33 @@ def run_subprocess(*args, **kwargs):
         raise RuntimeError(exc_msg)
 
     return full_output
+
+
+def get_pype_execute_args(*args):
+    """Arguments to run pype command.
+
+    Arguments for subprocess when need to spawn new pype process. Which may be
+    needed when new python process for pype scripts must be executed in build
+    pype.
+
+    ## Why is this needed?
+    Pype executed from code has different executable set to virtual env python
+    and must have path to script as first argument which is not needed for
+    build pype.
+
+    It is possible to pass any arguments that will be added after pype
+    executables.
+    """
+    pype_executable = os.environ["PYPE_EXECUTABLE"]
+    pype_args = [pype_executable]
+
+    executable_filename = os.path.basename(pype_executable)
+    if "python" in executable_filename.lower():
+        pype_args.append(
+            os.path.join(os.environ["PYPE_ROOT"], "start.py")
+        )
+
+    if args:
+        pype_args.extend(args)
+
+    return pype_args

pype/modules/ftrack/ftrack_server/event_server_cli.py

@@ -14,6 +14,7 @@ import uuid
 
 import ftrack_api
 import pymongo
+from pype.lib import get_pype_execute_args
 from pype.modules.ftrack.lib import (
     credentials,
     get_ftrack_url_from_settings
@@ -131,8 +132,9 @@ def legacy_server(ftrack_url):
 
         if subproc is None:
             if subproc_failed_count < max_fail_count:
+                args = get_pype_execute_args("run", subproc_path)
                 subproc = subprocess.Popen(
-                    ["python", subproc_path],
+                    args,
                     stdout=subprocess.PIPE
                 )
             elif subproc_failed_count == max_fail_count:
@@ -414,6 +416,56 @@ def main_loop(ftrack_url):
         time.sleep(1)
 
 
+def run_event_server(
+    ftrack_url,
+    ftrack_user,
+    ftrack_api_key,
+    ftrack_events_path,
+    no_stored_credentials,
+    store_credentials,
+    legacy,
+    clockify_api_key,
+    clockify_workspace
+):
+    if not no_stored_credentials:
+        cred = credentials.get_credentials(ftrack_url)
+        username = cred.get('username')
+        api_key = cred.get('api_key')
+
+    if clockify_workspace and clockify_api_key:
+        os.environ["CLOCKIFY_WORKSPACE"] = clockify_workspace
+        os.environ["CLOCKIFY_API_KEY"] = clockify_api_key
+
+    # Check url regex and accessibility
+    ftrack_url = check_ftrack_url(ftrack_url)
+    if not ftrack_url:
+        print('Exiting! < Please enter Ftrack server url >')
+        return 1
+
+    # Validate entered credentials
+    if not validate_credentials(ftrack_url, username, api_key):
+        print('Exiting! < Please enter valid credentials >')
+        return 1
+
+    if store_credentials:
+        credentials.save_credentials(username, api_key, ftrack_url)
+
+    # Set Ftrack environments
+    os.environ["FTRACK_SERVER"] = ftrack_url
+    os.environ["FTRACK_API_USER"] = username
+    os.environ["FTRACK_API_KEY"] = api_key
+    # TODO This won't work probably
+    if ftrack_events_path:
+        if isinstance(ftrack_events_path, (list, tuple)):
+            ftrack_events_path = os.pathsep.join(ftrack_events_path)
+        os.environ["FTRACK_EVENTS_PATH"] = ftrack_events_path
+
+    if legacy:
+        return legacy_server(ftrack_url)
+
+    return main_loop(ftrack_url)
+
+
 def main(argv):
     '''
     There are 4 values neccessary for event server:

pype/modules/ftrack/ftrack_server/socket_thread.py

@@ -6,6 +6,7 @@ import threading
 import traceback
 import subprocess
 from pype.api import Logger
+from pype.lib import get_pype_execute_args
 
 
 class SocketThread(threading.Thread):
@@ -57,22 +58,15 @@ class SocketThread(threading.Thread):
 
         env = os.environ.copy()
         env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id)
-        executable_args = [
-            sys.executable
-        ]
-        if getattr(sys, "frozen", False):
-            executable_args.append("run")
-
-        self.subproc = subprocess.Popen(
-            [
-                *executable_args,
-                self.filepath,
-                *self.additional_args,
-                str(self.port)
-            ],
-            env=env,
-            stdin=subprocess.PIPE
+        # Pype executable (with path to start script if not build)
+        args = get_pype_execute_args(
+            # Add `run` command
+            "run",
+            self.filepath,
+            *self.additional_args,
+            str(self.port)
         )
+        self.subproc = subprocess.Popen(args, env=env, stdin=subprocess.PIPE)
 
         # Listen for incoming connections
         sock.listen(1)

pype/modules/standalonepublish_action.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import subprocess
+from pype.lib import get_pype_execute_args
 from . import PypeModule, ITrayAction
 
 
@@ -29,13 +30,5 @@ class StandAlonePublishAction(PypeModule, ITrayAction):
         self.publish_paths.extend(publish_paths)
 
     def run_standalone_publisher(self):
-        from pype import tools
-        standalone_publisher_tool_path = os.path.join(
-            os.path.dirname(os.path.abspath(tools.__file__)),
-            "standalonepublish"
-        )
-        subprocess.Popen([
-            sys.executable,
-            standalone_publisher_tool_path,
-            os.pathsep.join(self.publish_paths).replace("\\", "/")
-        ])
+        args = get_pype_execute_args("standalonepublisher")
+        subprocess.Popen(args, creationflags=subprocess.DETACHED_PROCESS)

pype/plugins/global/publish/extract_burnin.py

@@ -4,10 +4,15 @@ import json
 import copy
 import tempfile
 
+import pype
 import pype.api
 import pyblish
-from pype.lib import should_decompress, \
-    get_decompress_dir, decompress
+from pype.lib import (
+    get_pype_execute_args,
+    should_decompress,
+    get_decompress_dir,
+    decompress
+)
 import shutil
 
 
@@ -125,7 +130,14 @@ class ExtractBurnin(pype.api.Extractor):
 
         anatomy = instance.context.data["anatomy"]
         scriptpath = self.burnin_script_path()
-        executable = self.python_executable_path()
+        # Executable args that will execute the script
+        # [pype executable, *pype script, "run"]
+        executable_args = get_pype_execute_args("run", scriptpath)
+
+        # Environments for script process
+        env = os.environ.copy()
+        # pop PYTHONPATH
+        env.pop("PYTHONPATH", None)
 
         for idx, repre in enumerate(tuple(instance.data["representations"])):
             self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"]))
@@ -256,17 +268,13 @@ class ExtractBurnin(pype.api.Extractor):
                 )
 
                 # Prepare subprocess arguments
-                args = [
-                    "\"{}\"".format(executable),
-                    "\"{}\"".format(scriptpath),
-                    "\"{}\"".format(temporary_json_filepath)
-                ]
-                subprcs_cmd = " ".join(args)
-                self.log.debug("Executing: {}".format(subprcs_cmd))
+                args = list(executable_args)
+                args.append(temporary_json_filepath)
+                self.log.debug("Executing: {}".format(" ".join(args)))
 
                 # Run burnin script
                 pype.api.run_subprocess(
-                    subprcs_cmd, shell=True, logger=self.log
+                    args, shell=True, logger=self.log, env=env
                 )
 
                 # Remove the temporary json
@@ -812,19 +820,9 @@ class ExtractBurnin(pype.api.Extractor):
 
     def burnin_script_path(self):
         """Return path to python script for burnin processing."""
-        # TODO maybe convert to Plugin's attribute
-        # Get script path.
-        module_path = os.environ["PYPE_ROOT"]
-
-        # There can be multiple paths in PYPE_ROOT, in which case
-        # we just take first one.
-        if os.pathsep in module_path:
-            module_path = module_path.split(os.pathsep)[0]
-
         scriptpath = os.path.normpath(
             os.path.join(
-                module_path,
-                "pype",
+                pype.PACKAGE_DIR,
                 "scripts",
                 "otio_burnin.py"
             )
@@ -833,17 +831,3 @@ class ExtractBurnin(pype.api.Extractor):
         self.log.debug("scriptpath: {}".format(scriptpath))
 
         return scriptpath
-
-    def python_executable_path(self):
-        """Return path to Python 3 executable."""
-        # TODO maybe convert to Plugin's attribute
-        # Get executable.
-        executable = os.getenv("PYPE_PYTHON_EXE")
-
-        # There can be multiple paths in PYPE_PYTHON_EXE, in which case
-        # we just take first one.
-        if os.pathsep in executable:
-            executable = executable.split(os.pathsep)[0]
-
-        self.log.debug("executable: {}".format(executable))
-        return executable

pype/pype_commands.py

@@ -28,19 +28,17 @@ class PypeCommands:
         user_role = "developer"
         settings.main(user_role)
 
-    def launch_eventservercli(self, args):
-        from pype.modules import ftrack
-        from pype.lib import execute
-
-        fname = os.path.join(
-            os.path.dirname(os.path.abspath(ftrack.__file__)),
-            "ftrack_server",
-            "event_server_cli.py"
+    @staticmethod
+    def launch_eventservercli(*args):
+        from pype.modules.ftrack.ftrack_server.event_server_cli import (
+            run_event_server
         )
+        return run_event_server(*args)
 
-        return execute([
-            sys.executable, "-u", fname
-        ])
+    @staticmethod
+    def launch_standalone_publisher():
+        from pype.tools import standalonepublish
+        standalonepublish.main()
 
     def publish(self, gui, paths):
         pass

pype/tools/standalonepublish/__init__.py

@@ -1,8 +1,10 @@
 from .app import (
-    show,
-    cli
+    main,
+    Window
+)
+
+
+__all__ = (
+    "main",
+    "Window"
 )
-__all__ = [
-    "show",
-    "cli"
-]

pype/tools/standalonepublish/__main__.py

@@ -1,35 +0,0 @@
-import os
-import sys
-import app
-import ctypes
-import signal
-from Qt import QtWidgets, QtGui
-from avalon import style
-from pype.api import resources
-
-
-if __name__ == "__main__":
-
-    # Allow to change icon of running process in windows taskbar
-    if os.name == "nt":
-        ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
-            u"standalonepublish"
-        )
-
-    qt_app = QtWidgets.QApplication([])
-    # app.setQuitOnLastWindowClosed(False)
-    qt_app.setStyleSheet(style.load_stylesheet())
-    icon = QtGui.QIcon(resources.pype_icon_filepath())
-    qt_app.setWindowIcon(icon)
-
-    def signal_handler(sig, frame):
-        print("You pressed Ctrl+C. Process ended.")
-        qt_app.quit()
-
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    window = app.Window(sys.argv[-1].split(os.pathsep))
-    window.show()
-
-    sys.exit(qt_app.exec_())

pype/tools/standalonepublish/app.py

@@ -1,7 +1,18 @@
+import os
+import sys
+import ctypes
+import signal
+
 from bson.objectid import ObjectId
-from Qt import QtWidgets, QtCore
-from widgets import AssetWidget, FamilyWidget, ComponentsWidget, ShadowWidget
+from Qt import QtWidgets, QtCore, QtGui
+
+from .widgets import (
+    AssetWidget, FamilyWidget, ComponentsWidget, ShadowWidget
+)
+from avalon import style
+from pype.api import resources
 from avalon.api import AvalonMongoDB
+from pype.modules import ModulesManager
 
 
 class Window(QtWidgets.QDialog):
@@ -194,3 +205,32 @@ class Window(QtWidgets.QDialog):
         data.update(self.widget_components.collect_data())
 
         return data
+
+
+def main():
+    # Allow to change icon of running process in windows taskbar
+    if os.name == "nt":
+        ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID(
+            u"standalonepublish"
+        )
+
+    qt_app = QtWidgets.QApplication([])
+    # app.setQuitOnLastWindowClosed(False)
+    qt_app.setStyleSheet(style.load_stylesheet())
+    icon = QtGui.QIcon(resources.pype_icon_filepath())
+    qt_app.setWindowIcon(icon)
+
+    def signal_handler(sig, frame):
+        print("You pressed Ctrl+C. Process ended.")
+        qt_app.quit()
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    modules_manager = ModulesManager()
+    module = modules_manager.modules_by_name["standalonepublish_tool"]
+
+    window = Window(module.publish_paths)
+    window.show()
+
+    sys.exit(qt_app.exec_())

pype/tools/standalonepublish/widgets/widget_components.py

@@ -9,6 +9,7 @@ from Qt import QtWidgets, QtCore
 from . import DropDataFrame
 from avalon import io
 from pype.api import execute, Logger
+from pype.lib import get_pype_execute_args
 
 log = Logger().get_logger("standalonepublisher")
 
@@ -207,10 +208,8 @@ def cli_publish(data, publish_paths, gui=True):
     if data.get("family", "").lower() == "editorial":
         envcopy["PYBLISH_SUSPEND_LOGS"] = "1"
 
-    result = execute(
-        [sys.executable, PUBLISH_SCRIPT_PATH],
-        env=envcopy
-    )
+    args = get_pype_execute_args("run", PUBLISH_SCRIPT_PATH)
+    result = execute(args, env=envcopy)
 
     result = {}
     if os.path.exists(json_data_path):

start.py

@@ -116,7 +116,7 @@ from igniter.tools import load_environments  # noqa: E402
 from igniter.bootstrap_repos import PypeVersion  # noqa: E402
 
 bootstrap = BootstrapRepos()
-silent_commands = ["run", "igniter"]
+silent_commands = ["run", "igniter", "standalonepublisher"]
 
 
 def set_environments() -> None:
@@ -276,23 +276,36 @@ def _determine_mongodb() -> str:
 def _initialize_environment(pype_version: PypeVersion) -> None:
     version_path = pype_version.path
     os.environ["PYPE_VERSION"] = pype_version.version
+    # set PYPE_ROOT to point to currently used Pype version.
+    os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix())
     # inject version to Python environment (sys.path, ...)
     print(">>> Injecting Pype version to running environment  ...")
     bootstrap.add_paths_from_directory(version_path)
 
-    # add venv 'site-packages' to PYTHONPATH
-    python_path = os.getenv("PYTHONPATH", "")
-    split_paths = python_path.split(os.pathsep)
-    # add pype tools
-    split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"))
-    # add common pype vendor
-    # (common for multiple Python interpreter versions)
-    split_paths.append(os.path.join(
-        os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common"))
-    os.environ["PYTHONPATH"] = os.pathsep.join(split_paths)
+    # Additional sys paths related to PYPE_ROOT directory
+    # TODO move additional paths to `boot` part when PYPE_ROOT will point
+    # to same hierarchy from code and from frozen pype
+    additional_paths = [
+        # add pype tools
+        os.path.join(os.environ["PYPE_ROOT"], "pype", "pype", "tools"),
+        # add common pype vendor
+        # (common for multiple Python interpreter versions)
+        os.path.join(
+            os.environ["PYPE_ROOT"],
+            "pype",
+            "pype",
+            "vendor",
+            "python",
+            "common"
+        )
+    ]
 
-    # set PYPE_ROOT to point to currently used Pype version.
-    os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix())
+    split_paths = os.getenv("PYTHONPATH", "").split(os.pathsep)
+    for path in additional_paths:
+        split_paths.insert(0, path)
+        sys.path.insert(0, path)
+
+    os.environ["PYTHONPATH"] = os.pathsep.join(split_paths)
 
 
 def _find_frozen_pype(use_version: str = None,
@@ -416,23 +429,33 @@ def _bootstrap_from_code(use_version):
     # add self to python paths
     repos.insert(0, pype_root)
     for repo in repos:
-        sys.path.append(repo)
+        sys.path.insert(0, repo)
 
     # add venv 'site-packages' to PYTHONPATH
     python_path = os.getenv("PYTHONPATH", "")
     split_paths = python_path.split(os.pathsep)
-    split_paths += repos
-    # add pype tools
-    split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"))
+    # Add repos as first in list
+    split_paths = repos + split_paths
     # last one should be venv site-packages
     # this is slightly convoluted as we can get here from frozen code too
     # in case when we are running without any version installed.
     if not getattr(sys, 'frozen', False):
         split_paths.append(site.getsitepackages()[-1])
-    # add common pype vendor
-    # (common for multiple Python interpreter versions)
-    split_paths.append(os.path.join(
-        os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common"))
+        # TODO move additional paths to `boot` part when PYPE_ROOT will point
+        # to same hierarchy from code and from frozen pype
+        additional_paths = [
+            # add pype tools
+            os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"),
+            # add common pype vendor
+            # (common for multiple Python interpreter versions)
+            os.path.join(
+                os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common"
+            )
+        ]
+        for path in additional_paths:
+            split_paths.insert(0, path)
+            sys.path.insert(0, path)
+
     os.environ["PYTHONPATH"] = os.pathsep.join(split_paths)
 
     return Path(version_path)
@@ -485,7 +508,7 @@ def boot():
     # ------------------------------------------------------------------------
     # Find Pype versions
     # ------------------------------------------------------------------------
-
+    # WARNING Environment PYPE_ROOT may change if frozen pype is executed
     if getattr(sys, 'frozen', False):
         # find versions of Pype to be used with frozen code
         try:
@@ -507,10 +530,15 @@ def boot():
         os.environ["PYPE_REPOS_ROOT"] = os.path.join(
             os.environ["PYPE_ROOT"], "repos")
 
-    # delete Pype module from cache so it is used from specific version
+    # delete Pype module and it's submodules from cache so it is used from
+    # specific version
+    modules_to_del = []
+    for module_name in tuple(sys.modules):
+        if module_name == "pype" or module_name.startswith("pype."):
+            modules_to_del.append(sys.modules.pop(module_name))
     try:
-        del sys.modules["pype"]
-        del sys.modules["pype.version"]
+        for module_name in modules_to_del:
+            del sys.modules[module_name]
     except AttributeError:
         pass
     except KeyError: