From 87e60dc5f50eec7bcec6153c70125a3343832b12 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 15:44:43 +0100 Subject: [PATCH 01/20] added function which returns arguments for pype subprocess --- pype/lib/__init__.py | 2 ++ pype/lib/execute.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index d10f3d199d..59eb2a645c 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -14,6 +14,7 @@ site.addsitedir( from .terminal import Terminal from .execute import ( + get_pype_execute_args, execute, run_subprocess ) @@ -112,6 +113,7 @@ from .editorial import ( terminal = Terminal __all__ = [ + "get_pype_execute_args", "execute", "run_subprocess", diff --git a/pype/lib/execute.py b/pype/lib/execute.py index 1f1adcdf23..6ad43c5e0d 100644 --- a/pype/lib/execute.py +++ b/pype/lib/execute.py @@ -133,3 +133,26 @@ def run_subprocess(*args, **kwargs): raise RuntimeError(exc_msg) return full_output + + +def get_pype_execute_args(): + """Arguments to run pype command. + + Arguments for subprocess when need to spawn new pype process. Which may be + needed when new python process for pype scripts must be executed in build + pype. + + ## Why is this needed? + Pype executed from code has different executable set to virtual env python + and must have path to script as first argument which is not needed for + build pype. + """ + pype_executable = os.environ["PYPE_EXECUTABLE"] + args = [pype_executable] + + executable_filename = os.path.dirname(pype_executable) + if "python" in executable_filename.lower(): + args.append( + os.path.join(os.environ["PYPE_ROOT"], "start.py") + ) + return args From 04af95ac0195cf78051c51fe29a4ab0f7a03ca46 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 15:52:46 +0100 Subject: [PATCH 02/20] standalone publisher and ftrack server are using `get_pype_execute_args` to spawn subprocesses --- pype/modules/ftrack/ftrack_server/socket_thread.py | 11 ++++++----- pype/modules/standalonepublish_action.py | 10 +++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index c638c9fa03..15681c00b8 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -6,6 +6,7 @@ import threading import traceback import subprocess from pype.api import Logger +from pype.lib import get_pype_execute_args class SocketThread(threading.Thread): @@ -57,11 +58,11 @@ class SocketThread(threading.Thread): env = os.environ.copy() env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) - executable_args = [ - sys.executable - ] - if getattr(sys, "frozen", False): - executable_args.append("run") + + # Pype executable + executable_args = get_pype_execute_args() + # Add `run` command + executable_args.append("run") self.subproc = subprocess.Popen( [ diff --git a/pype/modules/standalonepublish_action.py b/pype/modules/standalonepublish_action.py index 4bcb5b6018..8dcc68113d 100644 --- a/pype/modules/standalonepublish_action.py +++ b/pype/modules/standalonepublish_action.py @@ -1,6 +1,7 @@ import os import sys import subprocess +from pype.lib import get_pype_execute_args from . import PypeModule, ITrayAction @@ -29,13 +30,16 @@ class StandAlonePublishAction(PypeModule, ITrayAction): self.publish_paths.extend(publish_paths) def run_standalone_publisher(self): + # TODO add command to cli.py from pype import tools standalone_publisher_tool_path = os.path.join( os.path.dirname(os.path.abspath(tools.__file__)), "standalonepublish" ) - subprocess.Popen([ - sys.executable, + args = [ + *get_pype_execute_args(), + "run", standalone_publisher_tool_path, os.pathsep.join(self.publish_paths).replace("\\", "/") - ]) + ] + subprocess.Popen(args, creationflags=subprocess.DETACHED_PROCESS) From 3f9bc087657d03ffd574bcd38546a38ee554c47b Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 15:53:00 +0100 Subject: [PATCH 03/20] fixed executable filename --- pype/lib/execute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/lib/execute.py b/pype/lib/execute.py index 6ad43c5e0d..05357df6ae 100644 --- a/pype/lib/execute.py +++ b/pype/lib/execute.py @@ -150,7 +150,7 @@ def get_pype_execute_args(): pype_executable = os.environ["PYPE_EXECUTABLE"] args = [pype_executable] - executable_filename = os.path.dirname(pype_executable) + executable_filename = os.path.basename(pype_executable) if "python" in executable_filename.lower(): args.append( os.path.join(os.environ["PYPE_ROOT"], "start.py") From 2985925aa076ffa5f003c895be97a56b552059d6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 15:57:19 +0100 Subject: [PATCH 04/20] standalone publisher tool is using pype execute arguments when trigerring pyblish --- pype/tools/standalonepublish/widgets/widget_components.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pype/tools/standalonepublish/widgets/widget_components.py b/pype/tools/standalonepublish/widgets/widget_components.py index 7e0327f00a..1ba282ab83 100644 --- a/pype/tools/standalonepublish/widgets/widget_components.py +++ b/pype/tools/standalonepublish/widgets/widget_components.py @@ -9,6 +9,7 @@ from Qt import QtWidgets, QtCore from . import DropDataFrame from avalon import io from pype.api import execute, Logger +from pype.lib import get_pype_execute_args log = Logger().get_logger("standalonepublisher") @@ -207,8 +208,13 @@ def cli_publish(data, publish_paths, gui=True): if data.get("family", "").lower() == "editorial": envcopy["PYBLISH_SUSPEND_LOGS"] = "1" + args = [ + *get_pype_execute_args(), + "run", + PUBLISH_SCRIPT_PATH + ] result = execute( - [sys.executable, PUBLISH_SCRIPT_PATH], + args, env=envcopy ) From c59cb621b785e51fc3b1b6eeeb25f700619339e5 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 16:01:57 +0100 Subject: [PATCH 05/20] fixed legacy ftrack event server --- pype/modules/ftrack/ftrack_server/event_server_cli.py | 8 +++++++- pype/modules/ftrack/ftrack_server/socket_thread.py | 10 ++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pype/modules/ftrack/ftrack_server/event_server_cli.py b/pype/modules/ftrack/ftrack_server/event_server_cli.py index 96581f0a38..1c05e34f18 100644 --- a/pype/modules/ftrack/ftrack_server/event_server_cli.py +++ b/pype/modules/ftrack/ftrack_server/event_server_cli.py @@ -14,6 +14,7 @@ import uuid import ftrack_api import pymongo +from pype.lib import get_pype_execute_args from pype.modules.ftrack.lib import ( credentials, get_ftrack_url_from_settings @@ -131,8 +132,13 @@ def legacy_server(ftrack_url): if subproc is None: if subproc_failed_count < max_fail_count: + args = [ + *get_pype_execute_args(), + "run", + subproc_path + ] subproc = subprocess.Popen( - ["python", subproc_path], + args, stdout=subprocess.PIPE ) elif subproc_failed_count == max_fail_count: diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index 15681c00b8..51b0f6d305 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -59,14 +59,12 @@ class SocketThread(threading.Thread): env = os.environ.copy() env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) - # Pype executable - executable_args = get_pype_execute_args() - # Add `run` command - executable_args.append("run") - self.subproc = subprocess.Popen( [ - *executable_args, + # Pype executable (with path to start script if not build) + *get_pype_execute_args(), + # Add `run` command + "run", self.filepath, *self.additional_args, str(self.port) From cae34382a47ecb7c15087b0e9ff41a2bf05ce81d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 16:51:20 +0100 Subject: [PATCH 06/20] have to delete all pype modules from sys.modules --- start.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/start.py b/start.py index ad863481ff..5618920164 100644 --- a/start.py +++ b/start.py @@ -508,9 +508,13 @@ def boot(): os.environ["PYPE_ROOT"], "repos") # delete Pype module from cache so it is used from specific version + modules_to_del = [] + for module_name in sys.modules: + if module_name == "pype" or module_name.startswith("pype."): + modules_to_del.append(module_name) try: - del sys.modules["pype"] - del sys.modules["pype.version"] + for module_name in modules_to_del: + del sys.modules[module_name] except AttributeError: pass except KeyError: From 6ef953134362a23a088c12602a9252361002bb8c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 17:42:39 +0100 Subject: [PATCH 07/20] extract burnin use pype executable to execute burnin script --- pype/plugins/global/publish/extract_burnin.py | 53 ++++++------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 8d1b9c81e3..ee559d10ae 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -4,10 +4,15 @@ import json import copy import tempfile +import pype import pype.api import pyblish -from pype.lib import should_decompress, \ - get_decompress_dir, decompress +from pype.lib import ( + get_pype_execute_args, + should_decompress, + get_decompress_dir, + decompress +) import shutil @@ -125,7 +130,7 @@ class ExtractBurnin(pype.api.Extractor): anatomy = instance.context.data["anatomy"] scriptpath = self.burnin_script_path() - executable = self.python_executable_path() + executable_args = get_pype_execute_args() for idx, repre in enumerate(tuple(instance.data["representations"])): self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) @@ -256,17 +261,17 @@ class ExtractBurnin(pype.api.Extractor): ) # Prepare subprocess arguments - args = [ - "\"{}\"".format(executable), - "\"{}\"".format(scriptpath), - "\"{}\"".format(temporary_json_filepath) - ] - subprcs_cmd = " ".join(args) - self.log.debug("Executing: {}".format(subprcs_cmd)) + args = [] + args.extend(executable_args) + args.extend([ + scriptpath, + temporary_json_filepath + ]) + self.log.debug("Executing: {}".format(" ".join(args))) # Run burnin script pype.api.run_subprocess( - subprcs_cmd, shell=True, logger=self.log + args, shell=True, logger=self.log ) # Remove the temporary json @@ -812,19 +817,9 @@ class ExtractBurnin(pype.api.Extractor): def burnin_script_path(self): """Return path to python script for burnin processing.""" - # TODO maybe convert to Plugin's attribute - # Get script path. - module_path = os.environ["PYPE_ROOT"] - - # There can be multiple paths in PYPE_ROOT, in which case - # we just take first one. - if os.pathsep in module_path: - module_path = module_path.split(os.pathsep)[0] - scriptpath = os.path.normpath( os.path.join( - module_path, - "pype", + pype.PACKAGE_DIR, "scripts", "otio_burnin.py" ) @@ -833,17 +828,3 @@ class ExtractBurnin(pype.api.Extractor): self.log.debug("scriptpath: {}".format(scriptpath)) return scriptpath - - def python_executable_path(self): - """Return path to Python 3 executable.""" - # TODO maybe convert to Plugin's attribute - # Get executable. - executable = os.getenv("PYPE_PYTHON_EXE") - - # There can be multiple paths in PYPE_PYTHON_EXE, in which case - # we just take first one. - if os.pathsep in executable: - executable = executable.split(os.pathsep)[0] - - self.log.debug("executable: {}".format(executable)) - return executable From df08fe6a0bce471a223208416914cac5831f251d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 19:38:20 +0100 Subject: [PATCH 08/20] extract burnin can execute pype burnin script --- pype/plugins/global/publish/extract_burnin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index ee559d10ae..1051ff1a59 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -130,7 +130,15 @@ class ExtractBurnin(pype.api.Extractor): anatomy = instance.context.data["anatomy"] scriptpath = self.burnin_script_path() + # Executable args that will execute the script + # [pype executable, *pype script, "run"] executable_args = get_pype_execute_args() + executable_args.append("run") + + # Environments for script process + env = os.environ.copy() + # pop PYTHONPATH + env.pop("PYTHONPATH", None) for idx, repre in enumerate(tuple(instance.data["representations"])): self.log.debug("repre ({}): `{}`".format(idx + 1, repre["name"])) @@ -271,7 +279,7 @@ class ExtractBurnin(pype.api.Extractor): # Run burnin script pype.api.run_subprocess( - args, shell=True, logger=self.log + args, shell=True, logger=self.log, env=env ) # Remove the temporary json From 11b58ed36b1978b489438a7a371c95c531dfb468 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Fri, 29 Jan 2021 20:16:58 +0100 Subject: [PATCH 09/20] it is possible to run ftrack event server --- pype/cli.py | 47 ++++------------- .../ftrack/ftrack_server/event_server_cli.py | 50 +++++++++++++++++++ pype/pype_commands.py | 17 ++----- 3 files changed, 66 insertions(+), 48 deletions(-) diff --git a/pype/cli.py b/pype/cli.py index 62975bff31..cf1cd59cd1 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -88,43 +88,18 @@ def eventserver(debug, """ if debug: os.environ['PYPE_DEBUG'] = "3" - # map eventserver options - # TODO: switch eventserver to click, normalize option names - args = [] - if ftrack_url: - args.append('-ftrackurl') - args.append(ftrack_url) - if ftrack_user: - args.append('-ftrackuser') - args.append(ftrack_user) - - if ftrack_api_key: - args.append('-ftrackapikey') - args.append(ftrack_api_key) - - if ftrack_events_path: - args.append('-ftrackeventpaths') - args.append(ftrack_events_path) - - if no_stored_credentials: - args.append('-noloadcred') - - if store_credentials: - args.append('-storecred') - - if legacy: - args.append('-legacy') - - if clockify_api_key: - args.append('-clockifyapikey') - args.append(clockify_api_key) - - if clockify_workspace: - args.append('-clockifyworkspace') - args.append(clockify_workspace) - - PypeCommands().launch_eventservercli(args) + PypeCommands().launch_eventservercli( + ftrack_url, + ftrack_user, + ftrack_api_key, + ftrack_events_path, + no_stored_credentials, + store_credentials, + legacy, + clockify_api_key, + clockify_workspace + ) @main.command() diff --git a/pype/modules/ftrack/ftrack_server/event_server_cli.py b/pype/modules/ftrack/ftrack_server/event_server_cli.py index 1c05e34f18..cc51304fc6 100644 --- a/pype/modules/ftrack/ftrack_server/event_server_cli.py +++ b/pype/modules/ftrack/ftrack_server/event_server_cli.py @@ -420,6 +420,56 @@ def main_loop(ftrack_url): time.sleep(1) +def run_event_server( + ftrack_url, + ftrack_user, + ftrack_api_key, + ftrack_events_path, + no_stored_credentials, + store_credentials, + legacy, + clockify_api_key, + clockify_workspace +): + if not no_stored_credentials: + cred = credentials.get_credentials(ftrack_url) + username = cred.get('username') + api_key = cred.get('api_key') + + if clockify_workspace and clockify_api_key: + os.environ["CLOCKIFY_WORKSPACE"] = clockify_workspace + os.environ["CLOCKIFY_API_KEY"] = clockify_api_key + + # Check url regex and accessibility + ftrack_url = check_ftrack_url(ftrack_url) + if not ftrack_url: + print('Exiting! < Please enter Ftrack server url >') + return 1 + + # Validate entered credentials + if not validate_credentials(ftrack_url, username, api_key): + print('Exiting! < Please enter valid credentials >') + return 1 + + if store_credentials: + credentials.save_credentials(username, api_key, ftrack_url) + + # Set Ftrack environments + os.environ["FTRACK_SERVER"] = ftrack_url + os.environ["FTRACK_API_USER"] = username + os.environ["FTRACK_API_KEY"] = api_key + # TODO This won't work probably + if ftrack_events_path: + if isinstance(ftrack_events_path, (list, tuple)): + ftrack_events_path = os.pathsep.join(ftrack_events_path) + os.environ["FTRACK_EVENTS_PATH"] = ftrack_events_path + + if legacy: + return legacy_server(ftrack_url) + + return main_loop(ftrack_url) + + def main(argv): ''' There are 4 values neccessary for event server: diff --git a/pype/pype_commands.py b/pype/pype_commands.py index 1ec4d2c553..cc597f4beb 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -28,19 +28,12 @@ class PypeCommands: user_role = "developer" settings.main(user_role) - def launch_eventservercli(self, args): - from pype.modules import ftrack - from pype.lib import execute - - fname = os.path.join( - os.path.dirname(os.path.abspath(ftrack.__file__)), - "ftrack_server", - "event_server_cli.py" + @staticmethod + def launch_eventservercli(*args): + from pype.modules.ftrack.ftrack_server.event_server_cli import ( + run_event_server ) - - return execute([ - sys.executable, "-u", fname - ]) + return run_event_server(*args) def publish(self, gui, paths): pass From b61d515569c61c0e4323bcd5de17ffe4b3c3aa7d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:36:54 +0100 Subject: [PATCH 10/20] tools and common vendor are not set to build directory --- start.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/start.py b/start.py index 5618920164..b93500c302 100644 --- a/start.py +++ b/start.py @@ -276,23 +276,13 @@ def _determine_mongodb() -> str: def _initialize_environment(pype_version: PypeVersion) -> None: version_path = pype_version.path os.environ["PYPE_VERSION"] = pype_version.version + # set PYPE_ROOT to point to currently used Pype version. + os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) # inject version to Python environment (sys.path, ...) print(">>> Injecting Pype version to running environment ...") bootstrap.add_paths_from_directory(version_path) - # add venv 'site-packages' to PYTHONPATH - python_path = os.getenv("PYTHONPATH", "") - split_paths = python_path.split(os.pathsep) - # add pype tools - split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) - # add common pype vendor - # (common for multiple Python interpreter versions) - split_paths.append(os.path.join( - os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) - os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) - # set PYPE_ROOT to point to currently used Pype version. - os.environ["PYPE_ROOT"] = os.path.normpath(version_path.as_posix()) def _find_frozen_pype(use_version: str = None, From 96c8f922d875cb4e5449f289fa69e2669d979960 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:39:06 +0100 Subject: [PATCH 11/20] frozen pype adds sys paths to tools and common vendor to right dirs --- start.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/start.py b/start.py index b93500c302..35b80503bf 100644 --- a/start.py +++ b/start.py @@ -282,7 +282,28 @@ def _initialize_environment(pype_version: PypeVersion) -> None: print(">>> Injecting Pype version to running environment ...") bootstrap.add_paths_from_directory(version_path) + # Additional sys paths related to PYPE_ROOT directory + additional_paths = [ + # add pype tools + os.path.join(os.environ["PYPE_ROOT"], "pype", "pype", "tools"), + # add common pype vendor + # (common for multiple Python interpreter versions) + os.path.join( + os.environ["PYPE_ROOT"], + "pype", + "pype", + "vendor", + "python", + "common" + ) + ] + split_paths = os.getenv("PYTHONPATH", "").split(os.pathsep) + for path in additional_paths: + split_paths.insert(0, path) + sys.path.insert(0, path) + + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) def _find_frozen_pype(use_version: str = None, From 64806743456c61fcf1387420287917a72645f0dd Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:39:53 +0100 Subject: [PATCH 12/20] sys paths to tools and common vendor are in fron code initialization added only if not executed from frozen code --- start.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/start.py b/start.py index 35b80503bf..3e5d1715ed 100644 --- a/start.py +++ b/start.py @@ -433,17 +433,24 @@ def _bootstrap_from_code(use_version): python_path = os.getenv("PYTHONPATH", "") split_paths = python_path.split(os.pathsep) split_paths += repos - # add pype tools - split_paths.append(os.path.join(os.environ["PYPE_ROOT"], "pype", "tools")) # last one should be venv site-packages # this is slightly convoluted as we can get here from frozen code too # in case when we are running without any version installed. if not getattr(sys, 'frozen', False): split_paths.append(site.getsitepackages()[-1]) - # add common pype vendor - # (common for multiple Python interpreter versions) - split_paths.append(os.path.join( - os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common")) + additional_paths = [ + # add pype tools + os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"), + # add common pype vendor + # (common for multiple Python interpreter versions) + os.path.join( + os.environ["PYPE_ROOT"], "pype", "vendor", "python", "common" + ) + ] + for path in additional_paths: + split_paths.insert(0, path) + sys.path.insert(0, path) + os.environ["PYTHONPATH"] = os.pathsep.join(split_paths) return Path(version_path) From 5666f90879c86c65c3da97e840318f298e142b6e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:40:20 +0100 Subject: [PATCH 13/20] added TODO comments to tools and common vendor sys paths --- start.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/start.py b/start.py index 3e5d1715ed..4453e912f0 100644 --- a/start.py +++ b/start.py @@ -283,6 +283,8 @@ def _initialize_environment(pype_version: PypeVersion) -> None: bootstrap.add_paths_from_directory(version_path) # Additional sys paths related to PYPE_ROOT directory + # TODO move additional paths to `boot` part when PYPE_ROOT will point + # to same hierarchy from code and from frozen pype additional_paths = [ # add pype tools os.path.join(os.environ["PYPE_ROOT"], "pype", "pype", "tools"), @@ -438,6 +440,8 @@ def _bootstrap_from_code(use_version): # in case when we are running without any version installed. if not getattr(sys, 'frozen', False): split_paths.append(site.getsitepackages()[-1]) + # TODO move additional paths to `boot` part when PYPE_ROOT will point + # to same hierarchy from code and from frozen pype additional_paths = [ # add pype tools os.path.join(os.environ["PYPE_ROOT"], "pype", "tools"), From 161b0c14a06530676f6ccd1eac35e006c42fb5ac Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:41:00 +0100 Subject: [PATCH 14/20] prepend repo paths to sys.path instead of appending --- start.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/start.py b/start.py index 4453e912f0..93fff22e0c 100644 --- a/start.py +++ b/start.py @@ -429,12 +429,13 @@ def _bootstrap_from_code(use_version): # add self to python paths repos.insert(0, pype_root) for repo in repos: - sys.path.append(repo) + sys.path.insert(0, repo) # add venv 'site-packages' to PYTHONPATH python_path = os.getenv("PYTHONPATH", "") split_paths = python_path.split(os.pathsep) - split_paths += repos + # Add repos as first in list + split_paths = repos + split_paths # last one should be venv site-packages # this is slightly convoluted as we can get here from frozen code too # in case when we are running without any version installed. From 5a7bf4106b19da14cc316c9aab9d77a756676b9d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:41:12 +0100 Subject: [PATCH 15/20] added warning comment --- start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start.py b/start.py index 93fff22e0c..5e0532e2a7 100644 --- a/start.py +++ b/start.py @@ -508,7 +508,7 @@ def boot(): # ------------------------------------------------------------------------ # Find Pype versions # ------------------------------------------------------------------------ - + # WARNING Environment PYPE_ROOT may change if frozen pype is executed if getattr(sys, 'frozen', False): # find versions of Pype to be used with frozen code try: From 61ef0cbb0d299f1e4d0e84485c62deb521a5f1d1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 12:41:23 +0100 Subject: [PATCH 16/20] pype modules are also popped --- start.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/start.py b/start.py index 5e0532e2a7..0679d8e39f 100644 --- a/start.py +++ b/start.py @@ -530,11 +530,12 @@ def boot(): os.environ["PYPE_REPOS_ROOT"] = os.path.join( os.environ["PYPE_ROOT"], "repos") - # delete Pype module from cache so it is used from specific version + # delete Pype module and it's submodules from cache so it is used from + # specific version modules_to_del = [] - for module_name in sys.modules: + for module_name in tuple(sys.modules): if module_name == "pype" or module_name.startswith("pype."): - modules_to_del.append(module_name) + modules_to_del.append(sys.modules.pop(module_name)) try: for module_name in modules_to_del: del sys.modules[module_name] From 1f42ebc7366ce0fa9377818d3f73ae66d4c08973 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 1 Feb 2021 13:02:24 +0100 Subject: [PATCH 17/20] standalone publisher has it's own cli command --- pype/cli.py | 6 ++++ pype/modules/standalonepublish_action.py | 10 +----- pype/pype_commands.py | 5 +++ pype/tools/standalonepublish/__init__.py | 14 ++++---- pype/tools/standalonepublish/__main__.py | 35 ------------------- pype/tools/standalonepublish/app.py | 44 ++++++++++++++++++++++-- 6 files changed, 62 insertions(+), 52 deletions(-) delete mode 100644 pype/tools/standalonepublish/__main__.py diff --git a/pype/cli.py b/pype/cli.py index cf1cd59cd1..137ae327b2 100644 --- a/pype/cli.py +++ b/pype/cli.py @@ -31,6 +31,12 @@ def settings(dev=False): PypeCommands().launch_settings_gui(dev) +@main.command() +def standalonepublisher(): + """Show Pype Standalone publisher UI.""" + PypeCommands().launch_standalone_publisher() + + @main.command() @click.option("-d", "--debug", is_flag=True, help=("Run pype tray in debug mode")) diff --git a/pype/modules/standalonepublish_action.py b/pype/modules/standalonepublish_action.py index 8dcc68113d..04f6fac87a 100644 --- a/pype/modules/standalonepublish_action.py +++ b/pype/modules/standalonepublish_action.py @@ -30,16 +30,8 @@ class StandAlonePublishAction(PypeModule, ITrayAction): self.publish_paths.extend(publish_paths) def run_standalone_publisher(self): - # TODO add command to cli.py - from pype import tools - standalone_publisher_tool_path = os.path.join( - os.path.dirname(os.path.abspath(tools.__file__)), - "standalonepublish" - ) args = [ *get_pype_execute_args(), - "run", - standalone_publisher_tool_path, - os.pathsep.join(self.publish_paths).replace("\\", "/") + "standalonepublisher" ] subprocess.Popen(args, creationflags=subprocess.DETACHED_PROCESS) diff --git a/pype/pype_commands.py b/pype/pype_commands.py index cc597f4beb..58a3fe738c 100644 --- a/pype/pype_commands.py +++ b/pype/pype_commands.py @@ -35,6 +35,11 @@ class PypeCommands: ) return run_event_server(*args) + @staticmethod + def launch_standalone_publisher(): + from pype.tools import standalonepublish + standalonepublish.main() + def publish(self, gui, paths): pass diff --git a/pype/tools/standalonepublish/__init__.py b/pype/tools/standalonepublish/__init__.py index 29a4e52904..d2ef73af00 100644 --- a/pype/tools/standalonepublish/__init__.py +++ b/pype/tools/standalonepublish/__init__.py @@ -1,8 +1,10 @@ from .app import ( - show, - cli + main, + Window +) + + +__all__ = ( + "main", + "Window" ) -__all__ = [ - "show", - "cli" -] diff --git a/pype/tools/standalonepublish/__main__.py b/pype/tools/standalonepublish/__main__.py deleted file mode 100644 index 85a574f8dc..0000000000 --- a/pype/tools/standalonepublish/__main__.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import sys -import app -import ctypes -import signal -from Qt import QtWidgets, QtGui -from avalon import style -from pype.api import resources - - -if __name__ == "__main__": - - # Allow to change icon of running process in windows taskbar - if os.name == "nt": - ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID( - u"standalonepublish" - ) - - qt_app = QtWidgets.QApplication([]) - # app.setQuitOnLastWindowClosed(False) - qt_app.setStyleSheet(style.load_stylesheet()) - icon = QtGui.QIcon(resources.pype_icon_filepath()) - qt_app.setWindowIcon(icon) - - def signal_handler(sig, frame): - print("You pressed Ctrl+C. Process ended.") - qt_app.quit() - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - window = app.Window(sys.argv[-1].split(os.pathsep)) - window.show() - - sys.exit(qt_app.exec_()) diff --git a/pype/tools/standalonepublish/app.py b/pype/tools/standalonepublish/app.py index a22dae32b9..920dd32f7c 100644 --- a/pype/tools/standalonepublish/app.py +++ b/pype/tools/standalonepublish/app.py @@ -1,7 +1,18 @@ +import os +import sys +import ctypes +import signal + from bson.objectid import ObjectId -from Qt import QtWidgets, QtCore -from widgets import AssetWidget, FamilyWidget, ComponentsWidget, ShadowWidget +from Qt import QtWidgets, QtCore, QtGui + +from .widgets import ( + AssetWidget, FamilyWidget, ComponentsWidget, ShadowWidget +) +from avalon import style +from pype.api import resources from avalon.api import AvalonMongoDB +from pype.modules import ModulesManager class Window(QtWidgets.QDialog): @@ -194,3 +205,32 @@ class Window(QtWidgets.QDialog): data.update(self.widget_components.collect_data()) return data + + +def main(): + # Allow to change icon of running process in windows taskbar + if os.name == "nt": + ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID( + u"standalonepublish" + ) + + qt_app = QtWidgets.QApplication([]) + # app.setQuitOnLastWindowClosed(False) + qt_app.setStyleSheet(style.load_stylesheet()) + icon = QtGui.QIcon(resources.pype_icon_filepath()) + qt_app.setWindowIcon(icon) + + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + qt_app.quit() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + modules_manager = ModulesManager() + module = modules_manager.modules_by_name["standalonepublish_tool"] + + window = Window(module.publish_paths) + window.show() + + sys.exit(qt_app.exec_()) From 42b12ed28fb9be56f5ecaab093530539d42c2e32 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 2 Feb 2021 10:38:08 +0100 Subject: [PATCH 18/20] get_pype_execute_args can accept any number of arguments that will be added after executable --- pype/lib/execute.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pype/lib/execute.py b/pype/lib/execute.py index 05357df6ae..7e37e5d6da 100644 --- a/pype/lib/execute.py +++ b/pype/lib/execute.py @@ -135,7 +135,7 @@ def run_subprocess(*args, **kwargs): return full_output -def get_pype_execute_args(): +def get_pype_execute_args(*args): """Arguments to run pype command. Arguments for subprocess when need to spawn new pype process. Which may be @@ -146,13 +146,20 @@ def get_pype_execute_args(): Pype executed from code has different executable set to virtual env python and must have path to script as first argument which is not needed for build pype. + + It is possible to pass any arguments that will be added after pype + executables. """ pype_executable = os.environ["PYPE_EXECUTABLE"] - args = [pype_executable] + pype_args = [pype_executable] executable_filename = os.path.basename(pype_executable) if "python" in executable_filename.lower(): - args.append( + pype_args.append( os.path.join(os.environ["PYPE_ROOT"], "start.py") ) - return args + + if args: + pype_args.extend(args) + + return pype_args From 052ca7f08685d036c81533c3998b442185abf296 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 2 Feb 2021 10:39:04 +0100 Subject: [PATCH 19/20] simplified usage of get_pype_execute_args --- .../ftrack/ftrack_server/event_server_cli.py | 6 +----- .../ftrack/ftrack_server/socket_thread.py | 21 +++++++------------ pype/modules/standalonepublish_action.py | 5 +---- pype/plugins/global/publish/extract_burnin.py | 11 +++------- .../widgets/widget_components.py | 11 ++-------- 5 files changed, 15 insertions(+), 39 deletions(-) diff --git a/pype/modules/ftrack/ftrack_server/event_server_cli.py b/pype/modules/ftrack/ftrack_server/event_server_cli.py index cc51304fc6..27b25bd8cf 100644 --- a/pype/modules/ftrack/ftrack_server/event_server_cli.py +++ b/pype/modules/ftrack/ftrack_server/event_server_cli.py @@ -132,11 +132,7 @@ def legacy_server(ftrack_url): if subproc is None: if subproc_failed_count < max_fail_count: - args = [ - *get_pype_execute_args(), - "run", - subproc_path - ] + args = get_pype_execute_args("run", subproc_path) subproc = subprocess.Popen( args, stdout=subprocess.PIPE diff --git a/pype/modules/ftrack/ftrack_server/socket_thread.py b/pype/modules/ftrack/ftrack_server/socket_thread.py index 51b0f6d305..a895e0b900 100644 --- a/pype/modules/ftrack/ftrack_server/socket_thread.py +++ b/pype/modules/ftrack/ftrack_server/socket_thread.py @@ -58,20 +58,15 @@ class SocketThread(threading.Thread): env = os.environ.copy() env["PYPE_PROCESS_MONGO_ID"] = str(Logger.mongo_process_id) - - self.subproc = subprocess.Popen( - [ - # Pype executable (with path to start script if not build) - *get_pype_execute_args(), - # Add `run` command - "run", - self.filepath, - *self.additional_args, - str(self.port) - ], - env=env, - stdin=subprocess.PIPE + # Pype executable (with path to start script if not build) + args = get_pype_execute_args( + # Add `run` command + "run", + self.filepath, + *self.additional_args, + str(self.port) ) + self.subproc = subprocess.Popen(args, env=env, stdin=subprocess.PIPE) # Listen for incoming connections sock.listen(1) diff --git a/pype/modules/standalonepublish_action.py b/pype/modules/standalonepublish_action.py index 04f6fac87a..3cfee67c85 100644 --- a/pype/modules/standalonepublish_action.py +++ b/pype/modules/standalonepublish_action.py @@ -30,8 +30,5 @@ class StandAlonePublishAction(PypeModule, ITrayAction): self.publish_paths.extend(publish_paths) def run_standalone_publisher(self): - args = [ - *get_pype_execute_args(), - "standalonepublisher" - ] + args = get_pype_execute_args("standalonepublisher") subprocess.Popen(args, creationflags=subprocess.DETACHED_PROCESS) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 1051ff1a59..f3276972e6 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -132,8 +132,7 @@ class ExtractBurnin(pype.api.Extractor): scriptpath = self.burnin_script_path() # Executable args that will execute the script # [pype executable, *pype script, "run"] - executable_args = get_pype_execute_args() - executable_args.append("run") + executable_args = get_pype_execute_args("run", scriptpath) # Environments for script process env = os.environ.copy() @@ -269,12 +268,8 @@ class ExtractBurnin(pype.api.Extractor): ) # Prepare subprocess arguments - args = [] - args.extend(executable_args) - args.extend([ - scriptpath, - temporary_json_filepath - ]) + args = list(executable_args) + args.append(temporary_json_filepath) self.log.debug("Executing: {}".format(" ".join(args))) # Run burnin script diff --git a/pype/tools/standalonepublish/widgets/widget_components.py b/pype/tools/standalonepublish/widgets/widget_components.py index 1ba282ab83..8d627b7eed 100644 --- a/pype/tools/standalonepublish/widgets/widget_components.py +++ b/pype/tools/standalonepublish/widgets/widget_components.py @@ -208,15 +208,8 @@ def cli_publish(data, publish_paths, gui=True): if data.get("family", "").lower() == "editorial": envcopy["PYBLISH_SUSPEND_LOGS"] = "1" - args = [ - *get_pype_execute_args(), - "run", - PUBLISH_SCRIPT_PATH - ] - result = execute( - args, - env=envcopy - ) + args = get_pype_execute_args("run", PUBLISH_SCRIPT_PATH) + result = execute(args, env=envcopy) result = {} if os.path.exists(json_data_path): From b8e0a7479d7625bb641f011f68cf1b80d966832d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 2 Feb 2021 10:40:09 +0100 Subject: [PATCH 20/20] added standalone publisher to silent commands --- start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/start.py b/start.py index 0679d8e39f..5a34bbc11a 100644 --- a/start.py +++ b/start.py @@ -116,7 +116,7 @@ from igniter.tools import load_environments # noqa: E402 from igniter.bootstrap_repos import PypeVersion # noqa: E402 bootstrap = BootstrapRepos() -silent_commands = ["run", "igniter"] +silent_commands = ["run", "igniter", "standalonepublisher"] def set_environments() -> None: