From d6d6fbeb988268ba7a10bd01e6b8d2fbcd063907 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 17:09:56 +0100 Subject: [PATCH 1/6] renamed `_subprocess` to `run_subprocess` and moved to `execute.py` --- pype/lib/__init__.py | 18 +++++----- pype/lib/applications.py | 65 ----------------------------------- pype/lib/execute.py | 73 +++++++++++++++++++++++++++------------- 3 files changed, 58 insertions(+), 98 deletions(-) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 09cc998b7c..39f08340fd 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -2,7 +2,10 @@ """Pype module API.""" from .terminal import Terminal -from .execute import execute +from .execute import ( + execute, + run_subprocess +) from .log import PypeLogger, timeit from .mongo import ( decompose_url, @@ -48,8 +51,7 @@ from .applications import ( ApplicationNotFound, ApplicationManager, PreLaunchHook, - PostLaunchHook, - _subprocess + PostLaunchHook ) from .plugin_tools import ( @@ -83,6 +85,9 @@ from .ffmpeg_utils import ( terminal = Terminal __all__ = [ + "execute", + "run_subprocess", + "env_value_to_bool", "get_paths_from_environ", @@ -98,9 +103,6 @@ __all__ = [ "get_latest_version", "BuildWorkfile", - "PypeHook", - "execute_hook", - "ApplicationLaunchFailed", "ApplictionExecutableNotFound", "ApplicationNotFound", @@ -124,8 +126,6 @@ __all__ = [ "ffprobe_streams", "get_ffmpeg_tool_path", - "_subprocess", - "terminal", "Anatomy", "get_datetime_data", @@ -134,7 +134,7 @@ __all__ = [ "get_presets", "get_init_presets", "update_dict", - "execute", + "PypeLogger", "decompose_url", "compose_url", diff --git a/pype/lib/applications.py b/pype/lib/applications.py index c7d6464418..dad2a417a9 100644 --- a/pype/lib/applications.py +++ b/pype/lib/applications.py @@ -67,71 +67,6 @@ class ApplicationLaunchFailed(Exception): pass -# Special naming case for subprocess since its a built-in method. -def _subprocess(*args, **kwargs): - """Convenience method for getting output errors for subprocess. - - Entered arguments and keyword arguments are passed to subprocess Popen. - - Args: - *args: Variable length arument list passed to Popen. - **kwargs : Arbitary keyword arguments passed to Popen. Is possible to - pass `logging.Logger` object under "logger" if want to use - different than lib's logger. - - Returns: - str: Full output of subprocess concatenated stdout and stderr. - - Raises: - RuntimeError: Exception is raised if process finished with nonzero - return code. - """ - - # Get environents from kwarg or use current process environments if were - # not passed. - env = kwargs.get("env") or os.environ - # Make sure environment contains only strings - filtered_env = {k: str(v) for k, v in env.items()} - - # Use lib's logger if was not passed with kwargs. - logger = kwargs.pop("logger", log) - - # set overrides - kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) - kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) - kwargs['env'] = filtered_env - - proc = subprocess.Popen(*args, **kwargs) - - full_output = "" - _stdout, _stderr = proc.communicate() - if _stdout: - _stdout = _stdout.decode("utf-8") - full_output += _stdout - logger.debug(_stdout) - - if _stderr: - _stderr = _stderr.decode("utf-8") - # Add additional line break if output already containt stdout - if full_output: - full_output += "\n" - full_output += _stderr - logger.warning(_stderr) - - if proc.returncode != 0: - exc_msg = "Executing arguments was not successful: \"{}\"".format(args) - if _stdout: - exc_msg += "\n\nOutput:\n{}".format(_stdout) - - if _stderr: - exc_msg += "Error:\n{}".format(_stderr) - - raise RuntimeError(exc_msg) - - return full_output - - class ApplicationManager: def __init__(self): self.log = PypeLogger().get_logger(self.__class__.__name__) diff --git a/pype/lib/execute.py b/pype/lib/execute.py index d7951df384..1f1adcdf23 100644 --- a/pype/lib/execute.py +++ b/pype/lib/execute.py @@ -69,42 +69,67 @@ def execute(args, return popen.returncode -def _subprocess(*args, **kwargs): +def run_subprocess(*args, **kwargs): """Convenience method for getting output errors for subprocess. - .. seealso:: :mod:`subprocess` + Output logged when process finish. + Entered arguments and keyword arguments are passed to subprocess Popen. + + Args: + *args: Variable length arument list passed to Popen. + **kwargs : Arbitary keyword arguments passed to Popen. Is possible to + pass `logging.Logger` object under "logger" if want to use + different than lib's logger. + + Returns: + str: Full output of subprocess concatenated stdout and stderr. + + Raises: + RuntimeError: Exception is raised if process finished with nonzero + return code. """ - # make sure environment contains only strings - if not kwargs.get("env"): - filtered_env = {k: str(v) for k, v in os.environ.items()} - else: - filtered_env = {k: str(v) for k, v in kwargs.get("env").items()} + + # Get environents from kwarg or use current process environments if were + # not passed. + env = kwargs.get("env") or os.environ + # Make sure environment contains only strings + filtered_env = {k: str(v) for k, v in env.items()} + + # Use lib's logger if was not passed with kwargs. + logger = kwargs.pop("logger", log) # set overrides kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.STDOUT) + kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) kwargs['env'] = filtered_env proc = subprocess.Popen(*args, **kwargs) - output, error = proc.communicate() + full_output = "" + _stdout, _stderr = proc.communicate() + if _stdout: + _stdout = _stdout.decode("utf-8") + full_output += _stdout + logger.debug(_stdout) - if output: - output = output.decode("utf-8") - output += "\n" - for line in output.strip().split("\n"): - log.info(line) - - if error: - error = error.decode("utf-8") - error += "\n" - for line in error.strip().split("\n"): - log.error(line) + if _stderr: + _stderr = _stderr.decode("utf-8") + # Add additional line break if output already containt stdout + if full_output: + full_output += "\n" + full_output += _stderr + logger.warning(_stderr) if proc.returncode != 0: - raise ValueError( - "\"{}\" was not successful:\nOutput: {}\nError: {}".format( - args, output, error)) - return output + exc_msg = "Executing arguments was not successful: \"{}\"".format(args) + if _stdout: + exc_msg += "\n\nOutput:\n{}".format(_stdout) + + if _stderr: + exc_msg += "Error:\n{}".format(_stderr) + + raise RuntimeError(exc_msg) + + return full_output From c7a650986a025dccfba25134da22b68b1e0caa14 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 17:11:00 +0100 Subject: [PATCH 2/6] plugin_tools is again using `run_subprocess` instead of `execute` --- pype/lib/plugin_tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py index 6d074329bc..4c96aaae8f 100644 --- a/pype/lib/plugin_tools.py +++ b/pype/lib/plugin_tools.py @@ -7,7 +7,7 @@ import re import json import tempfile -from . import execute +from .execute import run_subprocess, execute from pype.settings import get_project_settings @@ -204,7 +204,7 @@ def decompress(target_dir, file_url, log = logging.getLogger(__name__) log.debug("Decompressing {}".format(subprocess_exr)) - execute.execute( + run_subprocess( subprocess_exr, shell=True, logger=log ) @@ -242,7 +242,7 @@ def should_decompress(file_url): and we can decompress (oiiotool supported) """ if oiio_supported(): - output = execute.execute([ + output = run_subprocess([ os.getenv("PYPE_OIIO_PATH"), "--info", "-v", file_url]) return "compression: \"dwaa\"" in output or \ From 7c11ca59cfc29d62052a76ab329fda112072f78e Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 17:11:38 +0100 Subject: [PATCH 3/6] merged `pype.lib` imports in `pype.api` to one tuple --- pype/api.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/pype/api.py b/pype/api.py index a2b4f22e72..9c0d796128 100644 --- a/pype/api.py +++ b/pype/api.py @@ -9,7 +9,15 @@ from .lib import ( PypeLogger, Anatomy, config, - execute + execute, + run_subprocess, + version_up, + get_asset, + get_hierarchy, + get_version_from_path, + get_last_version_from_path, + source_hash, + get_latest_version ) from .lib.mongo import ( @@ -37,19 +45,6 @@ from .action import ( RepairContextAction ) -from .lib import ( - version_up, - get_asset, - get_hierarchy, - get_version_from_path, - get_last_version_from_path, - source_hash, - get_latest_version -) - -# Special naming case for subprocess since its a built-in method. -from .lib import _subprocess as subprocess - # for backward compatibility with Pype 2 Logger = PypeLogger @@ -94,6 +89,6 @@ __all__ = [ "get_last_version_from_path", "source_hash", - "subprocess", + "run_subprocess", "get_latest_version" ] From 2acb4d52c0e2a0a5aa5e191181d531884d6f8232 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 17:13:24 +0100 Subject: [PATCH 4/6] replaced usage of `run_subprocess` in code --- pype/hosts/tvpaint/hooks/pre_install_pywin.py | 4 ++-- pype/plugins/global/publish/extract_burnin.py | 4 +++- pype/plugins/global/publish/extract_jpeg.py | 4 +++- pype/plugins/global/publish/extract_review.py | 7 ++++--- pype/plugins/global/publish/extract_review_slate.py | 10 ++++++---- pype/plugins/global/publish/extract_scanline_exr.py | 2 +- pype/plugins/hiero/publish/extract_review_cutup.py | 7 +++---- pype/plugins/photoshop/publish/extract_review.py | 4 ++-- .../standalonepublisher/publish/collect_editorial.py | 2 +- .../publish/extract_trim_video_audio.py | 5 +++-- pype/tests/test_lib_restructuralization.py | 2 +- 11 files changed, 29 insertions(+), 22 deletions(-) diff --git a/pype/hosts/tvpaint/hooks/pre_install_pywin.py b/pype/hosts/tvpaint/hooks/pre_install_pywin.py index ca9242c4c8..7abab33757 100644 --- a/pype/hosts/tvpaint/hooks/pre_install_pywin.py +++ b/pype/hosts/tvpaint/hooks/pre_install_pywin.py @@ -1,7 +1,7 @@ from pype.lib import ( PreLaunchHook, ApplicationLaunchFailed, - _subprocess + run_subprocess ) @@ -25,7 +25,7 @@ class PreInstallPyWin(PreLaunchHook): return try: - output = _subprocess( + output = run_subprocess( ["pip", "install", "pywin32==227"] ) self.log.debug("Pip install pywin32 output:\n{}'".format(output)) diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index d29af63483..fc3adb81d5 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -264,7 +264,9 @@ class ExtractBurnin(pype.api.Extractor): self.log.debug("Executing: {}".format(subprcs_cmd)) # Run burnin script - pype.api.subprocess(subprcs_cmd, shell=True, logger=self.log) + pype.api.run_subprocess( + subprcs_cmd, shell=True, logger=self.log + ) # Remove the temporary json os.remove(temporary_json_filepath) diff --git a/pype/plugins/global/publish/extract_jpeg.py b/pype/plugins/global/publish/extract_jpeg.py index af90d4366d..cdacf79c6d 100644 --- a/pype/plugins/global/publish/extract_jpeg.py +++ b/pype/plugins/global/publish/extract_jpeg.py @@ -107,7 +107,9 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin): # run subprocess self.log.debug("{}".format(subprocess_jpeg)) try: # temporary until oiiotool is supported cross platform - pype.api.subprocess(subprocess_jpeg, shell=True) + pype.api.run_subprocess( + subprocess_jpeg, shell=True, logger=self.log + ) except RuntimeError as exp: if "Compression" in str(exp): self.log.debug("Unsupported compression on input files. " + diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 37fe83bf10..132e5ab91e 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -206,7 +206,7 @@ class ExtractReview(pyblish.api.InstancePlugin): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - pype.api.subprocess( + pype.api.run_subprocess( subprcs_cmd, shell=True, logger=self.log ) @@ -1628,8 +1628,9 @@ class ExtractReview(pyblish.api.InstancePlugin): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - output = pype.api.subprocess(subprcs_cmd, shell=True) - self.log.debug("Output: {}".format(output)) + output = pype.api.run_subprocess( + subprcs_cmd, shell=True, logger=self.log + ) # create representation data repre_new.update({ diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 5cf632406c..3614b1e056 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -186,8 +186,9 @@ class ExtractReviewSlate(pype.api.Extractor): # run slate generation subprocess self.log.debug("Slate Executing: {}".format(slate_subprcs_cmd)) - slate_output = pype.api.subprocess(slate_subprcs_cmd, shell=True) - self.log.debug("Slate Output: {}".format(slate_output)) + slate_output = pype.api.run_subprocess( + slate_subprcs_cmd, shell=True, logger=self.log + ) # create ffmpeg concat text file path conc_text_file = input_file.replace(ext, "") + "_concat" + ".txt" @@ -221,8 +222,9 @@ class ExtractReviewSlate(pype.api.Extractor): # ffmpeg concat subprocess self.log.debug("Executing concat: {}".format(concat_subprcs_cmd)) - concat_output = pype.api.subprocess(concat_subprcs_cmd, shell=True) - self.log.debug("Output concat: {}".format(concat_output)) + concat_output = pype.api.run_subprocess( + concat_subprcs_cmd, shell=True, logger=self.log + ) self.log.debug("__ repre[tags]: {}".format(repre["tags"])) repre_update = { diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index 9c3073d61d..a801baa17c 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -65,7 +65,7 @@ class ExtractScanlineExr(pyblish.api.InstancePlugin): subprocess_exr = " ".join(oiio_cmd) self.log.info(f"running: {subprocess_exr}") - pype.api.subprocess(subprocess_exr) + pype.api.run_subprocess(subprocess_exr, logger=self.log) # raise error if there is no ouptput if not os.path.exists(os.path.join(stagingdir, original_name)): diff --git a/pype/plugins/hiero/publish/extract_review_cutup.py b/pype/plugins/hiero/publish/extract_review_cutup.py index 87e584d0b0..ec3a9ec17e 100644 --- a/pype/plugins/hiero/publish/extract_review_cutup.py +++ b/pype/plugins/hiero/publish/extract_review_cutup.py @@ -142,7 +142,7 @@ class ExtractReviewCutUp(pype.api.Extractor): ).format(**locals()) self.log.debug("ffprob_cmd: {}".format(ffprob_cmd)) - audio_check_output = pype.api.subprocess(ffprob_cmd) + audio_check_output = pype.api.run_subprocess(ffprob_cmd) self.log.debug( "audio_check_output: {}".format(audio_check_output)) @@ -177,7 +177,7 @@ class ExtractReviewCutUp(pype.api.Extractor): # try to get video native resolution data try: - resolution_output = pype.api.subprocess(( + resolution_output = pype.api.run_subprocess(( "\"{ffprobe_path}\" -i \"{full_input_path}\"" " -v error " "-select_streams v:0 -show_entries " @@ -290,8 +290,7 @@ class ExtractReviewCutUp(pype.api.Extractor): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - output = pype.api.subprocess(subprcs_cmd) - self.log.debug("Output: {}".format(output)) + pype.api.run_subprocess(subprcs_cmd, logger=self.log) repre_new = { "files": new_files, diff --git a/pype/plugins/photoshop/publish/extract_review.py b/pype/plugins/photoshop/publish/extract_review.py index 2753cbf366..aa9151bf6d 100644 --- a/pype/plugins/photoshop/publish/extract_review.py +++ b/pype/plugins/photoshop/publish/extract_review.py @@ -60,7 +60,7 @@ class ExtractReview(pype.api.Extractor): "-vframes", "1", thumbnail_path ] - output = pype.lib._subprocess(args) + output = pype.lib.run_subprocess(args) instance.data["representations"].append({ "name": "thumbnail", @@ -78,7 +78,7 @@ class ExtractReview(pype.api.Extractor): "-vframes", "1", mov_path ] - output = pype.lib._subprocess(args) + output = pype.lib.run_subprocess(args) self.log.debug(output) instance.data["representations"].append({ "name": "mov", diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index 7e532c3741..afbdd87b2d 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -35,7 +35,7 @@ class OTIO_View(pyblish.api.Action): file_path = os.path.join( representation["stagingDir"], representation["files"] ) - plib._subprocess(["otioview", file_path]) + plib.run_subprocess(["otioview", file_path]) class CollectEditorial(pyblish.api.InstancePlugin): diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py index 193902a9f6..1c53ae5f46 100644 --- a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py +++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py @@ -80,8 +80,9 @@ class ExtractTrimVideoAudio(pype.api.Extractor): self.log.info(f"Processing: {args}") ffmpeg_args = " ".join(args) - output = pype.api.subprocess(ffmpeg_args, shell=True) - self.log.info(output) + pype.api.run_subprocess( + ffmpeg_args, shell=True, logger=self.log + ) repr = { "name": ext[1:], diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py index 957167a8bf..840baff5e6 100644 --- a/pype/tests/test_lib_restructuralization.py +++ b/pype/tests/test_lib_restructuralization.py @@ -29,7 +29,7 @@ def test_backward_compatibility(printer): from pype.hosts.fusion.lib import switch_item from pype.lib import source_hash - from pype.lib import _subprocess + from pype.lib import run_subprocess except ImportError as e: raise From 93432c1836a2f74f2bd2e351bf235f40d1b52a6a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 18:56:50 +0100 Subject: [PATCH 5/6] hound fixes --- pype/lib/plugin_tools.py | 2 +- pype/plugins/global/publish/extract_review.py | 2 +- pype/plugins/global/publish/extract_review_slate.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py index 4c96aaae8f..c39c9401c3 100644 --- a/pype/lib/plugin_tools.py +++ b/pype/lib/plugin_tools.py @@ -7,7 +7,7 @@ import re import json import tempfile -from .execute import run_subprocess, execute +from .execute import run_subprocess from pype.settings import get_project_settings diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 132e5ab91e..19c9f24c98 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -1628,7 +1628,7 @@ class ExtractReview(pyblish.api.InstancePlugin): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - output = pype.api.run_subprocess( + pype.api.run_subprocess( subprcs_cmd, shell=True, logger=self.log ) diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 3614b1e056..65930ea8fa 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -186,7 +186,7 @@ class ExtractReviewSlate(pype.api.Extractor): # run slate generation subprocess self.log.debug("Slate Executing: {}".format(slate_subprcs_cmd)) - slate_output = pype.api.run_subprocess( + pype.api.run_subprocess( slate_subprcs_cmd, shell=True, logger=self.log ) @@ -222,7 +222,7 @@ class ExtractReviewSlate(pype.api.Extractor): # ffmpeg concat subprocess self.log.debug("Executing concat: {}".format(concat_subprcs_cmd)) - concat_output = pype.api.run_subprocess( + pype.api.run_subprocess( concat_subprcs_cmd, shell=True, logger=self.log ) From 705998f1c179fb55061f841975955dffdc567444 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 22 Dec 2020 18:57:31 +0100 Subject: [PATCH 6/6] removed vendor folder from pype's root --- vendor/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 vendor/README.md diff --git a/vendor/README.md b/vendor/README.md deleted file mode 100644 index e69de29bb2..0000000000