diff --git a/pype/api.py b/pype/api.py index a2b4f22e72..9c0d796128 100644 --- a/pype/api.py +++ b/pype/api.py @@ -9,7 +9,15 @@ from .lib import ( PypeLogger, Anatomy, config, - execute + execute, + run_subprocess, + version_up, + get_asset, + get_hierarchy, + get_version_from_path, + get_last_version_from_path, + source_hash, + get_latest_version ) from .lib.mongo import ( @@ -37,19 +45,6 @@ from .action import ( RepairContextAction ) -from .lib import ( - version_up, - get_asset, - get_hierarchy, - get_version_from_path, - get_last_version_from_path, - source_hash, - get_latest_version -) - -# Special naming case for subprocess since its a built-in method. -from .lib import _subprocess as subprocess - # for backward compatibility with Pype 2 Logger = PypeLogger @@ -94,6 +89,6 @@ __all__ = [ "get_last_version_from_path", "source_hash", - "subprocess", + "run_subprocess", "get_latest_version" ] diff --git a/pype/hosts/tvpaint/hooks/pre_install_pywin.py b/pype/hosts/tvpaint/hooks/pre_install_pywin.py index ca9242c4c8..7abab33757 100644 --- a/pype/hosts/tvpaint/hooks/pre_install_pywin.py +++ b/pype/hosts/tvpaint/hooks/pre_install_pywin.py @@ -1,7 +1,7 @@ from pype.lib import ( PreLaunchHook, ApplicationLaunchFailed, - _subprocess + run_subprocess ) @@ -25,7 +25,7 @@ class PreInstallPyWin(PreLaunchHook): return try: - output = _subprocess( + output = run_subprocess( ["pip", "install", "pywin32==227"] ) self.log.debug("Pip install pywin32 output:\n{}'".format(output)) diff --git a/pype/lib/__init__.py b/pype/lib/__init__.py index 7e33577f3e..691c105b76 100644 --- a/pype/lib/__init__.py +++ b/pype/lib/__init__.py @@ -2,7 +2,10 @@ """Pype module API.""" from .terminal import Terminal -from .execute import execute +from .execute import ( + execute, + run_subprocess +) from .log import PypeLogger, timeit from .mongo import ( decompose_url, @@ -53,8 +56,7 @@ from .applications import ( ApplicationNotFound, ApplicationManager, PreLaunchHook, - PostLaunchHook, - _subprocess + PostLaunchHook ) from .plugin_tools import ( @@ -99,6 +101,9 @@ from .editorial import ( terminal = Terminal __all__ = [ + "execute", + "run_subprocess", + "env_value_to_bool", "get_paths_from_environ", @@ -146,8 +151,6 @@ __all__ = [ "ffprobe_streams", "get_ffmpeg_tool_path", - "_subprocess", - "terminal", "merge_dict", @@ -155,7 +158,6 @@ __all__ = [ "get_datetime_data", - "execute", "PypeLogger", "decompose_url", "compose_url", diff --git a/pype/lib/applications.py b/pype/lib/applications.py index 9f43ae5542..a7697a889c 100644 --- a/pype/lib/applications.py +++ b/pype/lib/applications.py @@ -64,71 +64,6 @@ class ApplicationLaunchFailed(Exception): pass -# Special naming case for subprocess since its a built-in method. -def _subprocess(*args, **kwargs): - """Convenience method for getting output errors for subprocess. - - Entered arguments and keyword arguments are passed to subprocess Popen. - - Args: - *args: Variable length arument list passed to Popen. - **kwargs : Arbitary keyword arguments passed to Popen. Is possible to - pass `logging.Logger` object under "logger" if want to use - different than lib's logger. - - Returns: - str: Full output of subprocess concatenated stdout and stderr. - - Raises: - RuntimeError: Exception is raised if process finished with nonzero - return code. - """ - - # Get environents from kwarg or use current process environments if were - # not passed. - env = kwargs.get("env") or os.environ - # Make sure environment contains only strings - filtered_env = {k: str(v) for k, v in env.items()} - - # Use lib's logger if was not passed with kwargs. - logger = kwargs.pop("logger", log) - - # set overrides - kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) - kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) - kwargs['env'] = filtered_env - - proc = subprocess.Popen(*args, **kwargs) - - full_output = "" - _stdout, _stderr = proc.communicate() - if _stdout: - _stdout = _stdout.decode("utf-8") - full_output += _stdout - logger.debug(_stdout) - - if _stderr: - _stderr = _stderr.decode("utf-8") - # Add additional line break if output already containt stdout - if full_output: - full_output += "\n" - full_output += _stderr - logger.warning(_stderr) - - if proc.returncode != 0: - exc_msg = "Executing arguments was not successful: \"{}\"".format(args) - if _stdout: - exc_msg += "\n\nOutput:\n{}".format(_stdout) - - if _stderr: - exc_msg += "Error:\n{}".format(_stderr) - - raise RuntimeError(exc_msg) - - return full_output - - class ApplicationManager: def __init__(self): self.log = PypeLogger().get_logger(self.__class__.__name__) diff --git a/pype/lib/execute.py b/pype/lib/execute.py index d7951df384..1f1adcdf23 100644 --- a/pype/lib/execute.py +++ b/pype/lib/execute.py @@ -69,42 +69,67 @@ def execute(args, return popen.returncode -def _subprocess(*args, **kwargs): +def run_subprocess(*args, **kwargs): """Convenience method for getting output errors for subprocess. - .. seealso:: :mod:`subprocess` + Output logged when process finish. + Entered arguments and keyword arguments are passed to subprocess Popen. + + Args: + *args: Variable length arument list passed to Popen. + **kwargs : Arbitary keyword arguments passed to Popen. Is possible to + pass `logging.Logger` object under "logger" if want to use + different than lib's logger. + + Returns: + str: Full output of subprocess concatenated stdout and stderr. + + Raises: + RuntimeError: Exception is raised if process finished with nonzero + return code. """ - # make sure environment contains only strings - if not kwargs.get("env"): - filtered_env = {k: str(v) for k, v in os.environ.items()} - else: - filtered_env = {k: str(v) for k, v in kwargs.get("env").items()} + + # Get environents from kwarg or use current process environments if were + # not passed. + env = kwargs.get("env") or os.environ + # Make sure environment contains only strings + filtered_env = {k: str(v) for k, v in env.items()} + + # Use lib's logger if was not passed with kwargs. + logger = kwargs.pop("logger", log) # set overrides kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE) - kwargs['stderr'] = kwargs.get('stderr', subprocess.STDOUT) + kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE) kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE) kwargs['env'] = filtered_env proc = subprocess.Popen(*args, **kwargs) - output, error = proc.communicate() + full_output = "" + _stdout, _stderr = proc.communicate() + if _stdout: + _stdout = _stdout.decode("utf-8") + full_output += _stdout + logger.debug(_stdout) - if output: - output = output.decode("utf-8") - output += "\n" - for line in output.strip().split("\n"): - log.info(line) - - if error: - error = error.decode("utf-8") - error += "\n" - for line in error.strip().split("\n"): - log.error(line) + if _stderr: + _stderr = _stderr.decode("utf-8") + # Add additional line break if output already containt stdout + if full_output: + full_output += "\n" + full_output += _stderr + logger.warning(_stderr) if proc.returncode != 0: - raise ValueError( - "\"{}\" was not successful:\nOutput: {}\nError: {}".format( - args, output, error)) - return output + exc_msg = "Executing arguments was not successful: \"{}\"".format(args) + if _stdout: + exc_msg += "\n\nOutput:\n{}".format(_stdout) + + if _stderr: + exc_msg += "Error:\n{}".format(_stderr) + + raise RuntimeError(exc_msg) + + return full_output diff --git a/pype/lib/plugin_tools.py b/pype/lib/plugin_tools.py index 6d074329bc..c39c9401c3 100644 --- a/pype/lib/plugin_tools.py +++ b/pype/lib/plugin_tools.py @@ -7,7 +7,7 @@ import re import json import tempfile -from . import execute +from .execute import run_subprocess from pype.settings import get_project_settings @@ -204,7 +204,7 @@ def decompress(target_dir, file_url, log = logging.getLogger(__name__) log.debug("Decompressing {}".format(subprocess_exr)) - execute.execute( + run_subprocess( subprocess_exr, shell=True, logger=log ) @@ -242,7 +242,7 @@ def should_decompress(file_url): and we can decompress (oiiotool supported) """ if oiio_supported(): - output = execute.execute([ + output = run_subprocess([ os.getenv("PYPE_OIIO_PATH"), "--info", "-v", file_url]) return "compression: \"dwaa\"" in output or \ diff --git a/pype/plugins/global/publish/extract_burnin.py b/pype/plugins/global/publish/extract_burnin.py index 28dd6730cb..c14dfba50a 100644 --- a/pype/plugins/global/publish/extract_burnin.py +++ b/pype/plugins/global/publish/extract_burnin.py @@ -265,7 +265,9 @@ class ExtractBurnin(pype.api.Extractor): self.log.debug("Executing: {}".format(subprcs_cmd)) # Run burnin script - pype.api.subprocess(subprcs_cmd, shell=True, logger=self.log) + pype.api.run_subprocess( + subprcs_cmd, shell=True, logger=self.log + ) # Remove the temporary json os.remove(temporary_json_filepath) diff --git a/pype/plugins/global/publish/extract_jpeg.py b/pype/plugins/global/publish/extract_jpeg.py index d5e9a896dc..1c921a90d4 100644 --- a/pype/plugins/global/publish/extract_jpeg.py +++ b/pype/plugins/global/publish/extract_jpeg.py @@ -111,7 +111,9 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin): # run subprocess self.log.debug("{}".format(subprocess_jpeg)) try: # temporary until oiiotool is supported cross platform - pype.api.subprocess(subprocess_jpeg, shell=True) + pype.api.run_subprocess( + subprocess_jpeg, shell=True, logger=self.log + ) except RuntimeError as exp: if "Compression" in str(exp): self.log.debug("Unsupported compression on input files. " + diff --git a/pype/plugins/global/publish/extract_review.py b/pype/plugins/global/publish/extract_review.py index 9b8c097dca..5414ae5289 100644 --- a/pype/plugins/global/publish/extract_review.py +++ b/pype/plugins/global/publish/extract_review.py @@ -207,7 +207,7 @@ class ExtractReview(pyblish.api.InstancePlugin): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - pype.api.subprocess( + pype.api.run_subprocess( subprcs_cmd, shell=True, logger=self.log ) @@ -1629,8 +1629,9 @@ class ExtractReview(pyblish.api.InstancePlugin): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - output = pype.api.subprocess(subprcs_cmd, shell=True) - self.log.debug("Output: {}".format(output)) + pype.api.run_subprocess( + subprcs_cmd, shell=True, logger=self.log + ) # create representation data repre_new.update({ diff --git a/pype/plugins/global/publish/extract_review_slate.py b/pype/plugins/global/publish/extract_review_slate.py index 5cf632406c..65930ea8fa 100644 --- a/pype/plugins/global/publish/extract_review_slate.py +++ b/pype/plugins/global/publish/extract_review_slate.py @@ -186,8 +186,9 @@ class ExtractReviewSlate(pype.api.Extractor): # run slate generation subprocess self.log.debug("Slate Executing: {}".format(slate_subprcs_cmd)) - slate_output = pype.api.subprocess(slate_subprcs_cmd, shell=True) - self.log.debug("Slate Output: {}".format(slate_output)) + pype.api.run_subprocess( + slate_subprcs_cmd, shell=True, logger=self.log + ) # create ffmpeg concat text file path conc_text_file = input_file.replace(ext, "") + "_concat" + ".txt" @@ -221,8 +222,9 @@ class ExtractReviewSlate(pype.api.Extractor): # ffmpeg concat subprocess self.log.debug("Executing concat: {}".format(concat_subprcs_cmd)) - concat_output = pype.api.subprocess(concat_subprcs_cmd, shell=True) - self.log.debug("Output concat: {}".format(concat_output)) + pype.api.run_subprocess( + concat_subprcs_cmd, shell=True, logger=self.log + ) self.log.debug("__ repre[tags]: {}".format(repre["tags"])) repre_update = { diff --git a/pype/plugins/global/publish/extract_scanline_exr.py b/pype/plugins/global/publish/extract_scanline_exr.py index 9c3073d61d..a801baa17c 100644 --- a/pype/plugins/global/publish/extract_scanline_exr.py +++ b/pype/plugins/global/publish/extract_scanline_exr.py @@ -65,7 +65,7 @@ class ExtractScanlineExr(pyblish.api.InstancePlugin): subprocess_exr = " ".join(oiio_cmd) self.log.info(f"running: {subprocess_exr}") - pype.api.subprocess(subprocess_exr) + pype.api.run_subprocess(subprocess_exr, logger=self.log) # raise error if there is no ouptput if not os.path.exists(os.path.join(stagingdir, original_name)): diff --git a/pype/plugins/hiero/publish/extract_review_cutup.py b/pype/plugins/hiero/publish/extract_review_cutup.py index 87e584d0b0..ec3a9ec17e 100644 --- a/pype/plugins/hiero/publish/extract_review_cutup.py +++ b/pype/plugins/hiero/publish/extract_review_cutup.py @@ -142,7 +142,7 @@ class ExtractReviewCutUp(pype.api.Extractor): ).format(**locals()) self.log.debug("ffprob_cmd: {}".format(ffprob_cmd)) - audio_check_output = pype.api.subprocess(ffprob_cmd) + audio_check_output = pype.api.run_subprocess(ffprob_cmd) self.log.debug( "audio_check_output: {}".format(audio_check_output)) @@ -177,7 +177,7 @@ class ExtractReviewCutUp(pype.api.Extractor): # try to get video native resolution data try: - resolution_output = pype.api.subprocess(( + resolution_output = pype.api.run_subprocess(( "\"{ffprobe_path}\" -i \"{full_input_path}\"" " -v error " "-select_streams v:0 -show_entries " @@ -290,8 +290,7 @@ class ExtractReviewCutUp(pype.api.Extractor): # run subprocess self.log.debug("Executing: {}".format(subprcs_cmd)) - output = pype.api.subprocess(subprcs_cmd) - self.log.debug("Output: {}".format(output)) + pype.api.run_subprocess(subprcs_cmd, logger=self.log) repre_new = { "files": new_files, diff --git a/pype/plugins/photoshop/publish/extract_review.py b/pype/plugins/photoshop/publish/extract_review.py index 2753cbf366..aa9151bf6d 100644 --- a/pype/plugins/photoshop/publish/extract_review.py +++ b/pype/plugins/photoshop/publish/extract_review.py @@ -60,7 +60,7 @@ class ExtractReview(pype.api.Extractor): "-vframes", "1", thumbnail_path ] - output = pype.lib._subprocess(args) + output = pype.lib.run_subprocess(args) instance.data["representations"].append({ "name": "thumbnail", @@ -78,7 +78,7 @@ class ExtractReview(pype.api.Extractor): "-vframes", "1", mov_path ] - output = pype.lib._subprocess(args) + output = pype.lib.run_subprocess(args) self.log.debug(output) instance.data["representations"].append({ "name": "mov", diff --git a/pype/plugins/standalonepublisher/publish/collect_editorial.py b/pype/plugins/standalonepublisher/publish/collect_editorial.py index 7e532c3741..afbdd87b2d 100644 --- a/pype/plugins/standalonepublisher/publish/collect_editorial.py +++ b/pype/plugins/standalonepublisher/publish/collect_editorial.py @@ -35,7 +35,7 @@ class OTIO_View(pyblish.api.Action): file_path = os.path.join( representation["stagingDir"], representation["files"] ) - plib._subprocess(["otioview", file_path]) + plib.run_subprocess(["otioview", file_path]) class CollectEditorial(pyblish.api.InstancePlugin): diff --git a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py index 193902a9f6..1c53ae5f46 100644 --- a/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py +++ b/pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py @@ -80,8 +80,9 @@ class ExtractTrimVideoAudio(pype.api.Extractor): self.log.info(f"Processing: {args}") ffmpeg_args = " ".join(args) - output = pype.api.subprocess(ffmpeg_args, shell=True) - self.log.info(output) + pype.api.run_subprocess( + ffmpeg_args, shell=True, logger=self.log + ) repr = { "name": ext[1:], diff --git a/pype/tests/test_lib_restructuralization.py b/pype/tests/test_lib_restructuralization.py index 957167a8bf..840baff5e6 100644 --- a/pype/tests/test_lib_restructuralization.py +++ b/pype/tests/test_lib_restructuralization.py @@ -29,7 +29,7 @@ def test_backward_compatibility(printer): from pype.hosts.fusion.lib import switch_item from pype.lib import source_hash - from pype.lib import _subprocess + from pype.lib import run_subprocess except ImportError as e: raise diff --git a/vendor/README.md b/vendor/README.md deleted file mode 100644 index e69de29bb2..0000000000