Merge pull request #858 from pypeclub/feature/get_rid_of_subprocess

Renamed pype.lib._subprocess

pype/api.py

@@ -9,7 +9,15 @@ from .lib import (
     PypeLogger,
     Anatomy,
     config,
-    execute
+    execute,
+    run_subprocess,
+    version_up,
+    get_asset,
+    get_hierarchy,
+    get_version_from_path,
+    get_last_version_from_path,
+    source_hash,
+    get_latest_version
 )
 
 from .lib.mongo import (
@@ -37,19 +45,6 @@ from .action import (
     RepairContextAction
 )
 
-from .lib import (
-    version_up,
-    get_asset,
-    get_hierarchy,
-    get_version_from_path,
-    get_last_version_from_path,
-    source_hash,
-    get_latest_version
-)
-
-# Special naming case for subprocess since its a built-in method.
-from .lib import _subprocess as subprocess
-
 # for backward compatibility with Pype 2
 Logger = PypeLogger
 
@@ -94,6 +89,6 @@ __all__ = [
     "get_last_version_from_path",
     "source_hash",
 
-    "subprocess",
+    "run_subprocess",
     "get_latest_version"
 ]

pype/hosts/tvpaint/hooks/pre_install_pywin.py

@@ -1,7 +1,7 @@
 from pype.lib import (
     PreLaunchHook,
     ApplicationLaunchFailed,
-    _subprocess
+    run_subprocess
 )
 
 
@@ -25,7 +25,7 @@ class PreInstallPyWin(PreLaunchHook):
             return
 
         try:
-            output = _subprocess(
+            output = run_subprocess(
                 ["pip", "install", "pywin32==227"]
             )
             self.log.debug("Pip install pywin32 output:\n{}'".format(output))

pype/lib/__init__.py

@@ -2,7 +2,10 @@
 """Pype module API."""
 
 from .terminal import Terminal
-from .execute import execute
+from .execute import (
+    execute,
+    run_subprocess
+)
 from .log import PypeLogger, timeit
 from .mongo import (
     decompose_url,
@@ -53,8 +56,7 @@ from .applications import (
     ApplicationNotFound,
     ApplicationManager,
     PreLaunchHook,
-    PostLaunchHook,
-    _subprocess
+    PostLaunchHook
 )
 
 from .plugin_tools import (
@@ -99,6 +101,9 @@ from .editorial import (
 terminal = Terminal
 
 __all__ = [
+    "execute",
+    "run_subprocess",
+
     "env_value_to_bool",
     "get_paths_from_environ",
 
@@ -146,8 +151,6 @@ __all__ = [
     "ffprobe_streams",
     "get_ffmpeg_tool_path",
 
-    "_subprocess",
-
     "terminal",
 
     "merge_dict",
@@ -155,7 +158,6 @@ __all__ = [
 
     "get_datetime_data",
 
-    "execute",
     "PypeLogger",
     "decompose_url",
     "compose_url",

pype/lib/applications.py

@@ -64,71 +64,6 @@ class ApplicationLaunchFailed(Exception):
     pass
 
 
-# Special naming case for subprocess since its a built-in method.
-def _subprocess(*args, **kwargs):
-    """Convenience method for getting output errors for subprocess.
-
-    Entered arguments and keyword arguments are passed to subprocess Popen.
-
-    Args:
-        *args: Variable length arument list passed to Popen.
-        **kwargs : Arbitary keyword arguments passed to Popen. Is possible to
-            pass `logging.Logger` object under "logger" if want to use
-            different than lib's logger.
-
-    Returns:
-        str: Full output of subprocess concatenated stdout and stderr.
-
-    Raises:
-        RuntimeError: Exception is raised if process finished with nonzero
-            return code.
-    """
-
-    # Get environents from kwarg or use current process environments if were
-    # not passed.
-    env = kwargs.get("env") or os.environ
-    # Make sure environment contains only strings
-    filtered_env = {k: str(v) for k, v in env.items()}
-
-    # Use lib's logger if was not passed with kwargs.
-    logger = kwargs.pop("logger", log)
-
-    # set overrides
-    kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE)
-    kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE)
-    kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE)
-    kwargs['env'] = filtered_env
-
-    proc = subprocess.Popen(*args, **kwargs)
-
-    full_output = ""
-    _stdout, _stderr = proc.communicate()
-    if _stdout:
-        _stdout = _stdout.decode("utf-8")
-        full_output += _stdout
-        logger.debug(_stdout)
-
-    if _stderr:
-        _stderr = _stderr.decode("utf-8")
-        # Add additional line break if output already containt stdout
-        if full_output:
-            full_output += "\n"
-        full_output += _stderr
-        logger.warning(_stderr)
-
-    if proc.returncode != 0:
-        exc_msg = "Executing arguments was not successful: \"{}\"".format(args)
-        if _stdout:
-            exc_msg += "\n\nOutput:\n{}".format(_stdout)
-
-        if _stderr:
-            exc_msg += "Error:\n{}".format(_stderr)
-
-        raise RuntimeError(exc_msg)
-
-    return full_output
-
-
 class ApplicationManager:
     def __init__(self):
         self.log = PypeLogger().get_logger(self.__class__.__name__)

pype/lib/execute.py

@@ -69,42 +69,67 @@ def execute(args,
     return popen.returncode
 
 
-def _subprocess(*args, **kwargs):
+def run_subprocess(*args, **kwargs):
     """Convenience method for getting output errors for subprocess.
 
-    .. seealso:: :mod:`subprocess`
+    Output logged when process finish.
 
+    Entered arguments and keyword arguments are passed to subprocess Popen.
+
+    Args:
+        *args: Variable length arument list passed to Popen.
+        **kwargs : Arbitary keyword arguments passed to Popen. Is possible to
+            pass `logging.Logger` object under "logger" if want to use
+            different than lib's logger.
+
+    Returns:
+        str: Full output of subprocess concatenated stdout and stderr.
+
+    Raises:
+        RuntimeError: Exception is raised if process finished with nonzero
+            return code.
     """
-    # make sure environment contains only strings
-    if not kwargs.get("env"):
-        filtered_env = {k: str(v) for k, v in os.environ.items()}
-    else:
-        filtered_env = {k: str(v) for k, v in kwargs.get("env").items()}
+
+    # Get environents from kwarg or use current process environments if were
+    # not passed.
+    env = kwargs.get("env") or os.environ
+    # Make sure environment contains only strings
+    filtered_env = {k: str(v) for k, v in env.items()}
+
+    # Use lib's logger if was not passed with kwargs.
+    logger = kwargs.pop("logger", log)
 
     # set overrides
     kwargs['stdout'] = kwargs.get('stdout', subprocess.PIPE)
-    kwargs['stderr'] = kwargs.get('stderr', subprocess.STDOUT)
+    kwargs['stderr'] = kwargs.get('stderr', subprocess.PIPE)
     kwargs['stdin'] = kwargs.get('stdin', subprocess.PIPE)
     kwargs['env'] = filtered_env
 
     proc = subprocess.Popen(*args, **kwargs)
 
-    output, error = proc.communicate()
+    full_output = ""
+    _stdout, _stderr = proc.communicate()
+    if _stdout:
+        _stdout = _stdout.decode("utf-8")
+        full_output += _stdout
+        logger.debug(_stdout)
 
-    if output:
-        output = output.decode("utf-8")
-        output += "\n"
-        for line in output.strip().split("\n"):
-            log.info(line)
-
-    if error:
-        error = error.decode("utf-8")
-        error += "\n"
-        for line in error.strip().split("\n"):
-            log.error(line)
+    if _stderr:
+        _stderr = _stderr.decode("utf-8")
+        # Add additional line break if output already containt stdout
+        if full_output:
+            full_output += "\n"
+        full_output += _stderr
+        logger.warning(_stderr)
 
     if proc.returncode != 0:
-        raise ValueError(
-            "\"{}\" was not successful:\nOutput: {}\nError: {}".format(
-                args, output, error))
-    return output
+        exc_msg = "Executing arguments was not successful: \"{}\"".format(args)
+        if _stdout:
+            exc_msg += "\n\nOutput:\n{}".format(_stdout)
+
+        if _stderr:
+            exc_msg += "Error:\n{}".format(_stderr)
+
+        raise RuntimeError(exc_msg)
+
+    return full_output

pype/lib/plugin_tools.py

@@ -7,7 +7,7 @@ import re
 import json
 import tempfile
 
-from . import execute
+from .execute import run_subprocess
 
 from pype.settings import get_project_settings
 
@@ -204,7 +204,7 @@ def decompress(target_dir, file_url,
         log = logging.getLogger(__name__)
 
     log.debug("Decompressing {}".format(subprocess_exr))
-    execute.execute(
+    run_subprocess(
         subprocess_exr, shell=True, logger=log
     )
 
@@ -242,7 +242,7 @@ def should_decompress(file_url):
                 and we can decompress (oiiotool supported)
     """
     if oiio_supported():
-        output = execute.execute([
+        output = run_subprocess([
             os.getenv("PYPE_OIIO_PATH"),
             "--info", "-v", file_url])
         return "compression: \"dwaa\"" in output or \

pype/plugins/global/publish/extract_burnin.py

@@ -265,7 +265,9 @@ class ExtractBurnin(pype.api.Extractor):
                 self.log.debug("Executing: {}".format(subprcs_cmd))
 
                 # Run burnin script
-                pype.api.subprocess(subprcs_cmd, shell=True, logger=self.log)
+                pype.api.run_subprocess(
+                    subprcs_cmd, shell=True, logger=self.log
+                )
 
                 # Remove the temporary json
                 os.remove(temporary_json_filepath)

pype/plugins/global/publish/extract_jpeg.py

@@ -111,7 +111,9 @@ class ExtractJpegEXR(pyblish.api.InstancePlugin):
             # run subprocess
             self.log.debug("{}".format(subprocess_jpeg))
             try:  # temporary until oiiotool is supported cross platform
-                pype.api.subprocess(subprocess_jpeg, shell=True)
+                pype.api.run_subprocess(
+                    subprocess_jpeg, shell=True, logger=self.log
+                )
             except RuntimeError as exp:
                 if "Compression" in str(exp):
                     self.log.debug("Unsupported compression on input files. " +

pype/plugins/global/publish/extract_review.py

@@ -207,7 +207,7 @@ class ExtractReview(pyblish.api.InstancePlugin):
                 # run subprocess
                 self.log.debug("Executing: {}".format(subprcs_cmd))
 
-                pype.api.subprocess(
+                pype.api.run_subprocess(
                     subprcs_cmd, shell=True, logger=self.log
                 )
 
@@ -1629,8 +1629,9 @@ class ExtractReview(pyblish.api.InstancePlugin):
 
                 # run subprocess
                 self.log.debug("Executing: {}".format(subprcs_cmd))
-                output = pype.api.subprocess(subprcs_cmd, shell=True)
-                self.log.debug("Output: {}".format(output))
+                pype.api.run_subprocess(
+                    subprcs_cmd, shell=True, logger=self.log
+                )
 
                 # create representation data
                 repre_new.update({

pype/plugins/global/publish/extract_review_slate.py

@@ -186,8 +186,9 @@ class ExtractReviewSlate(pype.api.Extractor):
 
             # run slate generation subprocess
             self.log.debug("Slate Executing: {}".format(slate_subprcs_cmd))
-            slate_output = pype.api.subprocess(slate_subprcs_cmd, shell=True)
-            self.log.debug("Slate Output: {}".format(slate_output))
+            pype.api.run_subprocess(
+                slate_subprcs_cmd, shell=True, logger=self.log
+            )
 
             # create ffmpeg concat text file path
             conc_text_file = input_file.replace(ext, "") + "_concat" + ".txt"
@@ -221,8 +222,9 @@ class ExtractReviewSlate(pype.api.Extractor):
 
             # ffmpeg concat subprocess
             self.log.debug("Executing concat: {}".format(concat_subprcs_cmd))
-            concat_output = pype.api.subprocess(concat_subprcs_cmd, shell=True)
-            self.log.debug("Output concat: {}".format(concat_output))
+            pype.api.run_subprocess(
+                concat_subprcs_cmd, shell=True, logger=self.log
+            )
 
             self.log.debug("__ repre[tags]: {}".format(repre["tags"]))
             repre_update = {

pype/plugins/global/publish/extract_scanline_exr.py

@@ -65,7 +65,7 @@ class ExtractScanlineExr(pyblish.api.InstancePlugin):
 
                 subprocess_exr = " ".join(oiio_cmd)
                 self.log.info(f"running: {subprocess_exr}")
-                pype.api.subprocess(subprocess_exr)
+                pype.api.run_subprocess(subprocess_exr, logger=self.log)
 
                 # raise error if there is no ouptput
                 if not os.path.exists(os.path.join(stagingdir, original_name)):

pype/plugins/hiero/publish/extract_review_cutup.py

@@ -142,7 +142,7 @@ class ExtractReviewCutUp(pype.api.Extractor):
                 ).format(**locals())
 
                 self.log.debug("ffprob_cmd: {}".format(ffprob_cmd))
-                audio_check_output = pype.api.subprocess(ffprob_cmd)
+                audio_check_output = pype.api.run_subprocess(ffprob_cmd)
                 self.log.debug(
                     "audio_check_output: {}".format(audio_check_output))
 
@@ -177,7 +177,7 @@ class ExtractReviewCutUp(pype.api.Extractor):
 
                     # try to get video native resolution data
                     try:
-                        resolution_output = pype.api.subprocess((
+                        resolution_output = pype.api.run_subprocess((
                             "\"{ffprobe_path}\" -i \"{full_input_path}\""
                             " -v error "
                             "-select_streams v:0 -show_entries "
@@ -290,8 +290,7 @@ class ExtractReviewCutUp(pype.api.Extractor):
 
                 # run subprocess
                 self.log.debug("Executing: {}".format(subprcs_cmd))
-                output = pype.api.subprocess(subprcs_cmd)
-                self.log.debug("Output: {}".format(output))
+                pype.api.run_subprocess(subprcs_cmd, logger=self.log)
 
             repre_new = {
                 "files": new_files,

pype/plugins/photoshop/publish/extract_review.py

@@ -60,7 +60,7 @@ class ExtractReview(pype.api.Extractor):
             "-vframes", "1",
             thumbnail_path
         ]
-        output = pype.lib._subprocess(args)
+        output = pype.lib.run_subprocess(args)
 
         instance.data["representations"].append({
             "name": "thumbnail",
@@ -78,7 +78,7 @@ class ExtractReview(pype.api.Extractor):
             "-vframes", "1",
             mov_path
         ]
-        output = pype.lib._subprocess(args)
+        output = pype.lib.run_subprocess(args)
         self.log.debug(output)
         instance.data["representations"].append({
             "name": "mov",

pype/plugins/standalonepublisher/publish/collect_editorial.py

@@ -35,7 +35,7 @@ class OTIO_View(pyblish.api.Action):
         file_path = os.path.join(
             representation["stagingDir"], representation["files"]
         )
-        plib._subprocess(["otioview", file_path])
+        plib.run_subprocess(["otioview", file_path])
 
 
 class CollectEditorial(pyblish.api.InstancePlugin):

pype/plugins/standalonepublisher/publish/extract_trim_video_audio.py

@@ -80,8 +80,9 @@ class ExtractTrimVideoAudio(pype.api.Extractor):
 
             self.log.info(f"Processing: {args}")
             ffmpeg_args = " ".join(args)
-            output = pype.api.subprocess(ffmpeg_args, shell=True)
-            self.log.info(output)
+            pype.api.run_subprocess(
+                ffmpeg_args, shell=True, logger=self.log
+            )
 
             repr = {
                 "name": ext[1:],

pype/tests/test_lib_restructuralization.py

@@ -29,7 +29,7 @@ def test_backward_compatibility(printer):
         from pype.hosts.fusion.lib import switch_item
 
         from pype.lib import source_hash
-        from pype.lib import _subprocess
+        from pype.lib import run_subprocess
 
     except ImportError as e:
         raise