OP-3698 - implemented timout or Webpublisher's PS processing

openpype/lib/remote_publish.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from datetime import datetime
 import collections
 
@@ -9,6 +10,8 @@ import pyblish.api
 
 from openpype.client.mongo import OpenPypeMongoConnection
 from openpype.lib.plugin_tools import parse_json
+from openpype.lib.profiles_filtering import filter_profiles
+from openpype.api import get_project_settings
 
 ERROR_STATUS = "error"
 IN_PROGRESS_STATUS = "in_progress"
@@ -175,14 +178,8 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None):
     )
 
 
-def fail_batch(_id, batches_in_progress, dbcon):
-    """Set current batch as failed as there are some stuck batches."""
-    running_batches = [str(batch["_id"])
-                       for batch in batches_in_progress
-                       if batch["_id"] != _id]
-    msg = "There are still running batches {}\n". \
-        format("\n".join(running_batches))
-    msg += "Ask admin to check them and reprocess current batch"
+def fail_batch(_id, dbcon, msg):
+    """Set current batch as failed as there is some problem."""
     dbcon.update_one(
         {"_id": _id},
         {"$set":
@@ -259,3 +256,19 @@ def get_task_data(batch_dir):
             "Cannot parse batch meta in {} folder".format(task_data))
 
     return task_data
+
+
+def get_timeout(project_name, host_name, task_type):
+    """Returns timeout(seconds) from Setting profile."""
+    filter_data = {
+        "task_types": task_type,
+        "hosts": host_name
+    }
+    timeout_profiles = (get_project_settings(project_name)["webpublisher"]
+                                                          ["timeout_profiles"])
+    matching_item = filter_profiles(timeout_profiles, filter_data)
+    timeout = sys.maxsize
+    if matching_item:
+        timeout = matching_item["timeout"]
+
+    return timeout

openpype/pype_commands.py

@@ -15,6 +15,7 @@ from openpype.lib.remote_publish import (
     fail_batch,
     find_variant_key,
     get_task_data,
+    get_timeout,
     IN_PROGRESS_STATUS
 )
 
@@ -222,10 +223,17 @@ class PypeCommands:
 
         batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS}))
         if len(batches_in_progress) > 1:
-            fail_batch(_id, batches_in_progress, dbcon)
+            running_batches = [str(batch["_id"])
+                               for batch in batches_in_progress
+                               if batch["_id"] != _id]
+            msg = "There are still running batches {}\n". \
+                format("\n".join(running_batches))
+            msg += "Ask admin to check them and reprocess current batch"
+            fail_batch(_id, dbcon, msg)
             print("Another batch running, probably stuck, ask admin for help")
 
-        asset, task_name, _ = get_batch_asset_task_info(task_data["context"])
+        asset, task_name, task_type = get_batch_asset_task_info(
+            task_data["context"])
 
         application_manager = ApplicationManager()
         found_variant_key = find_variant_key(application_manager, host_name)
@@ -269,8 +277,17 @@ class PypeCommands:
 
         launched_app = application_manager.launch(app_name, **data)
 
+        timeout = get_timeout(project, host_name, task_type)
+
+        time_start = time.time()
         while launched_app.poll() is None:
             time.sleep(0.5)
+            if time.time() - time_start > timeout:
+                launched_app.terminate()
+                msg = "Timeout reached"
+                fail_batch(_id, dbcon, msg)
+                raise ValueError("Timeout reached")
+
 
     @staticmethod
     def remotepublish(project, batch_path, user_email, targets=None):