From a0f6a3f37971c30390f5a1b99d81f1a582ab5122 Mon Sep 17 00:00:00 2001
From: Aleks Berland <aleksberland@gmail.com>
Date: Mon, 25 Aug 2025 19:09:20 -0400
Subject: [PATCH] Implement upload retries for reviewable files and add
 user-friendly error handling in case of timeout. Update validation help
 documentation for upload failures.

---
 .../publish/help/validate_publish_dir.xml     | 18 +++++
 .../plugins/publish/integrate_review.py       | 68 +++++++++++++++++--
 2 files changed, 80 insertions(+), 6 deletions(-)

diff --git a/client/ayon_core/plugins/publish/help/validate_publish_dir.xml b/client/ayon_core/plugins/publish/help/validate_publish_dir.xml
index 9f62b264bf..0449e61fa2 100644
--- a/client/ayon_core/plugins/publish/help/validate_publish_dir.xml
+++ b/client/ayon_core/plugins/publish/help/validate_publish_dir.xml
@@ -1,5 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <root>
+<error id="upload_timeout">
+<title>Review upload timed out</title>
+<description>
+## Review upload failed after retries
+
+The connection to the AYON server timed out while uploading a reviewable file.
+
+### How to repair?
+
+1. Try publishing again. Intermittent network hiccups often resolve on retry.
+2. Ensure your network/VPN is stable and large uploads are allowed.
+3. If it keeps failing, try again later or contact your admin.
+
+<pre>File: {file}
+Error: {error}</pre>
+
+</description>
+</error>
 <error id="main">
 <title>Source directory not collected</title>
 <description>
diff --git a/client/ayon_core/plugins/publish/integrate_review.py b/client/ayon_core/plugins/publish/integrate_review.py
index 0a6b24adb4..c7ac5038d3 100644
--- a/client/ayon_core/plugins/publish/integrate_review.py
+++ b/client/ayon_core/plugins/publish/integrate_review.py
@@ -1,11 +1,14 @@
 import os
+import time
 
-import pyblish.api
 import ayon_api
+import pyblish.api
 from ayon_api.server_api import RequestTypes
-
 from ayon_core.lib import get_media_mime_type
-from ayon_core.pipeline.publish import get_publish_repre_path
+from ayon_core.pipeline.publish import (
+    PublishXmlValidationError,
+    get_publish_repre_path,
+)
 
 
 class IntegrateAYONReview(pyblish.api.InstancePlugin):
@@ -82,11 +85,12 @@ class IntegrateAYONReview(pyblish.api.InstancePlugin):
             headers = ayon_con.get_headers(content_type)
             headers["x-file-name"] = filename
             self.log.info(f"Uploading reviewable {repre_path}")
-            ayon_con.upload_file(
+            # Upload with retries and clear help if it keeps failing
+            self._upload_with_retries(
+                ayon_con,
                 endpoint,
                 repre_path,
-                headers=headers,
-                request_type=RequestTypes.post,
+                headers,
             )
 
     def _get_review_label(self, repre, uploaded_labels):
@@ -100,3 +104,55 @@ class IntegrateAYONReview(pyblish.api.InstancePlugin):
             idx += 1
             label = f"{orig_label}_{idx}"
         return label
+
+    def _upload_with_retries(
+        self,
+        ayon_con,
+        endpoint,
+        repre_path,
+        headers,
+        max_retries: int = 3,
+        backoff_seconds: int = 2,
+    ):
+        """Upload file with simple exponential backoff retries.
+
+        If all retries fail we raise a PublishXmlValidationError with a help key
+        to guide the user to retry publish.
+        """
+        last_error = None
+        for attempt in range(1, max_retries + 1):
+            try:
+                ayon_con.upload_file(
+                    endpoint,
+                    repre_path,
+                    headers=headers,
+                    request_type=RequestTypes.post,
+                )
+                return
+            except Exception as exc:  # noqa: BLE001 - bubble after retries
+                last_error = exc
+                # Log and retry with backoff if attempts remain
+                if attempt < max_retries:
+                    wait = backoff_seconds * (2 ** (attempt - 1))
+                    self.log.warning(
+                        f"Review upload failed (attempt {attempt}/{max_retries}): {exc}. "
+                        f"Retrying in {wait}s..."
+                    )
+                    try:
+                        time.sleep(wait)
+                    except Exception:  # Sleep errors are highly unlikely; continue
+                        pass
+                else:
+                    # Exhausted retries - raise a user-friendly validation error with help
+                    raise PublishXmlValidationError(
+                        self,
+                        (
+                            "Upload of reviewable timed out or failed after multiple attempts. "
+                            "Please try publishing again."
+                        ),
+                        key="upload_timeout",
+                        formatting_data={
+                            "file": repre_path,
+                            "error": str(last_error),
+                        },
+                    )