Merge pull request #2188 from pypeclub/feature/OP-1920_Add-command-line-running-of-Site-Sync-server

Add command line way of running site sync server

openpype/cli.py

@@ -357,3 +357,28 @@ def run(script):
 def runtests(folder, mark, pyargs):
     """Run all automatic tests after proper initialization via start.py"""
     PypeCommands().run_tests(folder, mark, pyargs)
+
+
+@main.command()
+@click.option("-d", "--debug",
+              is_flag=True, help=("Run process in debug mode"))
+@click.option("-a", "--active_site", required=True,
+              help="Name of active stie")
+def syncserver(debug, active_site):
+    """Run sync site server in background.
+
+        Some Site Sync use cases need to expose site to another one.
+        For example if majority of artists work in studio, they are not using
+        SS at all, but if you want to expose published assets to 'studio' site
+        to SFTP for only a couple of artists, some background process must
+        mark published assets to live on multiple sites (they might be
+        physically in same location - mounted shared disk).
+
+        Process mimics OP Tray with specific 'active_site' name, all
+        configuration for this "dummy" user comes from Setting or Local
+        Settings (configured by starting OP Tray with env
+        var OPENPYPE_LOCAL_ID set to 'active_site'.
+    """
+    if debug:
+        os.environ['OPENPYPE_DEBUG'] = '3'
+    PypeCommands().syncserver(active_site)

openpype/lib/local_settings.py

@@ -522,6 +522,11 @@ def get_local_site_id():
 
     Identifier is created if does not exists yet.
     """
+    # override local id from environment
+    # used for background syncing
+    if os.environ.get("OPENPYPE_LOCAL_ID"):
+        return os.environ["OPENPYPE_LOCAL_ID"]
+
     registry = OpenPypeSettingsRegistry()
     try:
         return registry.get_item("localId")

openpype/modules/default_modules/sync_server/providers/local_drive.py

@@ -89,6 +89,7 @@ class LocalDriveHandler(AbstractProvider):
         if not os.path.isfile(source_path):
             raise FileNotFoundError("Source file {} doesn't exist."
                                     .format(source_path))
+
         if overwrite:
             thread = threading.Thread(target=self._copy,
                                       args=(source_path, target_path))
@@ -181,7 +182,10 @@ class LocalDriveHandler(AbstractProvider):
 
     def _copy(self, source_path, target_path):
         print("copying {}->{}".format(source_path, target_path))
-        shutil.copy(source_path, target_path)
+        try:
+            shutil.copy(source_path, target_path)
+        except shutil.SameFileError:
+            print("same files, skipping")
 
     def _mark_progress(self, collection, file, representation, server, site,
                        source_path, target_path, direction):

openpype/modules/default_modules/sync_server/sync_server.py

@@ -246,6 +246,7 @@ class SyncServerThread(threading.Thread):
 
             asyncio.ensure_future(self.check_shutdown(), loop=self.loop)
             asyncio.ensure_future(self.sync_loop(), loop=self.loop)
+            log.info("Sync Server Started")
             self.loop.run_forever()
         except Exception:
             log.warning(

openpype/modules/default_modules/sync_server/sync_server_module.py

@@ -152,9 +152,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         if not site_name:
             site_name = self.DEFAULT_SITE
 
-        self.reset_provider_for_file(collection,
-                                     representation_id,
-                                     site_name=site_name, force=force)
+        self.reset_site_on_representation(collection,
+                                          representation_id,
+                                          site_name=site_name, force=force)
 
     # public facing API
     def remove_site(self, collection, representation_id, site_name,
@@ -176,10 +176,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         if not self.get_sync_project_setting(collection):
             raise ValueError("Project not configured")
 
-        self.reset_provider_for_file(collection,
-                                     representation_id,
-                                     site_name=site_name,
-                                     remove=True)
+        self.reset_site_on_representation(collection,
+                                          representation_id,
+                                          site_name=site_name,
+                                          remove=True)
         if remove_local_files:
             self._remove_local_file(collection, representation_id, site_name)
 
@@ -314,8 +314,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         """
         log.info("Pausing SyncServer for {}".format(representation_id))
         self._paused_representations.add(representation_id)
-        self.reset_provider_for_file(collection, representation_id,
-                                     site_name=site_name, pause=True)
+        self.reset_site_on_representation(collection, representation_id,
+                                          site_name=site_name, pause=True)
 
     def unpause_representation(self, collection, representation_id, site_name):
         """
@@ -334,8 +334,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         except KeyError:
             pass
         # self.paused_representations is not persistent
-        self.reset_provider_for_file(collection, representation_id,
-                                     site_name=site_name, pause=False)
+        self.reset_site_on_representation(collection, representation_id,
+                                          site_name=site_name, pause=False)
 
     def is_representation_paused(self, representation_id,
                                  check_parents=False, project_name=None):
@@ -799,12 +799,19 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
     def tray_init(self):
         """
-            Actual initialization of Sync Server.
+            Actual initialization of Sync Server for Tray.
 
             Called when tray is initialized, it checks if module should be
             enabled. If not, no initialization necessary.
         """
-        # import only in tray, because of Python2 hosts
+        self.server_init()
+
+        from .tray.app import SyncServerWindow
+        self.widget = SyncServerWindow(self)
+
+    def server_init(self):
+        """Actual initialization of Sync Server."""
+        # import only in tray or Python3, because of Python2 hosts
         from .sync_server import SyncServerThread
 
         if not self.enabled:
@@ -816,8 +823,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             return
 
         self.lock = threading.Lock()
+
         self.sync_server_thread = SyncServerThread(self)
 
+
     def tray_start(self):
         """
             Triggered when Tray is started.
@@ -829,6 +838,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         Returns:
             None
         """
+        self.server_start()
+
+    def server_start(self):
         if self.sync_project_settings and self.enabled:
             self.sync_server_thread.start()
         else:
@@ -841,6 +853,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
             Called from Module Manager
         """
+        self.server_exit()
+
+    def server_exit(self):
         if not self.sync_server_thread:
             return
 
@@ -850,6 +865,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             log.info("Stopping sync server server")
             self.sync_server_thread.is_running = False
             self.sync_server_thread.stop()
+            log.info("Sync server stopped")
         except Exception:
             log.warning(
                 "Error has happened during Killing sync server",
@@ -1319,9 +1335,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         return -1, None
 
-    def reset_provider_for_file(self, collection, representation_id,
-                                side=None, file_id=None, site_name=None,
-                                remove=False, pause=None, force=False):
+    def reset_site_on_representation(self, collection, representation_id,
+                                     side=None, file_id=None, site_name=None,
+                                     remove=False, pause=None, force=False):
         """
             Reset information about synchronization for particular 'file_id'
             and provider.

openpype/modules/default_modules/sync_server/tray/widgets.py

@@ -489,7 +489,7 @@ class _SyncRepresentationWidget(QtWidgets.QWidget):
                           format(check_progress))
                 continue
 
-            self.sync_server.reset_provider_for_file(
+            self.sync_server.reset_site_on_representation(
                 self.model.project,
                 representation_id,
                 site_name=site_name,
@@ -872,7 +872,7 @@ class SyncRepresentationDetailWidget(_SyncRepresentationWidget):
                           format(check_progress))
                 continue
 
-            self.sync_server.reset_provider_for_file(
+            self.sync_server.reset_site_on_representation(
                 self.model.project,
                 self.representation_id,
                 site_name=site_name,

openpype/plugins/publish/integrate_new.py

@@ -1029,6 +1029,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         """
         local_site = 'studio'  # default
         remote_site = None
+        always_accesible = []
         sync_server_presets = None
 
         if (instance.context.data["system_settings"]
@@ -1043,6 +1044,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
             if sync_server_presets["enabled"]:
                 local_site = sync_server_presets["config"].\
                     get("active_site", "studio").strip()
+                always_accesible = sync_server_presets["config"].\
+                    get("always_accessible_on", [])
                 if local_site == 'local':
                     local_site = local_site_id
 
@@ -1073,6 +1076,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
                 meta = {"name": remote_site.strip()}
                 rec["sites"].append(meta)
 
+            # add skeleton for site where it should be always synced to
+            for always_on_site in always_accesible:
+                if always_on_site not in [local_site, remote_site]:
+                    meta = {"name": always_on_site.strip()}
+                    rec["sites"].append(meta)
+
         return rec
 
     def handle_destination_files(self, integrated_file_sizes, mode):

openpype/pype_commands.py

@@ -358,3 +358,28 @@ class PypeCommands:
         cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str)
         print("Running {}".format(cmd))
         subprocess.run(cmd)
+
+    def syncserver(self, active_site):
+        """Start running sync_server in background."""
+        import signal
+        os.environ["OPENPYPE_LOCAL_ID"] = active_site
+
+        def signal_handler(sig, frame):
+            print("You pressed Ctrl+C. Process ended.")
+            sync_server_module.server_exit()
+            sys.exit(0)
+
+        signal.signal(signal.SIGINT, signal_handler)
+        signal.signal(signal.SIGTERM, signal_handler)
+
+        from openpype.modules import ModulesManager
+
+        manager = ModulesManager()
+        sync_server_module = manager.modules_by_name["sync_server"]
+
+        sync_server_module.server_init()
+        sync_server_module.server_start()
+
+        import time
+        while True:
+            time.sleep(1.0)

openpype/settings/defaults/project_settings/global.json

@@ -322,6 +322,7 @@
         "config": {
             "retry_cnt": "3",
             "loop_delay": "60",
+            "always_accessible_on": [],
             "active_site": "studio",
             "remote_site": "studio"
         },

openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json

@@ -26,15 +26,24 @@
                     "key": "loop_delay",
                     "label": "Loop Delay"
                 },
+                {
+                    "type": "list",
+                    "key": "always_accessible_on",
+                    "label": "Always accessible on sites",
+                    "object_type": "text"
+                },
+                {
+                    "type": "splitter"
+                },
                 {
                     "type": "text",
                     "key": "active_site",
-                    "label": "Active Site"
+                    "label": "User Default Active Site"
                 },
                 {
                     "type": "text",
                     "key": "remote_site",
-                    "label": "Remote Site"
+                    "label": "User Default Remote Site"
                 }
             ]
         },

website/docs/module_site_sync.md

@@ -140,3 +140,42 @@ Beware that ssh key expects OpenSSH format (`.pem`) not a Putty format (`.ppk`)!
 
 If a studio needs to use other services for cloud storage, or want to implement totally different storage providers, they can do so by writing their own provider plugin. We're working on a developer documentation, however, for now we recommend looking at `abstract_provider.py`and `gdrive.py` inside `openpype/modules/sync_server/providers` and using it as a template.
 
+### Running Site Sync in background
+
+Site Sync server synchronizes new published files from artist machine into configured remote location by default.
+
+There might be a use case where you need to synchronize between "non-artist" sites, for example between studio site and cloud. In this case
+you need to run Site Sync as a background process from a command line (via service etc) 24/7.
+
+To configure all sites where all published files should be synced eventually you need to configure `project_settings/global/sync_server/config/always_accessible_on` property in Settins (per project) first.
+
+![Set another non artist remote site](assets/site_sync_always_on.png)
+
+This is an example of:
+- Site Sync is enabled for a project
+- default active and remote sites are set to `studio` - eg. standard process: everyone is working in a studio, publishing to shared location etc.
+- (but this also allows any of the artists to work remotely, they would change their active site in their own Local Settings to `local` and configure local root.
+  This would result in everything artist publishes is saved first onto his local folder AND synchronized to `studio` site eventually.)
+- everything exported must also be eventually uploaded to `sftp` site
+
+This eventual synchronization between `studio` and `sftp` sites must be physically handled by background process.
+
+As current implementation relies heavily on Settings and Local Settings, background process for a specific site ('studio' for example) must be configured via Tray first to `syncserver` command to work.
+
+To do this:
+
+- run OP `Tray` with environment variable OPENPYPE_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.)
+- start `Tray`
+- check `Local ID` in information dialog after clicking on version number in the Tray
+- open `Local Settings` in the `Tray`
+- configure for each project necessary active site and remote site
+- close `Tray`
+- run OP from a command line with `syncserver` and `--active_site` arguments
+
+
+This is an example how to trigger background synching process where active (source) site is `studio`. 
+(It is expected that OP is installed on a machine, `openpype_console` is on PATH. If not, add full path to executable.
+)
+```shell
+openpype_console syncserver --active_site studio
+```