OP-1978 - merge OP-1937

openpype/cli.py

@@ -345,3 +345,28 @@ def run(script):
 def runtests(folder, mark, pyargs):
     """Run all automatic tests after proper initialization via start.py"""
     PypeCommands().run_tests(folder, mark, pyargs)
+
+
+@main.command()
+@click.option("-d", "--debug",
+              is_flag=True, help=("Run process in debug mode"))
+@click.option("-a", "--active_site", required=True,
+              help="Name of active stie")
+def syncserver(debug, active_site):
+    """Run sync site server in background.
+
+        Some Site Sync use cases need to expose site to another one.
+        For example if majority of artists work in studio, they are not using
+        SS at all, but if you want to expose published assets to 'studio' site
+        to SFTP for only a couple of artists, some background process must
+        mark published assets to live on multiple sites (they might be
+        physically in same location - mounted shared disk).
+
+        Process mimics OP Tray with specific 'active_site' name, all
+        configuration for this "dummy" user comes from Setting or Local
+        Settings (configured by starting OP Tray with env
+        var OPENPYPE_LOCAL_ID set to 'active_site'.
+    """
+    if debug:
+        os.environ['OPENPYPE_DEBUG'] = '3'
+    PypeCommands().syncserver(active_site)

openpype/lib/local_settings.py

@@ -522,6 +522,11 @@ def get_local_site_id():
 
     Identifier is created if does not exists yet.
     """
+    # override local id from environment
+    # used for background syncing
+    if os.environ.get("OPENPYPE_LOCAL_ID"):
+        return os.environ["OPENPYPE_LOCAL_ID"]
+
     registry = OpenPypeSettingsRegistry()
     try:
         return registry.get_item("localId")

openpype/modules/default_modules/sync_server/providers/local_drive.py

@@ -89,6 +89,7 @@ class LocalDriveHandler(AbstractProvider):
         if not os.path.isfile(source_path):
             raise FileNotFoundError("Source file {} doesn't exist."
                                     .format(source_path))
+
         if overwrite:
             thread = threading.Thread(target=self._copy,
                                       args=(source_path, target_path))
@@ -181,7 +182,10 @@ class LocalDriveHandler(AbstractProvider):
 
     def _copy(self, source_path, target_path):
         print("copying {}->{}".format(source_path, target_path))
-        shutil.copy(source_path, target_path)
+        try:
+            shutil.copy(source_path, target_path)
+        except shutil.SameFileError:
+            print("same files, skipping")
 
     def _mark_progress(self, collection, file, representation, server, site,
                        source_path, target_path, direction):

openpype/modules/default_modules/sync_server/providers/sftp.py

@@ -36,7 +36,6 @@ class SFTPHandler(AbstractProvider):
 
     def __init__(self, project_name, site_name, tree=None, presets=None):
         self.presets = None
-        self.active = False
         self.project_name = project_name
         self.site_name = site_name
         self.root = None

openpype/modules/default_modules/sync_server/sync_server.py

@@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name,
                                          remote_site_name,
                                          True
                                          )
+
+    module.handle_alternate_site(collection, representation, remote_site_name,
+                                 file["_id"], file_id)
+
     return file_id
 
 
@@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name,
                                          local_site,
                                          True
                                          )
+
+    module.handle_alternate_site(collection, representation, remote_site_name,
+                                 file["_id"], file_id)
+
     return file_id
 
 
@@ -246,6 +254,7 @@ class SyncServerThread(threading.Thread):
 
             asyncio.ensure_future(self.check_shutdown(), loop=self.loop)
             asyncio.ensure_future(self.sync_loop(), loop=self.loop)
+            log.info("Sync Server Started")
             self.loop.run_forever()
         except Exception:
             log.warning(

openpype/modules/default_modules/sync_server/sync_server_module.py

@@ -109,6 +109,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         # some parts of code need to run sequentially, not in async
         self.lock = None
+        self._sync_system_settings = None
         # settings for all enabled projects for sync
         self._sync_project_settings = None
         self.sync_server_thread = None  # asyncio requires new thread
@@ -152,9 +153,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         if not site_name:
             site_name = self.DEFAULT_SITE
 
-        self.reset_provider_for_file(collection,
-                                     representation_id,
-                                     site_name=site_name, force=force)
+        self.reset_site_on_representation(collection,
+                                          representation_id,
+                                          site_name=site_name, force=force)
 
     # public facing API
     def remove_site(self, collection, representation_id, site_name,
@@ -176,10 +177,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         if not self.get_sync_project_setting(collection):
             raise ValueError("Project not configured")
 
-        self.reset_provider_for_file(collection,
-                                     representation_id,
-                                     site_name=site_name,
-                                     remove=True)
+        self.reset_site_on_representation(collection,
+                                          representation_id,
+                                          site_name=site_name,
+                                          remove=True)
         if remove_local_files:
             self._remove_local_file(collection, representation_id, site_name)
 
@@ -314,8 +315,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         """
         log.info("Pausing SyncServer for {}".format(representation_id))
         self._paused_representations.add(representation_id)
-        self.reset_provider_for_file(collection, representation_id,
-                                     site_name=site_name, pause=True)
+        self.reset_site_on_representation(collection, representation_id,
+                                          site_name=site_name, pause=True)
 
     def unpause_representation(self, collection, representation_id, site_name):
         """
@@ -334,8 +335,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         except KeyError:
             pass
         # self.paused_representations is not persistent
-        self.reset_provider_for_file(collection, representation_id,
-                                     site_name=site_name, pause=False)
+        self.reset_site_on_representation(collection, representation_id,
+                                          site_name=site_name, pause=False)
 
     def is_representation_paused(self, representation_id,
                                  check_parents=False, project_name=None):
@@ -769,6 +770,43 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
                     enabled_projects.append(project_name)
 
         return enabled_projects
+
+    def handle_alternate_site(self, collection, representation, processed_site,
+                              file_id, synced_file_id):
+        """
+            For special use cases where one site vendors another.
+
+            Current use case is sftp site vendoring (exposing) same data as
+            regular site (studio). Each site is accessible for different
+            audience. 'studio' for artists in a studio, 'sftp' for externals.
+
+            Change of file status on one site actually means same change on
+            'alternate' site. (eg. artists publish to 'studio', 'sftp' is using
+            same location >> file is accesible on 'sftp' site right away.
+
+            Args:
+                collection (str): name of project
+                representation (dict)
+                processed_site (str): real site_name of published/uploaded file
+                file_id (ObjectId): DB id of file handled
+                synced_file_id (str): id of the created file returned
+                    by provider
+        """
+        sites = self.sync_system_settings.get("sites", {})
+        for site_name, site_info in sites.items():
+            if processed_site in site_info.get("alternative_sites", []):
+                query = {
+                    "_id": representation["_id"]
+                }
+                elem = {"name": "sftp",
+                        "created_dt": datetime.now(),
+                        "id": synced_file_id}
+
+                self.log.debug("Adding alternate {} to {}".format(
+                    site_name, representation["_id"]))
+                self._add_site(collection, query,
+                               [representation], elem,
+                               site_name, file_id=file_id, force=True)
     """ End of Public API """
 
     def get_local_file_path(self, collection, site_name, file_path):
@@ -799,12 +837,19 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
     def tray_init(self):
         """
-            Actual initialization of Sync Server.
+            Actual initialization of Sync Server for Tray.
 
             Called when tray is initialized, it checks if module should be
             enabled. If not, no initialization necessary.
         """
-        # import only in tray, because of Python2 hosts
+        self.server_init()
+
+        from .tray.app import SyncServerWindow
+        self.widget = SyncServerWindow(self)
+
+    def server_init(self):
+        """Actual initialization of Sync Server."""
+        # import only in tray or Python3, because of Python2 hosts
         from .sync_server import SyncServerThread
 
         if not self.enabled:
@@ -816,6 +861,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             return
 
         self.lock = threading.Lock()
+
         self.sync_server_thread = SyncServerThread(self)
 
     def tray_start(self):
@@ -829,6 +875,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         Returns:
             None
         """
+        self.server_start()
+
+    def server_start(self):
         if self.sync_project_settings and self.enabled:
             self.sync_server_thread.start()
         else:
@@ -841,6 +890,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
             Called from Module Manager
         """
+        self.server_exit()
+
+    def server_exit(self):
         if not self.sync_server_thread:
             return
 
@@ -850,6 +902,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             log.info("Stopping sync server server")
             self.sync_server_thread.is_running = False
             self.sync_server_thread.stop()
+            log.info("Sync server stopped")
         except Exception:
             log.warning(
                 "Error has happened during Killing sync server",
@@ -892,6 +945,14 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         return self._connection
 
+    @property
+    def sync_system_settings(self):
+        if self._sync_system_settings is None:
+            self._sync_system_settings = get_system_settings()["modules"].\
+                get("sync_server")
+
+        return self._sync_system_settings
+
     @property
     def sync_project_settings(self):
         if self._sync_project_settings is None:
@@ -977,9 +1038,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
                 (dict): {'studio': {'provider':'local_drive'...},
                          'MY_LOCAL': {'provider':....}}
         """
-        sys_sett = get_system_settings()
-        sync_sett = sys_sett["modules"].get("sync_server")
-
+        sync_sett = self.sync_system_settings
         project_enabled = True
         if project_name:
             project_enabled = project_name in self.get_enabled_projects()
@@ -1037,8 +1096,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             if provider:
                 return provider
 
-        sys_sett = get_system_settings()
-        sync_sett = sys_sett["modules"].get("sync_server")
+        sync_sett = self.sync_system_settings
         for conf_site, detail in sync_sett.get("sites", {}).items():
             sites[conf_site] = detail.get("provider")
 
@@ -1319,9 +1377,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         return -1, None
 
-    def reset_provider_for_file(self, collection, representation_id,
-                                side=None, file_id=None, site_name=None,
-                                remove=False, pause=None, force=False):
+    def reset_site_on_representation(self, collection, representation_id,
+                                     side=None, file_id=None, site_name=None,
+                                     remove=False, pause=None, force=False):
         """
             Reset information about synchronization for particular 'file_id'
             and provider.
@@ -1407,9 +1465,12 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         update = {
             "$set": {"files.$[f].sites.$[s]": elem}
         }
+        if not isinstance(file_id, ObjectId):
+            file_id = ObjectId(file_id)
+
         arr_filter = [
             {'s.name': site_name},
-            {'f._id': ObjectId(file_id)}
+            {'f._id': file_id}
         ]
 
         self._update_site(collection, query, update, arr_filter)

openpype/modules/default_modules/sync_server/tray/widgets.py

@@ -154,7 +154,7 @@ class SyncProjectListWidget(QtWidgets.QWidget):
            selected_index.isValid() and \
            not self._selection_changed:
             mode = QtCore.QItemSelectionModel.Select | \
-                   QtCore.QItemSelectionModel.Rows
+                QtCore.QItemSelectionModel.Rows
             self.project_list.selectionModel().select(selected_index, mode)
 
         if self.current_project:
@@ -489,7 +489,7 @@ class _SyncRepresentationWidget(QtWidgets.QWidget):
                           format(check_progress))
                 continue
 
-            self.sync_server.reset_provider_for_file(
+            self.sync_server.reset_site_on_representation(
                 self.model.project,
                 representation_id,
                 site_name=site_name,
@@ -872,7 +872,7 @@ class SyncRepresentationDetailWidget(_SyncRepresentationWidget):
                           format(check_progress))
                 continue
 
-            self.sync_server.reset_provider_for_file(
+            self.sync_server.reset_site_on_representation(
                 self.model.project,
                 self.representation_id,
                 site_name=site_name,

openpype/plugins/publish/integrate_new.py

@@ -1029,29 +1029,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         """
         local_site = 'studio'  # default
         remote_site = None
-        sync_server_presets = None
-
-        if (instance.context.data["system_settings"]
-                                 ["modules"]
-                                 ["sync_server"]
-                                 ["enabled"]):
-            sync_server_presets = (instance.context.data["project_settings"]
-                                                        ["global"]
-                                                        ["sync_server"])
-
-            local_site_id = openpype.api.get_local_site_id()
-            if sync_server_presets["enabled"]:
-                local_site = sync_server_presets["config"].\
-                    get("active_site", "studio").strip()
-                if local_site == 'local':
-                    local_site = local_site_id
-
-                remote_site = sync_server_presets["config"].get("remote_site")
-                if remote_site == local_site:
-                    remote_site = None
-
-                if remote_site == 'local':
-                    remote_site = local_site_id
+        always_accesible = []
+        sync_project_presets = None
 
         rec = {
             "_id": io.ObjectId(),
@@ -1066,12 +1045,92 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if sites:
             rec["sites"] = sites
         else:
+            system_sync_server_presets = (
+                instance.context.data["system_settings"]
+                                     ["modules"]
+                                     ["sync_server"])
+            log.debug("system_sett:: {}".format(system_sync_server_presets))
+
+            if system_sync_server_presets["enabled"]:
+                sync_project_presets = (
+                    instance.context.data["project_settings"]
+                                         ["global"]
+                                         ["sync_server"])
+
+            if sync_project_presets and sync_project_presets["enabled"]:
+                local_site, remote_site = self._get_sites(sync_project_presets)
+
+                always_accesible = sync_project_presets["config"]. \
+                    get("always_accessible_on", [])
+
+            already_attached_sites = {}
             meta = {"name": local_site, "created_dt": datetime.now()}
             rec["sites"] = [meta]
+            already_attached_sites[meta["name"]] = meta["created_dt"]
 
-            if remote_site:
+            if sync_project_presets and sync_project_presets["enabled"]:
+                # add remote
                 meta = {"name": remote_site.strip()}
                 rec["sites"].append(meta)
+                already_attached_sites[meta["name"]] = None
+
+                # add skeleton for site where it should be always synced to
+                for always_on_site in always_accesible:
+                    if always_on_site not in already_attached_sites.keys():
+                        meta = {"name": always_on_site.strip()}
+                        rec["sites"].append(meta)
+                        already_attached_sites[meta["name"]] = None
+
+                # add alternative sites
+                rec = self._add_alternative_sites(system_sync_server_presets,
+                                                  already_attached_sites,
+                                                  rec)
+
+            log.debug("final sites:: {}".format(rec["sites"]))
+
+        return rec
+
+    def _get_sites(self, sync_project_presets):
+        """Returns tuple (local_site, remote_site)"""
+        local_site_id = openpype.api.get_local_site_id()
+        local_site = sync_project_presets["config"]. \
+            get("active_site", "studio").strip()
+
+        if local_site == 'local':
+            local_site = local_site_id
+
+        remote_site = sync_project_presets["config"].get("remote_site")
+        if remote_site == local_site:
+            remote_site = None
+
+        if remote_site == 'local':
+            remote_site = local_site_id
+
+        return local_site, remote_site
+
+    def _add_alternative_sites(self,
+                               system_sync_server_presets,
+                               already_attached_sites,
+                               rec):
+        """Loop through all configured sites and add alternatives.
+
+            See SyncServerModule.handle_alternate_site
+        """
+        conf_sites = system_sync_server_presets.get("sites", {})
+
+        for site_name, site_info in conf_sites.items():
+            alt_sites = set(site_info.get("alternative_sites", []))
+            for added_site in already_attached_sites.keys():
+                if added_site in alt_sites:
+                    if site_name in already_attached_sites.keys():
+                        continue
+                    meta = {"name": site_name}
+                    real_created = already_attached_sites[added_site]
+                    # alt site inherits state of 'created_dt'
+                    if real_created:
+                        meta["created_dt"] = real_created
+                    rec["sites"].append(meta)
+                    already_attached_sites[meta["name"]] = real_created
 
         return rec
 

openpype/pype_commands.py

@@ -352,3 +352,28 @@ class PypeCommands:
         cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str)
         print("Running {}".format(cmd))
         subprocess.run(cmd)
+
+    def syncserver(self, active_site):
+        """Start running sync_server in background."""
+        import signal
+        os.environ["OPENPYPE_LOCAL_ID"] = active_site
+
+        def signal_handler(sig, frame):
+            print("You pressed Ctrl+C. Process ended.")
+            sync_server_module.server_exit()
+            sys.exit(0)
+
+        signal.signal(signal.SIGINT, signal_handler)
+        signal.signal(signal.SIGTERM, signal_handler)
+
+        from openpype.modules import ModulesManager
+
+        manager = ModulesManager()
+        sync_server_module = manager.modules_by_name["sync_server"]
+
+        sync_server_module.server_init()
+        sync_server_module.server_start()
+
+        import time
+        while True:
+            time.sleep(1.0)

openpype/settings/defaults/project_settings/global.json

@@ -319,6 +319,7 @@
         "config": {
             "retry_cnt": "3",
             "loop_delay": "60",
+            "always_accessible_on": [],
             "active_site": "studio",
             "remote_site": "studio"
         },

openpype/settings/entities/dict_conditional.py

@@ -762,6 +762,17 @@ class SyncServerProviders(DictConditionalEntity):
 
         enum_children = []
         for provider_code, configurables in system_settings_schema.items():
+            # any site could be exposed or vendorized by different site
+            # eg studio site content could be mapped on sftp site, single file
+            # accessible via 2 different protocols (sites)
+            configurables.append(
+                {
+                    "type": "list",
+                    "key": "alternative_sites",
+                    "label": "Alternative sites",
+                    "object_type": "text"
+                }
+            )
             label = provider_code_to_label.get(provider_code) or provider_code
 
             enum_children.append({

openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json

@@ -26,15 +26,24 @@
                     "key": "loop_delay",
                     "label": "Loop Delay"
                 },
+                {
+                    "type": "list",
+                    "key": "always_accessible_on",
+                    "label": "Always accessible on sites",
+                    "object_type": "text"
+                },
+                {
+                    "type": "splitter"
+                },
                 {
                     "type": "text",
                     "key": "active_site",
-                    "label": "Active Site"
+                    "label": "User Default Active Site"
                 },
                 {
                     "type": "text",
                     "key": "remote_site",
-                    "label": "Remote Site"
+                    "label": "User Default Remote Site"
                 }
             ]
         },

openpype/tests/mongo_performance.py

@@ -104,8 +104,8 @@ class TestPerformance():
                         "name": "mb",
                         "parent": {"oid": '{}'.format(id)},
                         "data": {
-                            "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name),  # noqa
-                            "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"  # noqa
+                            "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name),  # noqa: E501
+                            "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"  # noqa: E501
                         },
                         "type": "representation",
                         "schema": "openpype:representation-2.0"
@@ -188,21 +188,21 @@ class TestPerformance():
                         create_files=False):
         ret = [
             {
-                 "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                 "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                  "_id": '{}'.format(file_id),
                  "hash": "temphash",
                  "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
                  "size": random.randint(0, self.MAX_FILE_SIZE_B)
             },
             {
-                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                 "_id": '{}'.format(file_id2),
                 "hash": "temphash",
                 "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
                 "size": random.randint(0, self.MAX_FILE_SIZE_B)
             },
             {
-                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                 "_id": '{}'.format(file_id3),
                 "hash": "temphash",
                 "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
@@ -223,8 +223,8 @@ class TestPerformance():
         ret = {}
         ret['{}'.format(file_id)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236
@@ -232,16 +232,16 @@ class TestPerformance():
 
         ret['{}'.format(file_id2)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236
         }
         ret['{}'.format(file_id3)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236

website/docs/module_site_sync.md

@@ -27,6 +27,38 @@ To use synchronization, *Site Sync* needs to be enabled globally in **OpenPype S
 
 ![Configure module](assets/site_sync_system.png)
 
+### Sites 
+
+By default there are two sites created for each OpenPype installation:
+- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled.
+- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID.
+
+Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. 
+
+Many different sites can be created and configured on the system level, and some or all can be assigned to each project.
+
+Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup).
+
+Sites could be configured differently per project basis. 
+
+Each new site needs to be created first in `System Settings`. Most important feature of site is its Provider, select one from already prepared Providers.
+
+#### Alternative sites 
+
+This attribute is meant for special use cases only.
+
+One of the use cases is sftp site vendoring (exposing) same data as regular site (studio). Each site is accessible for different audience. 'studio' for artists in a studio via shared disk, 'sftp' for externals via sftp server with mounted 'studio' drive.
+
+Change of file status on one site actually means same change on 'alternate' site occured too. (eg. artists publish to 'studio', 'sftp' is using
+same location >> file is accessible on 'sftp' site right away, no need to sync it anyhow.)
+
+##### Example
+![Configure module](assets/site_sync_system_sites.png)
+Admin created new `sftp` site which is handled by `SFTP` provider. Somewhere in the studio SFTP server is deployed on a machine that has access to `studio` drive.
+
+Alternative sites work both way:
+- everything published to `studio` is accessible on a `sftp` site too
+- everything published to `sftp` (most probably via artist's local disk - artists publishes locally, representation is marked to be synced to `sftp`. Immediately after it is synced, it is marked to be available on `studio` too for artists in the studio to use.)
 
 ## Project Settings
 
@@ -45,21 +77,6 @@ Artists can also override which site they use as active and remote if need be.
 ![Local overrides](assets/site_sync_local_setting.png)
 
 
-## Sites 
-
-By default there are two sites created for each OpenPype installation:
-- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled.
-- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID.
-
-Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. 
-
-Many different sites can be created and configured on the system level, and some or all can be assigned to each project.
-
-Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup).
-
-Sites could be configured differently per project basis. 
-
-
 ## Providers
 
 Each site implements a so called `provider` which handles most common operations (list files, copy files etc.) and provides interface with a particular type of storage. (disk, gdrive, aws, etc.)
@@ -140,3 +157,42 @@ Beware that ssh key expects OpenSSH format (`.pem`) not a Putty format (`.ppk`)!
 
 If a studio needs to use other services for cloud storage, or want to implement totally different storage providers, they can do so by writing their own provider plugin. We're working on a developer documentation, however, for now we recommend looking at `abstract_provider.py`and `gdrive.py` inside `openpype/modules/sync_server/providers` and using it as a template.
 
+### Running Site Sync in background
+
+Site Sync server synchronizes new published files from artist machine into configured remote location by default.
+
+There might be a use case where you need to synchronize between "non-artist" sites, for example between studio site and cloud. In this case
+you need to run Site Sync as a background process from a command line (via service etc) 24/7.
+
+To configure all sites where all published files should be synced eventually you need to configure `project_settings/global/sync_server/config/always_accessible_on` property in Settins (per project) first.
+
+![Set another non artist remote site](assets/site_sync_always_on.png)
+
+This is an example of:
+- Site Sync is enabled for a project
+- default active and remote sites are set to `studio` - eg. standard process: everyone is working in a studio, publishing to shared location etc.
+- (but this also allows any of the artists to work remotely, they would change their active site in their own Local Settings to `local` and configure local root.
+  This would result in everything artist publishes is saved first onto his local folder AND synchronized to `studio` site eventually.)
+- everything exported must also be eventually uploaded to `sftp` site
+
+This eventual synchronization between `studio` and `sftp` sites must be physically handled by background process.
+
+As current implementation relies heavily on Settings and Local Settings, background process for a specific site ('studio' for example) must be configured via Tray first to `syncserver` command to work.
+
+To do this:
+
+- run OP `Tray` with environment variable OPENPYPE_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.)
+- start `Tray`
+- check `Local ID` in information dialog after clicking on version number in the Tray
+- open `Local Settings` in the `Tray`
+- configure for each project necessary active site and remote site
+- close `Tray`
+- run OP from a command line with `syncserver` and `--active_site` arguments
+
+
+This is an example how to trigger background synching process where active (source) site is `studio`. 
+(It is expected that OP is installed on a machine, `openpype_console` is on PATH. If not, add full path to executable.
+)
+```shell
+openpype_console syncserver --active_site studio
+```