Merge pull request #2206 from pypeclub/feature/OP-1937_Add-alternative-site-for-Site-Sync

Add alternative sites for Site Sync

openpype/modules/default_modules/sync_server/providers/sftp.py

@@ -1,8 +1,6 @@
 import os
 import os.path
 import time
-import sys
-import six
 import threading
 import platform
 
@@ -14,6 +12,7 @@ log = Logger().get_logger("SyncServer")
 pysftp = None
 try:
     import pysftp
+    import paramiko
 except (ImportError, SyntaxError):
     pass
 
@@ -37,7 +36,6 @@ class SFTPHandler(AbstractProvider):
 
     def __init__(self, project_name, site_name, tree=None, presets=None):
         self.presets = None
-        self.active = False
         self.project_name = project_name
         self.site_name = site_name
         self.root = None
@@ -64,7 +62,6 @@ class SFTPHandler(AbstractProvider):
         self.sftp_key_pass = provider_presets["sftp_key_pass"]
 
         self._tree = None
-        self.active = True
 
     @property
     def conn(self):
@@ -80,7 +77,9 @@ class SFTPHandler(AbstractProvider):
         Returns:
             (boolean)
         """
-        return self.conn is not None
+        return self.presets.get(self.CODE) and \
+            self.presets[self.CODE].get("sftp_host") and \
+            self.conn is not None
 
     @classmethod
     def get_system_settings_schema(cls):
@@ -108,7 +107,7 @@ class SFTPHandler(AbstractProvider):
         editable = [
             # credentials could be overriden on Project or User level
             {
-                'key': "sftp_server",
+                'key': "sftp_host",
                 'label': "SFTP host name",
                 'type': 'text'
             },
@@ -426,7 +425,10 @@ class SFTPHandler(AbstractProvider):
         if self.sftp_key_pass:
             conn_params['private_key_pass'] = self.sftp_key_pass
 
-        return pysftp.Connection(**conn_params)
+        try:
+            return pysftp.Connection(**conn_params)
+        except paramiko.ssh_exception.SSHException:
+            log.warning("Couldn't connect", exc_info=True)
 
     def _mark_progress(self, collection, file, representation, server, site,
                        source_path, target_path, direction):

openpype/modules/default_modules/sync_server/sync_server.py

@@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name,
                                          remote_site_name,
                                          True
                                          )
+
+    module.handle_alternate_site(collection, representation, remote_site_name,
+                                 file["_id"], file_id)
+
     return file_id
 
 
@@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name,
                                          local_site,
                                          True
                                          )
+
+    module.handle_alternate_site(collection, representation, remote_site_name,
+                                 file["_id"], file_id)
+
     return file_id
 
 

openpype/modules/default_modules/sync_server/sync_server_module.py

@@ -109,6 +109,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         # some parts of code need to run sequentially, not in async
         self.lock = None
+        self._sync_system_settings = None
         # settings for all enabled projects for sync
         self._sync_project_settings = None
         self.sync_server_thread = None  # asyncio requires new thread
@@ -769,6 +770,57 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
                     enabled_projects.append(project_name)
 
         return enabled_projects
+
+    def handle_alternate_site(self, collection, representation, processed_site,
+                              file_id, synced_file_id):
+        """
+            For special use cases where one site vendors another.
+
+            Current use case is sftp site vendoring (exposing) same data as
+            regular site (studio). Each site is accessible for different
+            audience. 'studio' for artists in a studio, 'sftp' for externals.
+
+            Change of file status on one site actually means same change on
+            'alternate' site. (eg. artists publish to 'studio', 'sftp' is using
+            same location >> file is accesible on 'sftp' site right away.
+
+            Args:
+                collection (str): name of project
+                representation (dict)
+                processed_site (str): real site_name of published/uploaded file
+                file_id (ObjectId): DB id of file handled
+                synced_file_id (str): id of the created file returned
+                    by provider
+        """
+        sites = self.sync_system_settings.get("sites", {})
+        sites[self.DEFAULT_SITE] = {"provider": "local_drive",
+                                    "alternative_sites": []}
+
+        alternate_sites = []
+        for site_name, site_info in sites.items():
+            conf_alternative_sites = site_info.get("alternative_sites", [])
+            if processed_site in conf_alternative_sites:
+                alternate_sites.append(site_name)
+                continue
+            if processed_site == site_name and conf_alternative_sites:
+                alternate_sites.extend(conf_alternative_sites)
+                continue
+
+        alternate_sites = set(alternate_sites)
+
+        for alt_site in alternate_sites:
+            query = {
+                "_id": representation["_id"]
+            }
+            elem = {"name": alt_site,
+                    "created_dt": datetime.now(),
+                    "id": synced_file_id}
+
+            self.log.debug("Adding alternate {} to {}".format(
+                alt_site, representation["_id"]))
+            self._add_site(collection, query,
+                           [representation], elem,
+                           site_name, file_id=file_id, force=True)
     """ End of Public API """
 
     def get_local_file_path(self, collection, site_name, file_path):
@@ -826,7 +878,6 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         self.sync_server_thread = SyncServerThread(self)
 
-
     def tray_start(self):
         """
             Triggered when Tray is started.
@@ -908,6 +959,14 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
 
         return self._connection
 
+    @property
+    def sync_system_settings(self):
+        if self._sync_system_settings is None:
+            self._sync_system_settings = get_system_settings()["modules"].\
+                get("sync_server")
+
+        return self._sync_system_settings
+
     @property
     def sync_project_settings(self):
         if self._sync_project_settings is None:
@@ -993,9 +1052,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
                 (dict): {'studio': {'provider':'local_drive'...},
                          'MY_LOCAL': {'provider':....}}
         """
-        sys_sett = get_system_settings()
-        sync_sett = sys_sett["modules"].get("sync_server")
-
+        sync_sett = self.sync_system_settings
         project_enabled = True
         if project_name:
             project_enabled = project_name in self.get_enabled_projects()
@@ -1053,10 +1110,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
             if provider:
                 return provider
 
-        sys_sett = get_system_settings()
-        sync_sett = sys_sett["modules"].get("sync_server")
-        for site, detail in sync_sett.get("sites", {}).items():
-            sites[site] = detail.get("provider")
+        sync_sett = self.sync_system_settings
+        for conf_site, detail in sync_sett.get("sites", {}).items():
+            sites[conf_site] = detail.get("provider")
 
         return sites.get(site, 'N/A')
 
@@ -1423,9 +1479,12 @@ class SyncServerModule(OpenPypeModule, ITrayModule):
         update = {
             "$set": {"files.$[f].sites.$[s]": elem}
         }
+        if not isinstance(file_id, ObjectId):
+            file_id = ObjectId(file_id)
+
         arr_filter = [
             {'s.name': site_name},
-            {'f._id': ObjectId(file_id)}
+            {'f._id': file_id}
         ]
 
         self._update_site(collection, query, update, arr_filter)

openpype/modules/default_modules/sync_server/tray/widgets.py

@@ -154,7 +154,7 @@ class SyncProjectListWidget(QtWidgets.QWidget):
            selected_index.isValid() and \
            not self._selection_changed:
             mode = QtCore.QItemSelectionModel.Select | \
-                   QtCore.QItemSelectionModel.Rows
+                QtCore.QItemSelectionModel.Rows
             self.project_list.selectionModel().select(selected_index, mode)
 
         if self.current_project:

openpype/plugins/publish/integrate_new.py

@@ -1030,31 +1030,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         local_site = 'studio'  # default
         remote_site = None
         always_accesible = []
-        sync_server_presets = None
-
-        if (instance.context.data["system_settings"]
-                                 ["modules"]
-                                 ["sync_server"]
-                                 ["enabled"]):
-            sync_server_presets = (instance.context.data["project_settings"]
-                                                        ["global"]
-                                                        ["sync_server"])
-
-            local_site_id = openpype.api.get_local_site_id()
-            if sync_server_presets["enabled"]:
-                local_site = sync_server_presets["config"].\
-                    get("active_site", "studio").strip()
-                always_accesible = sync_server_presets["config"].\
-                    get("always_accessible_on", [])
-                if local_site == 'local':
-                    local_site = local_site_id
-
-                remote_site = sync_server_presets["config"].get("remote_site")
-                if remote_site == local_site:
-                    remote_site = None
-
-                if remote_site == 'local':
-                    remote_site = local_site_id
+        sync_project_presets = None
 
         rec = {
             "_id": io.ObjectId(),
@@ -1069,18 +1045,92 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin):
         if sites:
             rec["sites"] = sites
         else:
+            system_sync_server_presets = (
+                instance.context.data["system_settings"]
+                                     ["modules"]
+                                     ["sync_server"])
+            log.debug("system_sett:: {}".format(system_sync_server_presets))
+
+            if system_sync_server_presets["enabled"]:
+                sync_project_presets = (
+                    instance.context.data["project_settings"]
+                                         ["global"]
+                                         ["sync_server"])
+
+            if sync_project_presets and sync_project_presets["enabled"]:
+                local_site, remote_site = self._get_sites(sync_project_presets)
+
+                always_accesible = sync_project_presets["config"]. \
+                    get("always_accessible_on", [])
+
+            already_attached_sites = {}
             meta = {"name": local_site, "created_dt": datetime.now()}
             rec["sites"] = [meta]
+            already_attached_sites[meta["name"]] = meta["created_dt"]
 
-            if remote_site:
+            if sync_project_presets and sync_project_presets["enabled"]:
+                # add remote
                 meta = {"name": remote_site.strip()}
                 rec["sites"].append(meta)
+                already_attached_sites[meta["name"]] = None
 
-            # add skeleton for site where it should be always synced to
-            for always_on_site in always_accesible:
-                if always_on_site not in [local_site, remote_site]:
-                    meta = {"name": always_on_site.strip()}
+                # add skeleton for site where it should be always synced to
+                for always_on_site in always_accesible:
+                    if always_on_site not in already_attached_sites.keys():
+                        meta = {"name": always_on_site.strip()}
+                        rec["sites"].append(meta)
+                        already_attached_sites[meta["name"]] = None
+
+                # add alternative sites
+                rec = self._add_alternative_sites(system_sync_server_presets,
+                                                  already_attached_sites,
+                                                  rec)
+
+            log.debug("final sites:: {}".format(rec["sites"]))
+
+        return rec
+
+    def _get_sites(self, sync_project_presets):
+        """Returns tuple (local_site, remote_site)"""
+        local_site_id = openpype.api.get_local_site_id()
+        local_site = sync_project_presets["config"]. \
+            get("active_site", "studio").strip()
+
+        if local_site == 'local':
+            local_site = local_site_id
+
+        remote_site = sync_project_presets["config"].get("remote_site")
+        if remote_site == local_site:
+            remote_site = None
+
+        if remote_site == 'local':
+            remote_site = local_site_id
+
+        return local_site, remote_site
+
+    def _add_alternative_sites(self,
+                               system_sync_server_presets,
+                               already_attached_sites,
+                               rec):
+        """Loop through all configured sites and add alternatives.
+
+            See SyncServerModule.handle_alternate_site
+        """
+        conf_sites = system_sync_server_presets.get("sites", {})
+
+        for site_name, site_info in conf_sites.items():
+            alt_sites = set(site_info.get("alternative_sites", []))
+            for added_site in already_attached_sites.keys():
+                if added_site in alt_sites:
+                    if site_name in already_attached_sites.keys():
+                        continue
+                    meta = {"name": site_name}
+                    real_created = already_attached_sites[added_site]
+                    # alt site inherits state of 'created_dt'
+                    if real_created:
+                        meta["created_dt"] = real_created
                     rec["sites"].append(meta)
+                    already_attached_sites[meta["name"]] = real_created
 
         return rec
 

openpype/settings/entities/dict_conditional.py

@@ -762,6 +762,17 @@ class SyncServerProviders(DictConditionalEntity):
 
         enum_children = []
         for provider_code, configurables in system_settings_schema.items():
+            # any site could be exposed or vendorized by different site
+            # eg studio site content could be mapped on sftp site, single file
+            # accessible via 2 different protocols (sites)
+            configurables.append(
+                {
+                    "type": "list",
+                    "key": "alternative_sites",
+                    "label": "Alternative sites",
+                    "object_type": "text"
+                }
+            )
             label = provider_code_to_label.get(provider_code) or provider_code
 
             enum_children.append({

openpype/tests/mongo_performance.py

@@ -104,8 +104,8 @@ class TestPerformance():
                         "name": "mb",
                         "parent": {"oid": '{}'.format(id)},
                         "data": {
-                            "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name),  # noqa
-                            "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"  # noqa
+                            "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name),  # noqa: E501
+                            "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}"  # noqa: E501
                         },
                         "type": "representation",
                         "schema": "openpype:representation-2.0"
@@ -188,21 +188,21 @@ class TestPerformance():
                         create_files=False):
         ret = [
             {
-                 "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                 "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                  "_id": '{}'.format(file_id),
                  "hash": "temphash",
                  "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
                  "size": random.randint(0, self.MAX_FILE_SIZE_B)
             },
             {
-                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                 "_id": '{}'.format(file_id2),
                 "hash": "temphash",
                 "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
                 "size": random.randint(0, self.MAX_FILE_SIZE_B)
             },
             {
-                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i),  #noqa
+                "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i),  # noqa: E501
                 "_id": '{}'.format(file_id3),
                 "hash": "temphash",
                 "sites": self.get_sites(self.MAX_NUMBER_OF_SITES),
@@ -223,8 +223,8 @@ class TestPerformance():
         ret = {}
         ret['{}'.format(file_id)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236
@@ -232,16 +232,16 @@ class TestPerformance():
 
         ret['{}'.format(file_id2)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236
         }
         ret['{}'.format(file_id3)] = {
             "path": "{root[work]}" +
-                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  #noqa
-                    "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa
+                    "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/"  # noqa: E501
+                    "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i),  # noqa: E501
             "hash": "temphash",
             "sites": ["studio"],
             "size": 87236

website/docs/module_site_sync.md

@@ -27,6 +27,38 @@ To use synchronization, *Site Sync* needs to be enabled globally in **OpenPype S
 
 ![Configure module](assets/site_sync_system.png)
 
+### Sites 
+
+By default there are two sites created for each OpenPype installation:
+- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled.
+- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID.
+
+Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. 
+
+Many different sites can be created and configured on the system level, and some or all can be assigned to each project.
+
+Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup).
+
+Sites could be configured differently per project basis. 
+
+Each new site needs to be created first in `System Settings`. Most important feature of site is its Provider, select one from already prepared Providers.
+
+#### Alternative sites 
+
+This attribute is meant for special use cases only.
+
+One of the use cases is sftp site vendoring (exposing) same data as regular site (studio). Each site is accessible for different audience. 'studio' for artists in a studio via shared disk, 'sftp' for externals via sftp server with mounted 'studio' drive.
+
+Change of file status on one site actually means same change on 'alternate' site occured too. (eg. artists publish to 'studio', 'sftp' is using
+same location >> file is accessible on 'sftp' site right away, no need to sync it anyhow.)
+
+##### Example
+![Configure module](assets/site_sync_system_sites.png)
+Admin created new `sftp` site which is handled by `SFTP` provider. Somewhere in the studio SFTP server is deployed on a machine that has access to `studio` drive.
+
+Alternative sites work both way:
+- everything published to `studio` is accessible on a `sftp` site too
+- everything published to `sftp` (most probably via artist's local disk - artists publishes locally, representation is marked to be synced to `sftp`. Immediately after it is synced, it is marked to be available on `studio` too for artists in the studio to use.)
 
 ## Project Settings
 
@@ -45,21 +77,6 @@ Artists can also override which site they use as active and remote if need be.
 ![Local overrides](assets/site_sync_local_setting.png)
 
 
-## Sites 
-
-By default there are two sites created for each OpenPype installation:
-- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled.
-- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID.
-
-Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. 
-
-Many different sites can be created and configured on the system level, and some or all can be assigned to each project.
-
-Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup).
-
-Sites could be configured differently per project basis. 
-
-
 ## Providers
 
 Each site implements a so called `provider` which handles most common operations (list files, copy files etc.) and provides interface with a particular type of storage. (disk, gdrive, aws, etc.)