From 1aafb697e44888074de1abad59bde216429d459e Mon Sep 17 00:00:00 2001
From: Petr Kalis <petr.kalis@gmail.com>
Date: Tue, 11 May 2021 14:49:13 +0200
Subject: [PATCH] SyncServer - change settings to new format

Sites are configured in System
Schemas and defaults were modified
All providers carry dict of modifiable properties for Local Settings
---
 .../providers/abstract_provider.py            |  27 +--
 .../modules/sync_server/providers/gdrive.py   |  55 ++++--
 openpype/modules/sync_server/providers/lib.py |  11 ++
 .../sync_server/providers/local_drive.py      |  19 +-
 .../modules/sync_server/sync_server_module.py | 180 +++++++++++++-----
 openpype/modules/sync_server/tray/lib.py      |   2 +-
 openpype/modules/sync_server/tray/widgets.py  |   2 +-
 .../defaults/project_settings/global.json     |   7 -
 .../defaults/system_settings/modules.json     |   3 +-
 openpype/settings/entities/enum_entity.py     |  28 +--
 .../schema_project_syncserver.json            |   5 -
 11 files changed, 212 insertions(+), 127 deletions(-)

diff --git a/openpype/modules/sync_server/providers/abstract_provider.py b/openpype/modules/sync_server/providers/abstract_provider.py
index 7cd42fb4fa..2e9632134c 100644
--- a/openpype/modules/sync_server/providers/abstract_provider.py
+++ b/openpype/modules/sync_server/providers/abstract_provider.py
@@ -27,37 +27,16 @@ class AbstractProvider:
             (boolean)
         """
 
+    @classmethod
     @abc.abstractmethod
-    def set_editable_properties(self):
+    def get_configurable_items(cls):
         """
-            Sets dictionary of editable properties with scopes.
+            Returns filtered dict of editable properties
 
-            Example:
-                { 'credentials_url': {'scopes': [utils.EditableScopes.SYSTEM],
-                                      'type': 'text'}}
-        """
-
-    @abc.abstractmethod
-    def get_editable_properties(self, scopes):
-        """
-            Returns filtered list of editable properties
-
-            Args:
-                scopes (list) of utils.EditableScopes (optional - filter on)
 
             Returns:
                 (dict)
         """
-        if not scopes:
-            return self._editable_properties
-
-        editable = {}
-        for scope in scopes:
-            for key, properties in self._editable_properties.items():
-                if scope in properties['scope']:
-                    editable[key] = properties
-
-        return editable
 
     @abc.abstractmethod
     def upload_file(self, source_path, path,
diff --git a/openpype/modules/sync_server/providers/gdrive.py b/openpype/modules/sync_server/providers/gdrive.py
index f2c18a04a9..e79927590b 100644
--- a/openpype/modules/sync_server/providers/gdrive.py
+++ b/openpype/modules/sync_server/providers/gdrive.py
@@ -1,22 +1,32 @@
 from __future__ import print_function
 import os.path
-from googleapiclient.discovery import build
-import google.oauth2.service_account as service_account
-from googleapiclient import errors
-from .abstract_provider import AbstractProvider
-from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
+import time
+import sys
+from setuptools.extern import six
+
 from openpype.api import Logger
 from openpype.api import get_system_settings
+from .abstract_provider import AbstractProvider
 from ..utils import time_function, ResumableError, EditableScopes
-import time
 
+log = Logger().get_logger("SyncServer")
+
+try:
+    from googleapiclient.discovery import build
+    import google.oauth2.service_account as service_account
+    from googleapiclient import errors
+    from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
+except (ImportError, SyntaxError):
+    if six.PY3:
+        six.reraise(*sys.exc_info())
+
+    # handle imports from Python 2 hosts - in those only basic methods are used
+    log.warning("Import failed, imported from Python 2, operations will fail.")
 
 SCOPES = ['https://www.googleapis.com/auth/drive.metadata.readonly',
           'https://www.googleapis.com/auth/drive.file',
           'https://www.googleapis.com/auth/drive.readonly']  # for write|delete
 
-log = Logger().get_logger("SyncServer")
-
 
 class GDriveHandler(AbstractProvider):
     """
@@ -54,8 +64,7 @@ class GDriveHandler(AbstractProvider):
         self.active = False
         self.project_name = project_name
         self.site_name = site_name
-
-        self._editable_properties = {}
+        self.service = None
 
         self.presets = presets
         if not self.presets:
@@ -63,7 +72,7 @@ class GDriveHandler(AbstractProvider):
                      format(site_name))
             return
 
-        if not os.path.exists(self.presets["credentials_url"]):
+        if not os.path.exists(self.presets.get("credentials_url", "")):
             log.info("Sync Server: No credentials for Gdrive provider! ")
             return
 
@@ -78,7 +87,6 @@ class GDriveHandler(AbstractProvider):
 
         self._tree = tree
         self.active = True
-        self.set_editable_properties()
 
     def is_active(self):
         """
@@ -86,18 +94,29 @@ class GDriveHandler(AbstractProvider):
         Returns:
             (boolean)
         """
-        return self.active
+        return self.service is not None
 
-    def set_editable_properties(self):
+    @classmethod
+    def get_configurable_items(cls):
+        """
+            Returns filtered dict of editable properties.
+
+
+            Returns:
+                (dict)
+        """
         editable = {
+            # credentials could be override on Project or User level
             'credential_url': {'scope': [EditableScopes.PROJECT,
                                          EditableScopes.LOCAL],
+                               'label': "Credentials url",
                                'type': 'text'},
-
-            'roots': {'scope': [EditableScopes.PROJECT],
-                      'type': 'dict'}
+            # roots could be override only on Project leve, User cannot
+            'root': {'scope': [EditableScopes.PROJECT],
+                     'label': "Roots",
+                     'type': 'dict'}
         }
-        self._editable_properties = editable
+        return editable
 
     def get_roots_config(self, anatomy=None):
         """
diff --git a/openpype/modules/sync_server/providers/lib.py b/openpype/modules/sync_server/providers/lib.py
index f9c4309724..816ccca981 100644
--- a/openpype/modules/sync_server/providers/lib.py
+++ b/openpype/modules/sync_server/providers/lib.py
@@ -65,6 +65,17 @@ class ProviderFactory:
         info = self._get_creator_info(provider)
         return info[1]
 
+    def get_provider_configurable_items(self, provider):
+        """
+            Returns dict of modifiable properties for 'provider'.
+
+            Provider contains information which its properties and on what
+            level could be override
+        """
+        provider_info = self._get_creator_info(provider)
+
+        return provider_info[0].get_configurable_items()
+
     def _get_creator_info(self, provider):
         """
             Collect all necessary info for provider. Currently only creator
diff --git a/openpype/modules/sync_server/providers/local_drive.py b/openpype/modules/sync_server/providers/local_drive.py
index 2a96094f22..2d37d0e1c4 100644
--- a/openpype/modules/sync_server/providers/local_drive.py
+++ b/openpype/modules/sync_server/providers/local_drive.py
@@ -25,18 +25,25 @@ class LocalDriveHandler(AbstractProvider):
         self._editable_properties = {}
 
         self.active = self.is_active()
-        self.set_editable_properties()
 
     def is_active(self):
         return True
 
-    def set_editable_properties(self):
+    @classmethod
+    def get_configurable_items(cls):
+        """
+            Returns filtered dict of editable properties
+
+            Returns:
+                (dict)
+        """
         editable = {
-            'roots': {'scope': [EditableScopes.PROJECT,
-                                EditableScopes.LOCAL],
-                      'type': 'dict'}
+            'root': {'scope': [EditableScopes.PROJECT,
+                               EditableScopes.LOCAL],
+                     'label': "Roots",
+                     'type': 'dict'}
         }
-        self._editable_properties = editable
+        return editable
 
     def upload_file(self, source_path, target_path,
                     server, collection, file, representation, site,
diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py
index e29861c20c..1e12db84a1 100644
--- a/openpype/modules/sync_server/sync_server_module.py
+++ b/openpype/modules/sync_server/sync_server_module.py
@@ -9,10 +9,12 @@ from .. import PypeModule, ITrayModule
 from openpype.api import (
     Anatomy,
     get_project_settings,
+    get_system_settings,
     get_local_site_id)
 from openpype.lib import PypeLogger
 
 from .providers.local_drive import LocalDriveHandler
+from .providers import lib
 
 from .utils import time_function, SyncStatus
 
@@ -390,31 +392,99 @@ class SyncServerModule(PypeModule, ITrayModule):
 
         return remote_site
 
-    def get_configurable_items(self):
-        pass
-
-    def get_configurable_items_for_site(self, project_name, site_name):
+    def get_configurable_items(self, scope=None):
         """
-            Returns list of items that should be configurable by User
+            Returns list of items that could be configurable for all projects.
+
+            Could be filtered by 'scope' argument (list)
+
+            Args:
+                scope (list of utils.EditableScope) (optional)
 
             Returns:
-                (list of dict)
-                [{key:"root", label:"root", value:"valueFromSettings"}]
+                (dict of dict)
+                {projectA: {
+                    siteA : {
+                        key:"root", label:"root", value:"valueFromSettings"
+                    }
+                }
         """
-        # sites = set(self.get_active_sites(project_name),
-        #             self.get_remote_sites(project_name))
-        # for site in sites:
-        #     if site_name
-
-    def _get_configurable_items_for_project(self, project_name):
-        from .providers import lib
-        sites = set(self.get_active_sites(project_name),
-                    self.get_remote_sites(project_name))
         editable = {}
-        for site in sites:
-            provider_name = self.get_provider_for_site(project_name, site)
+        for project in self.connection.projects():
+            project_name = project["name"]
+            items = self.get_configurable_items_for_project(project_name,
+                                                            scope)
+            editable.update(items)
 
+        return editable
 
+    def get_configurable_items_for_project(self, project_name, scope=None):
+        """
+            Returns list of items that could be configurable for specific
+            'project_name'
+
+            Args:
+                project_name (str)
+                scope (list of utils.EditableScope) (optional)
+
+            Returns:
+                (dict of dict)
+                {projectA: {
+                    siteA : {
+                        key:"root", label:"root", value:"valueFromSettings"
+                    }
+                }
+        """
+        sites = set(self.get_active_sites(project_name)) | \
+            set(self.get_remote_sites(project_name))
+        editable = {}
+        for site_name in sites:
+            items = self.get_configurable_items_for_site(project_name,
+                                                         site_name,
+                                                         scope)
+            editable.update(items)
+
+        return editable
+
+    def get_configurable_items_for_site(self, project_name, site_name,
+                                        scope=None):
+        """
+            Returns list of items that could be configurable.
+
+            Args:
+                project_name (str)
+                site_name (str)
+                scope (list of utils.EditableScope) (optional)
+
+            Returns:
+                (dict of dict)
+                {projectA: {
+                    siteA : {
+                        key:"root", label:"root", value:"valueFromSettings"
+                    }
+                }
+        """
+        provider_name = self.get_provider_for_site(site=site_name)
+        items = lib.factory.get_provider_configurable_items(provider_name,
+                                                            scope)
+
+        if not scope:
+            return {project_name: {site_name: items}}
+
+        editable = {}
+        sync_s = self.get_sync_project_setting(project_name, True)
+        for scope in set([scope]):
+            for key, properties in items.items():
+                if scope in properties['scope']:
+                    val = sync_s.get("sites", {}).get(site_name, {}).get(key)
+                    editable = {
+                        "key": key,
+                        "value": val,
+                        "label": properties.get("label"),
+                        "type": properties.get("type"),
+                    }
+
+        return {project_name: {site_name: editable}}
 
     def reset_timer(self):
         """
@@ -432,7 +502,7 @@ class SyncServerModule(PypeModule, ITrayModule):
             for project in self.connection.projects():
                 project_name = project["name"]
                 project_settings = self.get_sync_project_setting(project_name)
-                if project_settings:
+                if project_settings and project_settings.get("enabled"):
                     enabled_projects.append(project_name)
 
         return enabled_projects
@@ -584,55 +654,60 @@ class SyncServerModule(PypeModule, ITrayModule):
 
         return self._sync_project_settings
 
-    def set_sync_project_settings(self):
+    def set_sync_project_settings(self, exclude_locals=False):
         """
             Set sync_project_settings for all projects (caching)
-
+            Args:
+                exclude_locals (bool): ignore overrides from Local Settings
             For performance
         """
         sync_project_settings = {}
 
+        # sites are now configured system wide
+        sys_sett = get_system_settings()
+        sync_sett = sys_sett["modules"].get("sync_server")
+        system_sites = {}
+        for site, detail in sync_sett.get("sites", {}).items():
+            system_sites[site] = detail
+
         for collection in self.connection.database.collection_names(False):
             sync_settings = self._parse_sync_settings_from_settings(
-                get_project_settings(collection))
-            if sync_settings:
-                default_sites = self._get_default_site_configs()
-                sync_settings['sites'].update(default_sites)
-                sync_project_settings[collection] = sync_settings
+                get_project_settings(collection,
+                                     exclude_locals=exclude_locals))
+
+            default_sites = self._get_default_site_configs()
+            sync_settings['sites'].update(default_sites)
+            sync_settings['sites'].update(system_sites)
+            sync_project_settings[collection] = sync_settings
 
         if not sync_project_settings:
             log.info("No enabled and configured projects for sync.")
 
         self._sync_project_settings = sync_project_settings
 
-    def get_sync_project_setting(self, project_name):
+    def get_sync_project_setting(self, project_name, exclude_locals=False):
         """ Handles pulling sync_server's settings for enabled 'project_name'
 
             Args:
                 project_name (str): used in project settings
+                exclude_locals (bool): ignore overrides from Local Settings
             Returns:
                 (dict): settings dictionary for the enabled project,
                     empty if no settings or sync is disabled
         """
         # presets set already, do not call again and again
         # self.log.debug("project preset {}".format(self.presets))
-        if self.sync_project_settings and \
-                self.sync_project_settings.get(project_name):
-            return self.sync_project_settings.get(project_name)
+        if not self.sync_project_settings or \
+               not self.sync_project_settings.get(project_name):
+            self.set_sync_project_settings(project_name, exclude_locals)
 
-        settings = get_project_settings(project_name)
-        return self._parse_sync_settings_from_settings(settings)
+        return self.sync_project_settings.get(project_name)
 
     def _parse_sync_settings_from_settings(self, settings):
         """ settings from api.get_project_settings, TOOD rename """
         sync_settings = settings.get("global").get("sync_server")
-        if not sync_settings:
-            log.info("No project setting not syncing.")
-            return {}
-        if sync_settings.get("enabled"):
-            return sync_settings
 
-        return {}
+        return sync_settings
 
     def _get_default_site_configs(self):
         """
@@ -643,16 +718,29 @@ class SyncServerModule(PypeModule, ITrayModule):
                      get_local_site_id(): default_config}
         return all_sites
 
-    def get_provider_for_site(self, project_name, site):
+    def get_provider_for_site(self, project_name=None, site=None):
         """
-            Return provider name for site.
+            Return provider name for site (unique name across all projects.
         """
-        site_preset = self.get_sync_project_setting(project_name)["sites"].\
-            get(site)
-        if site_preset:
-            return site_preset["provider"]
+        sites = {self.DEFAULT_SITE: "local_drive",
+                 self.LOCAL_SITE: "local_drive",
+                 get_local_site_id(): "local_drive"}
 
-        return "NA"
+        if site in sites.keys():
+            return sites[site]
+
+        if project_name:  # backward compatibility
+            proj_settings = self.get_sync_project_setting(project_name)
+            provider = proj_settings.get("sites", {}).get(site, {}).\
+                get("provider")
+            return provider
+
+        sys_sett = get_system_settings()
+        sync_sett = sys_sett["modules"].get("sync_server")
+        for site, detail in sync_sett.get("sites", {}).items():
+            sites[site] = detail.get("provider")
+
+        return sites.get(site, 'N/A')
 
     @time_function
     def get_sync_representations(self, collection, active_site, remote_site):
@@ -1130,7 +1218,7 @@ class SyncServerModule(PypeModule, ITrayModule):
                              format(site_name))
             return
 
-        provider_name = self.get_provider_for_site(collection, site_name)
+        provider_name = self.get_provider_for_site(site=site_name)
 
         if provider_name == 'local_drive':
             query = {
diff --git a/openpype/modules/sync_server/tray/lib.py b/openpype/modules/sync_server/tray/lib.py
index c1f8eaf629..25c600abd2 100644
--- a/openpype/modules/sync_server/tray/lib.py
+++ b/openpype/modules/sync_server/tray/lib.py
@@ -158,7 +158,7 @@ def translate_provider_for_icon(sync_server, project, site):
     """
     if site == sync_server.DEFAULT_SITE:
         return sync_server.DEFAULT_SITE
-    return sync_server.get_provider_for_site(project, site)
+    return sync_server.get_provider_for_site(site=site)
 
 
 def get_item_by_id(model, object_id):
diff --git a/openpype/modules/sync_server/tray/widgets.py b/openpype/modules/sync_server/tray/widgets.py
index e80f91e09f..eae912206e 100644
--- a/openpype/modules/sync_server/tray/widgets.py
+++ b/openpype/modules/sync_server/tray/widgets.py
@@ -236,7 +236,7 @@ class _SyncRepresentationWidget(QtWidgets.QWidget):
 
         for site, progress in {active_site: local_progress,
                                remote_site: remote_progress}.items():
-            provider = self.sync_server.get_provider_for_site(project, site)
+            provider = self.sync_server.get_provider_for_site(site=site)
             if provider == 'local_drive':
                 if 'studio' in site:
                     txt = " studio version"
diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json
index 48b7a24b0d..1f54bed03c 100644
--- a/openpype/settings/defaults/project_settings/global.json
+++ b/openpype/settings/defaults/project_settings/global.json
@@ -267,13 +267,6 @@
             "remote_site": "studio"
         },
         "sites": {
-            "gdrive": {
-                "provider": "gdrive",
-                "credentials_url": "",
-                "root": {
-                    "work": ""
-                }
-            }
         }
     },
     "project_plugins": {
diff --git a/openpype/settings/defaults/system_settings/modules.json b/openpype/settings/defaults/system_settings/modules.json
index 6e4b493116..5c4aa6c485 100644
--- a/openpype/settings/defaults/system_settings/modules.json
+++ b/openpype/settings/defaults/system_settings/modules.json
@@ -135,7 +135,8 @@
         "workspace_name": ""
     },
     "sync_server": {
-        "enabled": false
+        "enabled": false,
+        "sites": {}
     },
     "deadline": {
         "enabled": true,
diff --git a/openpype/settings/entities/enum_entity.py b/openpype/settings/entities/enum_entity.py
index a5492cd727..c6021b68de 100644
--- a/openpype/settings/entities/enum_entity.py
+++ b/openpype/settings/entities/enum_entity.py
@@ -231,21 +231,17 @@ class ProvidersEnum(BaseEnumEntity):
         self.placeholder = None
 
     def _get_enum_values(self):
-        # from openpype.modules.sync_server.providers import lib as lib_providers
-        #
-        # providers = lib_providers.factory.providers
-        #
-        # valid_keys = set()
-        # enum_items = []
-        # for provider_code, provider_info in providers.items():
-        #     provider, _ = provider_info
-        #     enum_items.append({provider_code: provider.LABEL})
-        #     valid_keys.add(provider_code)
+        from openpype.modules.sync_server.providers import lib as lib_providers
+
+        providers = lib_providers.factory.providers
+
         valid_keys = set()
-        enum_items = []
-        if not valid_keys:
-            enum_items.append({'': 'N/A'})
-            valid_keys.add('')
+        valid_keys.add('')
+        enum_items = [{'': 'Choose Provider'}]
+        for provider_code, provider_info in providers.items():
+            provider, _ = provider_info
+            enum_items.append({provider_code: provider.LABEL})
+            valid_keys.add(provider_code)
 
         return enum_items, valid_keys
 
@@ -259,7 +255,3 @@ class ProvidersEnum(BaseEnumEntity):
             self._current_value = value_on_not_set
 
         self.value_on_not_set = value_on_not_set
-
-
-# class ActiveSiteEnum
-# class RemoteSiteEnum
\ No newline at end of file
diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json
index ea1b8fc9da..bb5ebea45a 100644
--- a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json
+++ b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json
@@ -49,11 +49,6 @@
         {
             "type": "dict",
             "children": [
-            {
-                "type": "text",
-                "key": "provider",
-                "label": "Provider"
-            },
             {
                 "type": "text",
                 "key": "credentials_url",