From ee885051d915a20fe5e004a27c40a246f1e156de Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 30 Mar 2022 12:35:27 +0200 Subject: [PATCH 01/10] Added compute_resource_sync_sites to sync_server_module This method will be used in integrate_new to logically separate Site Sync parts. --- .../modules/sync_server/sync_server_module.py | 107 +++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index caf58503f1..7126c17e17 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -157,7 +157,6 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation_id, site_name=site_name, force=force) - # public facing API def remove_site(self, collection, representation_id, site_name, remove_local_files=False): """ @@ -184,6 +183,112 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if remove_local_files: self._remove_local_file(collection, representation_id, site_name) + def compute_resource_sync_sites(self, project_name): + """Get available resource sync sites state for publish process. + + Returns dict with prepared state of sync sites for 'project_name'. + It checks if Site Sync is enabled, handles alternative sites. + Publish process stores this dictionary as a part of representation + document in DB. + + Example: + [ + { + 'name': '42abbc09-d62a-44a4-815c-a12cd679d2d7', + 'created_dt': datetime.datetime(2022, 3, 30, 12, 16, 9, 778637) + }, + {'name': 'studio'}, + {'name': 'SFTP'} + ] -- representation is published locally, artist or Settings have set + remote site as 'studio'. 'SFTP' is alternate site to 'studio'. Eg. + whenever file is on 'studio', it is also on 'SFTP'. + """ + + def create_metadata(name, created=True): + """Create sync site metadata for site with `name`""" + metadata = {"name": name} + if created: + metadata["created_dt"] = datetime.now() + return metadata + + if ( + not self.sync_system_settings["enabled"] or + not self.sync_project_settings[project_name]["enabled"]): + return [create_metadata(self.DEFAULT_SITE)] + + local_site = self.get_active_site(project_name) + remote_site = self.get_remote_site(project_name) + + # Attached sites metadata by site name + # That is the local site, remote site, the always accesible sites + # and their alternate sites (alias of sites with different protocol) + attached_sites = dict() + attached_sites[local_site] = create_metadata(local_site) + + if remote_site and remote_site not in attached_sites: + attached_sites[remote_site] = create_metadata(remote_site, + created=False) + + # add skeleton for sites where it should be always synced to + # usually it would be a backup site which is handled by separate + # background process + for site in self._get_always_accessible_sites(project_name): + if site not in attached_sites: + attached_sites[site] = create_metadata(site, created=False) + + attached_sites = self._add_alternative_sites(attached_sites) + + return list(attached_sites.values()) + + def _get_always_accessible_sites(self, project_name): + """Sites that synced to as a part of background process. + + Artist machine doesn't handle those, explicit Tray with that site name + as a local id must be running. + Example is dropbox site serving as a backup solution + """ + always_accessible_sites = ( + self.get_sync_project_setting(project_name)["config"]. + get("always_accessible_on", []) + ) + return [site.strip() for site in always_accessible_sites] + + def _add_alternative_sites(self, attached_sites): + """Add skeleton document for alternative sites + + Each new configured site in System Setting could serve as a alternative + site, it's a kind of alias. It means that files on 'a site' are + physically accessible also on 'a alternative' site. + Example is sftp site serving studio files via sftp protocol, physically + file is only in studio, sftp server has this location mounted. + """ + additional_sites = self.sync_system_settings.get("sites", {}) + + for site_name, site_info in additional_sites.items(): + # Get alternate sites (stripped names) for this site name + alt_sites = site_info.get("alternative_sites", []) + alt_sites = [site.strip() for site in alt_sites] + alt_sites = set(alt_sites) + + # If no alternative sites we don't need to add + if not alt_sites: + continue + + # Take a copy of data of the first alternate site that is already + # defined as an attached site to match the same state. + match_meta = next((attached_sites[site] for site in alt_sites + if site in attached_sites), None) + if not match_meta: + continue + + alt_site_meta = copy.deepcopy(match_meta) + alt_site_meta["name"] = site_name + + # Note: We change mutable `attached_site` dict in-place + attached_sites[site_name] = alt_site_meta + + return attached_sites + def clear_project(self, collection, site_name): """ Clear 'collection' of 'site_name' and its local files From d41e99cb345bb3407e5e1af5d05de981f1a49795 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Apr 2022 14:14:48 +0200 Subject: [PATCH 02/10] Fix - added recursive configuration for alternative sites --- .../modules/sync_server/sync_server_module.py | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 7126c17e17..d2f341786c 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -229,6 +229,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): attached_sites[remote_site] = create_metadata(remote_site, created=False) + attached_sites = self._add_alternative_sites(attached_sites) # add skeleton for sites where it should be always synced to # usually it would be a backup site which is handled by separate # background process @@ -236,8 +237,6 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if site not in attached_sites: attached_sites[site] = create_metadata(site, created=False) - attached_sites = self._add_alternative_sites(attached_sites) - return list(attached_sites.values()) def _get_always_accessible_sites(self, project_name): @@ -264,9 +263,11 @@ class SyncServerModule(OpenPypeModule, ITrayModule): """ additional_sites = self.sync_system_settings.get("sites", {}) + alt_site_pairs = self._get_alt_site_pairs(additional_sites) + for site_name, site_info in additional_sites.items(): # Get alternate sites (stripped names) for this site name - alt_sites = site_info.get("alternative_sites", []) + alt_sites = alt_site_pairs.get(site_name) alt_sites = [site.strip() for site in alt_sites] alt_sites = set(alt_sites) @@ -289,6 +290,44 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return attached_sites + def _get_alt_site_pairs(self, conf_sites): + """Returns dict of site and its alternative sites. + + If `site` has alternative site, it means that alt_site has 'site' as + alternative site + Args: + conf_sites (dict) + Returns: + (dict): {'site': [alternative sites]...} + """ + alt_site_pairs = {} + for site_name, site_info in conf_sites.items(): + alt_sites = set(site_info.get("alternative_sites", [])) + if not alt_site_pairs.get(site_name): + alt_site_pairs[site_name] = [] + + alt_site_pairs[site_name].extend(alt_sites) + + for alt_site in alt_sites: + if not alt_site_pairs.get(alt_site): + alt_site_pairs[alt_site] = [] + alt_site_pairs[alt_site].extend([site_name]) + + # transitive relationship, eg site is alternative to another which is + # alternative to nex site + loop = True + while loop: + loop = False + for site, alt_sites in alt_site_pairs.items(): + for alt_site in alt_sites: + for alt_alt_site in alt_site_pairs.get(alt_site, []): + if ( alt_alt_site != site + and alt_alt_site not in alt_sites): + alt_site_pairs[site].append(alt_alt_site) + loop = True + + return alt_site_pairs + def clear_project(self, collection, site_name): """ Clear 'collection' of 'site_name' and its local files From b3767433733c07d7a89ef53f2b62edf9c09fc37e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Apr 2022 14:15:38 +0200 Subject: [PATCH 03/10] Fix - moved conftest to be applicable for all kind of tests --- tests/{integration => }/conftest.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{integration => }/conftest.py (100%) diff --git a/tests/integration/conftest.py b/tests/conftest.py similarity index 100% rename from tests/integration/conftest.py rename to tests/conftest.py From 586a429beabec5b7c71ee44ba6896f383ae4a2e6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 6 Apr 2022 14:15:56 +0200 Subject: [PATCH 04/10] Added basic test for alternate site method --- .../modules/sync_server/test_module_api.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/unit/openpype/modules/sync_server/test_module_api.py diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py new file mode 100644 index 0000000000..377045e229 --- /dev/null +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -0,0 +1,59 @@ +"""Test file for Sync Server, tests API methods, currently for integrate_new + + File: + creates temporary directory and downloads .zip file from GDrive + unzips .zip file + uses content of .zip file (MongoDB's dumps) to import to new databases + with use of 'monkeypatch_session' modifies required env vars + temporarily + runs battery of tests checking that site operation for Sync Server + module are working + removes temporary folder + removes temporary databases (?) +""" +import pytest + +import sys, os + +os.environ["OPENPYPE_MONGO"] = "mongodb://localhost:27017" +os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" +os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" +os.environ["AVALON_TIMEOUT"] = '3000' +os.environ["OPENPYPE_DEBUG"] = "3" + +os.environ["AVALON_PROJECT"] = "petr_test" +os.environ["AVALON_DB"] = "avalon" +os.environ["QT_PREFERRED_BINDING"] = "PySide2" +os.environ["QT_VERBOSE"] = "true" + +from tests.lib.testing_classes import ModuleUnitTest + + +class TestModuleApi(ModuleUnitTest): + + REPRESENTATION_ID = "60e578d0c987036c6a7b741d" + + TEST_FILES = [("1eCwPljuJeOI8A3aisfOIBKKjcmIycTEt", + "test_site_operations.zip", '')] + + @pytest.fixture(scope="module") + def setup_sync_server_module(self, dbcon): + """Get sync_server_module from ModulesManager""" + from openpype.modules import ModulesManager + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + yield sync_server + + def test_get_alt_site_pairs(self, setup_sync_server_module): + conf_sites = {'SFTP': {"alternative_sites": ["studio"]}, + "studio2": {"alternative_sites": ["studio"]}} + + ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) + expected = {"SFTP": ["studio", "studio2"], + "studio": ["SFTP", "studio2"], + "studio2": ["studio", "SFTP"]} + assert ret == expected, "Not matching result" + + +test_case = TestModuleApi() From 8fde20646bc440e40d949d36bce941e80392f885 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 7 Apr 2022 10:25:40 +0200 Subject: [PATCH 05/10] Hound --- openpype/modules/sync_server/sync_server_module.py | 4 ++-- .../openpype/modules/sync_server/test_module_api.py | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index d2f341786c..0a70830255 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -265,7 +265,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): alt_site_pairs = self._get_alt_site_pairs(additional_sites) - for site_name, site_info in additional_sites.items(): + for site_name in additional_sites.keys(): # Get alternate sites (stripped names) for this site name alt_sites = alt_site_pairs.get(site_name) alt_sites = [site.strip() for site in alt_sites] @@ -321,7 +321,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): for site, alt_sites in alt_site_pairs.items(): for alt_site in alt_sites: for alt_alt_site in alt_site_pairs.get(alt_site, []): - if ( alt_alt_site != site + if (alt_alt_site != site and alt_alt_site not in alt_sites): alt_site_pairs[site].append(alt_alt_site) loop = True diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py index 377045e229..b6ba2a01b6 100644 --- a/tests/unit/openpype/modules/sync_server/test_module_api.py +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -13,19 +13,6 @@ """ import pytest -import sys, os - -os.environ["OPENPYPE_MONGO"] = "mongodb://localhost:27017" -os.environ["AVALON_MONGO"] = "mongodb://localhost:27017" -os.environ["OPENPYPE_DATABASE_NAME"] = "openpype" -os.environ["AVALON_TIMEOUT"] = '3000' -os.environ["OPENPYPE_DEBUG"] = "3" - -os.environ["AVALON_PROJECT"] = "petr_test" -os.environ["AVALON_DB"] = "avalon" -os.environ["QT_PREFERRED_BINDING"] = "PySide2" -os.environ["QT_VERBOSE"] = "true" - from tests.lib.testing_classes import ModuleUnitTest From 43a68681d6fa71254158006fb642074da51205c4 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 12:21:00 +0200 Subject: [PATCH 06/10] Refactor - changed logic to loop through alt sites --- .../modules/sync_server/sync_server_module.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 0a70830255..ebdcffdab7 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -313,18 +313,23 @@ class SyncServerModule(OpenPypeModule, ITrayModule): alt_site_pairs[alt_site] = [] alt_site_pairs[alt_site].extend([site_name]) - # transitive relationship, eg site is alternative to another which is - # alternative to nex site - loop = True - while loop: - loop = False - for site, alt_sites in alt_site_pairs.items(): - for alt_site in alt_sites: - for alt_alt_site in alt_site_pairs.get(alt_site, []): - if (alt_alt_site != site - and alt_alt_site not in alt_sites): - alt_site_pairs[site].append(alt_alt_site) - loop = True + for site_name, alt_sites in alt_site_pairs.items(): + sites_queue = deque(alt_sites) + while sites_queue: + alt_site = sites_queue.popleft() + + # safety against wrong config + # {"SFTP": {"alternative_site": "SFTP"} + if alt_site == site_name or alt_site not in alt_site_pairs: + continue + + for alt_alt_site in alt_site_pairs[alt_site]: + if ( + alt_alt_site != site_name + and alt_alt_site not in alt_sites + ): + alt_sites.append(alt_alt_site) + sites_queue.append(alt_alt_site) return alt_site_pairs @@ -992,6 +997,11 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if self.enabled and sync_settings.get('enabled'): sites.append(self.LOCAL_SITE) + active_site = sync_settings["config"]["active_site"] + # for Tray running background process + if active_site not in sites and active_site == get_local_site_id(): + sites.append(active_site) + return sites def tray_init(self): From e0db71ba07edafe3e23caf01c77d0101df47c337 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 16:57:01 +0200 Subject: [PATCH 07/10] Refactor - changed to defaultdict --- openpype/modules/sync_server/sync_server_module.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index ebdcffdab7..596aeb8b39 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -4,7 +4,7 @@ from datetime import datetime import threading import platform import copy -from collections import deque +from collections import deque, defaultdict from avalon.api import AvalonMongoDB @@ -300,18 +300,13 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Returns: (dict): {'site': [alternative sites]...} """ - alt_site_pairs = {} + alt_site_pairs = defaultdict(list) for site_name, site_info in conf_sites.items(): alt_sites = set(site_info.get("alternative_sites", [])) - if not alt_site_pairs.get(site_name): - alt_site_pairs[site_name] = [] - alt_site_pairs[site_name].extend(alt_sites) for alt_site in alt_sites: - if not alt_site_pairs.get(alt_site): - alt_site_pairs[alt_site] = [] - alt_site_pairs[alt_site].extend([site_name]) + alt_site_pairs[alt_site].append(site_name) for site_name, alt_sites in alt_site_pairs.items(): sites_queue = deque(alt_sites) From 654d0e3ada68e38cf01f2b5df680be73226b26d8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 14 Apr 2022 11:30:01 +0200 Subject: [PATCH 08/10] Update tests/unit/openpype/modules/sync_server/test_module_api.py Co-authored-by: Roy Nieterau --- tests/unit/openpype/modules/sync_server/test_module_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py index b6ba2a01b6..14613604dd 100644 --- a/tests/unit/openpype/modules/sync_server/test_module_api.py +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -33,7 +33,7 @@ class TestModuleApi(ModuleUnitTest): yield sync_server def test_get_alt_site_pairs(self, setup_sync_server_module): - conf_sites = {'SFTP': {"alternative_sites": ["studio"]}, + conf_sites = {"SFTP": {"alternative_sites": ["studio"]}, "studio2": {"alternative_sites": ["studio"]}} ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) From 2c1114706721bad0465c7a76a046751c9665b5ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Apr 2022 10:15:16 +0200 Subject: [PATCH 09/10] Changed list to set --- openpype/modules/sync_server/sync_server_module.py | 8 ++++---- .../unit/openpype/modules/sync_server/test_module_api.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 596aeb8b39..3744a21b43 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -300,13 +300,13 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Returns: (dict): {'site': [alternative sites]...} """ - alt_site_pairs = defaultdict(list) + alt_site_pairs = defaultdict(set) for site_name, site_info in conf_sites.items(): alt_sites = set(site_info.get("alternative_sites", [])) - alt_site_pairs[site_name].extend(alt_sites) + alt_site_pairs[site_name].update(alt_sites) for alt_site in alt_sites: - alt_site_pairs[alt_site].append(site_name) + alt_site_pairs[alt_site].add(site_name) for site_name, alt_sites in alt_site_pairs.items(): sites_queue = deque(alt_sites) @@ -323,7 +323,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): alt_alt_site != site_name and alt_alt_site not in alt_sites ): - alt_sites.append(alt_alt_site) + alt_sites.add(alt_alt_site) sites_queue.append(alt_alt_site) return alt_site_pairs diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py index 14613604dd..b7d3383c0b 100644 --- a/tests/unit/openpype/modules/sync_server/test_module_api.py +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -37,9 +37,9 @@ class TestModuleApi(ModuleUnitTest): "studio2": {"alternative_sites": ["studio"]}} ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) - expected = {"SFTP": ["studio", "studio2"], - "studio": ["SFTP", "studio2"], - "studio2": ["studio", "SFTP"]} + expected = {"SFTP": {"studio", "studio2"}, + "studio": {"SFTP", "studio2"}, + "studio2": {"studio", "SFTP"}} assert ret == expected, "Not matching result" From 687f769242025413e56908a47611b681186f8162 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 19 Apr 2022 10:23:59 +0200 Subject: [PATCH 10/10] Added more complex test for deeper alternative site tree --- .../modules/sync_server/test_module_api.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py index b7d3383c0b..a484977758 100644 --- a/tests/unit/openpype/modules/sync_server/test_module_api.py +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -42,5 +42,23 @@ class TestModuleApi(ModuleUnitTest): "studio2": {"studio", "SFTP"}} assert ret == expected, "Not matching result" + def test_get_alt_site_pairs_deep(self, setup_sync_server_module): + conf_sites = {"A": {"alternative_sites": ["C"]}, + "B": {"alternative_sites": ["C"]}, + "C": {"alternative_sites": ["D"]}, + "D": {"alternative_sites": ["A"]}, + "F": {"alternative_sites": ["G"]}, + "G": {"alternative_sites": ["F"]}, + } + + ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) + expected = {"A": {"B", "C", "D"}, + "B": {"A", "C", "D"}, + "C": {"A", "B", "D"}, + "D": {"A", "B", "C"}, + "F": {"G"}, + "G": {"F"}} + assert ret == expected, "Not matching result" + test_case = TestModuleApi()