diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 2c27571f9f..3744a21b43 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -4,7 +4,7 @@ from datetime import datetime import threading import platform import copy -from collections import deque +from collections import deque, defaultdict from avalon.api import AvalonMongoDB @@ -157,7 +157,6 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation_id, site_name=site_name, force=force) - # public facing API def remove_site(self, collection, representation_id, site_name, remove_local_files=False): """ @@ -184,6 +183,151 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if remove_local_files: self._remove_local_file(collection, representation_id, site_name) + def compute_resource_sync_sites(self, project_name): + """Get available resource sync sites state for publish process. + + Returns dict with prepared state of sync sites for 'project_name'. + It checks if Site Sync is enabled, handles alternative sites. + Publish process stores this dictionary as a part of representation + document in DB. + + Example: + [ + { + 'name': '42abbc09-d62a-44a4-815c-a12cd679d2d7', + 'created_dt': datetime.datetime(2022, 3, 30, 12, 16, 9, 778637) + }, + {'name': 'studio'}, + {'name': 'SFTP'} + ] -- representation is published locally, artist or Settings have set + remote site as 'studio'. 'SFTP' is alternate site to 'studio'. Eg. + whenever file is on 'studio', it is also on 'SFTP'. + """ + + def create_metadata(name, created=True): + """Create sync site metadata for site with `name`""" + metadata = {"name": name} + if created: + metadata["created_dt"] = datetime.now() + return metadata + + if ( + not self.sync_system_settings["enabled"] or + not self.sync_project_settings[project_name]["enabled"]): + return [create_metadata(self.DEFAULT_SITE)] + + local_site = self.get_active_site(project_name) + remote_site = self.get_remote_site(project_name) + + # Attached sites metadata by site name + # That is the local site, remote site, the always accesible sites + # and their alternate sites (alias of sites with different protocol) + attached_sites = dict() + attached_sites[local_site] = create_metadata(local_site) + + if remote_site and remote_site not in attached_sites: + attached_sites[remote_site] = create_metadata(remote_site, + created=False) + + attached_sites = self._add_alternative_sites(attached_sites) + # add skeleton for sites where it should be always synced to + # usually it would be a backup site which is handled by separate + # background process + for site in self._get_always_accessible_sites(project_name): + if site not in attached_sites: + attached_sites[site] = create_metadata(site, created=False) + + return list(attached_sites.values()) + + def _get_always_accessible_sites(self, project_name): + """Sites that synced to as a part of background process. + + Artist machine doesn't handle those, explicit Tray with that site name + as a local id must be running. + Example is dropbox site serving as a backup solution + """ + always_accessible_sites = ( + self.get_sync_project_setting(project_name)["config"]. + get("always_accessible_on", []) + ) + return [site.strip() for site in always_accessible_sites] + + def _add_alternative_sites(self, attached_sites): + """Add skeleton document for alternative sites + + Each new configured site in System Setting could serve as a alternative + site, it's a kind of alias. It means that files on 'a site' are + physically accessible also on 'a alternative' site. + Example is sftp site serving studio files via sftp protocol, physically + file is only in studio, sftp server has this location mounted. + """ + additional_sites = self.sync_system_settings.get("sites", {}) + + alt_site_pairs = self._get_alt_site_pairs(additional_sites) + + for site_name in additional_sites.keys(): + # Get alternate sites (stripped names) for this site name + alt_sites = alt_site_pairs.get(site_name) + alt_sites = [site.strip() for site in alt_sites] + alt_sites = set(alt_sites) + + # If no alternative sites we don't need to add + if not alt_sites: + continue + + # Take a copy of data of the first alternate site that is already + # defined as an attached site to match the same state. + match_meta = next((attached_sites[site] for site in alt_sites + if site in attached_sites), None) + if not match_meta: + continue + + alt_site_meta = copy.deepcopy(match_meta) + alt_site_meta["name"] = site_name + + # Note: We change mutable `attached_site` dict in-place + attached_sites[site_name] = alt_site_meta + + return attached_sites + + def _get_alt_site_pairs(self, conf_sites): + """Returns dict of site and its alternative sites. + + If `site` has alternative site, it means that alt_site has 'site' as + alternative site + Args: + conf_sites (dict) + Returns: + (dict): {'site': [alternative sites]...} + """ + alt_site_pairs = defaultdict(set) + for site_name, site_info in conf_sites.items(): + alt_sites = set(site_info.get("alternative_sites", [])) + alt_site_pairs[site_name].update(alt_sites) + + for alt_site in alt_sites: + alt_site_pairs[alt_site].add(site_name) + + for site_name, alt_sites in alt_site_pairs.items(): + sites_queue = deque(alt_sites) + while sites_queue: + alt_site = sites_queue.popleft() + + # safety against wrong config + # {"SFTP": {"alternative_site": "SFTP"} + if alt_site == site_name or alt_site not in alt_site_pairs: + continue + + for alt_alt_site in alt_site_pairs[alt_site]: + if ( + alt_alt_site != site_name + and alt_alt_site not in alt_sites + ): + alt_sites.add(alt_alt_site) + sites_queue.append(alt_alt_site) + + return alt_site_pairs + def clear_project(self, collection, site_name): """ Clear 'collection' of 'site_name' and its local files diff --git a/tests/integration/conftest.py b/tests/conftest.py similarity index 100% rename from tests/integration/conftest.py rename to tests/conftest.py diff --git a/tests/unit/openpype/modules/sync_server/test_module_api.py b/tests/unit/openpype/modules/sync_server/test_module_api.py new file mode 100644 index 0000000000..a484977758 --- /dev/null +++ b/tests/unit/openpype/modules/sync_server/test_module_api.py @@ -0,0 +1,64 @@ +"""Test file for Sync Server, tests API methods, currently for integrate_new + + File: + creates temporary directory and downloads .zip file from GDrive + unzips .zip file + uses content of .zip file (MongoDB's dumps) to import to new databases + with use of 'monkeypatch_session' modifies required env vars + temporarily + runs battery of tests checking that site operation for Sync Server + module are working + removes temporary folder + removes temporary databases (?) +""" +import pytest + +from tests.lib.testing_classes import ModuleUnitTest + + +class TestModuleApi(ModuleUnitTest): + + REPRESENTATION_ID = "60e578d0c987036c6a7b741d" + + TEST_FILES = [("1eCwPljuJeOI8A3aisfOIBKKjcmIycTEt", + "test_site_operations.zip", '')] + + @pytest.fixture(scope="module") + def setup_sync_server_module(self, dbcon): + """Get sync_server_module from ModulesManager""" + from openpype.modules import ModulesManager + + manager = ModulesManager() + sync_server = manager.modules_by_name["sync_server"] + yield sync_server + + def test_get_alt_site_pairs(self, setup_sync_server_module): + conf_sites = {"SFTP": {"alternative_sites": ["studio"]}, + "studio2": {"alternative_sites": ["studio"]}} + + ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) + expected = {"SFTP": {"studio", "studio2"}, + "studio": {"SFTP", "studio2"}, + "studio2": {"studio", "SFTP"}} + assert ret == expected, "Not matching result" + + def test_get_alt_site_pairs_deep(self, setup_sync_server_module): + conf_sites = {"A": {"alternative_sites": ["C"]}, + "B": {"alternative_sites": ["C"]}, + "C": {"alternative_sites": ["D"]}, + "D": {"alternative_sites": ["A"]}, + "F": {"alternative_sites": ["G"]}, + "G": {"alternative_sites": ["F"]}, + } + + ret = setup_sync_server_module._get_alt_site_pairs(conf_sites) + expected = {"A": {"B", "C", "D"}, + "B": {"A", "C", "D"}, + "C": {"A", "B", "D"}, + "D": {"A", "B", "C"}, + "F": {"G"}, + "G": {"F"}} + assert ret == expected, "Not matching result" + + +test_case = TestModuleApi()