From ea8b3b79b1c3426194a49db7ac5c6d909a0c1d38 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 29 Mar 2022 13:34:25 +0200 Subject: [PATCH 01/21] OP-2951 - added force_only_broken argument to sync methods Cleaned up representation in sync methods --- .../modules/sync_server/sync_server_module.py | 46 +++++++++++-------- openpype/modules/sync_server/utils.py | 5 ++ 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index caf58503f1..9895a6d430 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -23,7 +23,7 @@ from openpype.settings.lib import ( from .providers.local_drive import LocalDriveHandler from .providers import lib -from .utils import time_function, SyncStatus +from .utils import time_function, SyncStatus, SiteAlreadyPresentError log = PypeLogger().get_logger("SyncServer") @@ -129,7 +129,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): """ Start of Public API """ def add_site(self, collection, representation_id, site_name=None, - force=False): + force=False, force_only_broken=False): """ Adds new site to representation to be synced. @@ -143,6 +143,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation_id (string): MongoDB _id value site_name (string): name of configured and active site force (bool): reset site if exists + force_only_broken (bool): reset only if "error" present Returns: throws ValueError if any issue @@ -155,7 +156,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.reset_site_on_representation(collection, representation_id, - site_name=site_name, force=force) + site_name=site_name, + force=force, + force_only_broken=force_only_broken) # public facing API def remove_site(self, collection, representation_id, site_name, @@ -281,7 +284,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): os.path.getmtime(local_file_path)) elem = {"name": site_name, "created_dt": created_dt} - self._add_site(collection, query, [repre], elem, + self._add_site(collection, query, repre, elem, site_name=site_name, file_id=repre_file["_id"]) sites_added += 1 @@ -819,7 +822,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.log.debug("Adding alternate {} to {}".format( alt_site, representation["_id"])) self._add_site(collection, query, - [representation], elem, + representation, elem, alt_site, file_id=file_id, force=True) """ End of Public API """ @@ -1394,7 +1397,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): def reset_site_on_representation(self, collection, representation_id, side=None, file_id=None, site_name=None, - remove=False, pause=None, force=False): + remove=False, pause=None, force=False, + force_only_broken=False): """ Reset information about synchronization for particular 'file_id' and provider. @@ -1417,6 +1421,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): remove (bool): if True remove site altogether pause (bool or None): if True - pause, False - unpause force (bool): hard reset - currently only for add_site + force_only_broken(bool): reset site only if there is "error" field Returns: throws ValueError @@ -1425,7 +1430,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): "_id": ObjectId(representation_id) } - representation = list(self.connection.database[collection].find(query)) + representation = self.connection.database[collection].find_one(query) if not representation: raise ValueError("Representation {} not found in {}". format(representation_id, collection)) @@ -1456,7 +1461,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation, site_name, pause) else: # add new site to all files for representation self._add_site(collection, query, representation, elem, site_name, - force) + force=force, force_only_broken=force_only_broken) def _update_site(self, collection, query, update, arr_filter): """ @@ -1511,7 +1516,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Throws ValueError if 'site_name' not found on 'representation' """ found = False - for repre_file in representation.pop().get("files"): + for repre_file in representation.get("files"): for site in repre_file.get("sites"): if site.get("name") == site_name: found = True @@ -1537,7 +1542,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): """ found = False site = None - for repre_file in representation.pop().get("files"): + for repre_file in representation.get("files"): for site in repre_file.get("sites"): if site["name"] == site_name: found = True @@ -1564,34 +1569,39 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self._update_site(collection, query, update, arr_filter) def _add_site(self, collection, query, representation, elem, site_name, - force=False, file_id=None): + force=False, file_id=None, force_only_broken=False): """ Adds 'site_name' to 'representation' on 'collection' Args: - representation (list of 1 dict) + representation (dict) file_id (ObjectId) Use 'force' to remove existing or raises ValueError """ - reseted_existing = False - for repre_file in representation.pop().get("files"): + reset_existing = False + files = representation.get("files", []) + if not files: + log.debug("No files for {}".format(representation["_id"])) + return + + for repre_file in files: if file_id and file_id != repre_file["_id"]: continue for site in repre_file.get("sites"): if site["name"] == site_name: - if force: + if force or (force_only_broken and site.get("error")): self._reset_site_for_file(collection, query, elem, repre_file["_id"], site_name) - reseted_existing = True + reset_existing = True else: msg = "Site {} already present".format(site_name) log.info(msg) - raise ValueError(msg) + raise SiteAlreadyPresentError(msg) - if reseted_existing: + if reset_existing: return if not file_id: diff --git a/openpype/modules/sync_server/utils.py b/openpype/modules/sync_server/utils.py index 85e4e03f77..03f362202f 100644 --- a/openpype/modules/sync_server/utils.py +++ b/openpype/modules/sync_server/utils.py @@ -8,6 +8,11 @@ class ResumableError(Exception): pass +class SiteAlreadyPresentError(Exception): + """Representation has already site skeleton present.""" + pass + + class SyncStatus: DO_NOTHING = 0 DO_UPLOAD = 1 From d340d05bf01a5f8beda6cdae1736cb59219c4a07 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 29 Mar 2022 13:37:12 +0200 Subject: [PATCH 02/21] OP-2951 - implemented synching referenced files in workfile When workfile is synched, it checks for referenced files (added by Loader) and tries to sync them too. --- openpype/plugins/load/add_site.py | 72 ++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index 95001691e2..0ddce6e160 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -1,9 +1,19 @@ from openpype.modules import ModulesManager from openpype.pipeline import load +:from openpype.lib.avalon_context import get_linked_ids_for_representations +from openpype.modules.sync_server.utils import SiteAlreadyPresentError class AddSyncSite(load.LoaderPlugin): - """Add sync site to representation""" + """Add sync site to representation + + If family of synced representation is 'workfile', it looks for all + representations which are referenced (loaded) in workfile with content of + 'inputLinks'. + It doesn't do any checks for site, most common use case is when artist is + downloading workfile to his local site, but it might be helpful when + artist is re-uploading broken representation on remote site also. + """ representations = ["*"] families = ["*"] @@ -12,21 +22,61 @@ class AddSyncSite(load.LoaderPlugin): icon = "download" color = "#999999" + _sync_server = None + + @property + def sync_server(self): + if not self._sync_server: + manager = ModulesManager() + self._sync_server = manager.modules_by_name["sync_server"] + + return self._sync_server + def load(self, context, name=None, namespace=None, data=None): self.log.info("Adding {} to representation: {}".format( data["site_name"], data["_id"])) - self.add_site_to_representation(data["project_name"], - data["_id"], - data["site_name"]) + family = context["representation"]["context"]["family"] + project_name = data["project_name"] + repre_id = data["_id"] + + add_ids = [repre_id] + if family == "workfile": + links = get_linked_ids_for_representations(project_name, + add_ids, + link_type="reference") + add_ids.extend(links) + + add_ids = set(add_ids) + self.log.info("Add to repre_ids {}".format(add_ids)) + is_main = True + for add_repre_id in add_ids: + self.add_site_to_representation(project_name, + add_repre_id, + data["site_name"], + is_main) + is_main = False + self.log.debug("Site added.") - @staticmethod - def add_site_to_representation(project_name, representation_id, site_name): - """Adds new site to representation_id, resets if exists""" - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] - sync_server.add_site(project_name, representation_id, site_name, - force=True) + def add_site_to_representation(self, project_name, representation_id, + site_name, is_main): + """Adds new site to representation_id, resets if exists + + Args: + project_name (str) + representation_id (ObjectId): + site_name (str) + is_main (bool): true for really downloaded, false for references, + force redownload main file always, for references only if + broken + """ + try: + self.sync_server.add_site(project_name, representation_id, + site_name, + force=is_main, + force_only_broken=not is_main) + except SiteAlreadyPresentError: + self.log.debug("Site present", exc_info=True) def filepath_from_context(self, context): """No real file loading""" From a197334a251404d06f89ec3de6940db68c4b1dde Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 29 Mar 2022 13:57:21 +0200 Subject: [PATCH 03/21] OP-2951 - added function to collect referenced representation ids --- openpype/lib/avalon_context.py | 120 +++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index b4e6abb72d..e8a365ec39 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -1971,3 +1971,123 @@ def get_last_workfile( return os.path.normpath(os.path.join(workdir, filename)) return filename + + +@with_avalon +def get_linked_ids_for_representations(project, repre_ids, dbcon=None, + link_type=None, max_depth=0): + """Returns list of linked ids of particular type (if provided). + + Goes from representations to version, back to representations + Args: + project (str) + repre_ids (list) or (ObjectId) + dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection + with Session. + link_type (str): ['reference', '..] + max_depth (int): limit how many levels of recursion + Returns: + (list) of ObjectId - linked representations + """ + if not dbcon: + log.debug("Using `avalon.io` for query.") + dbcon = avalon.io + # Make sure is installed + dbcon.install() + + if dbcon.Session["AVALON_PROJECT"] != project: + dbcon.Session["AVALON_PROJECT"] = project + + if not isinstance(repre_ids, list): + repre_ids = [repre_ids] + + versions = avalon.io.find( + { + "_id": {"$in": repre_ids}, + "type": "representation" + }, + projection={"parent": True} + ) + version_ids = [version["parent"] for version in versions] + + graph_lookup = { + "from": project, + "startWith": "$data.inputLinks.id", + "connectFromField": "data.inputLinks.id", + "connectToField": "_id", + "as": "outputs_recursive", + "depthField": "depth" + } + if max_depth != 0: + # We offset by -1 since 0 basically means no recursion + # but the recursion only happens after the initial lookup + # for outputs. + graph_lookup["maxDepth"] = max_depth - 1 + + match = { + "_id": {"$in": version_ids}, + "type": "version" + } + + pipeline_ = [ + # Match + {"$match": match}, + # Recursive graph lookup for inputs + {"$graphLookup": graph_lookup} + ] + + result = dbcon.aggregate(pipeline_) + referenced_version_ids = _process_referenced_pipeline_result(result, + link_type) + + representations = avalon.io.find( + { + "parent": {"$in": list(referenced_version_ids)}, + "type": "representation" + }, + projection={"_id": True} + ) + ref_ids = {representation["_id"] for representation in representations} + return list(ref_ids) + + +def _process_referenced_pipeline_result(result, link_type): + """Filters result from pipeline for particular link_type. + + Pipeline cannot use link_type directly in a query. + Returns: + (list) + """ + referenced_version_ids = set() + correctly_linked_ids = set() + for item in result: + correctly_linked_ids = _filter_input_links(item["data"]["inputLinks"], + link_type, + correctly_linked_ids) + + # outputs_recursive in random order, sort by _id + outputs_recursive = sorted(item.get("outputs_recursive", []), + key=lambda d: d["_id"]) + # go from oldest to newest + # only older _id can reference another newer _id + for output in outputs_recursive[::-1]: + if output["_id"] not in correctly_linked_ids: # leaf + continue + + correctly_linked_ids = _filter_input_links( + output["data"].get("inputLinks", []), + link_type, + correctly_linked_ids) + + referenced_version_ids.add(output["_id"]) + + return referenced_version_ids + + +def _filter_input_links(input_links, link_type, correctly_linked_ids): + for input_link in input_links: + if not link_type or input_link["type"] == link_type: + correctly_linked_ids.add(input_link.get("id") or + input_link.get("_id")) # legacy + + return correctly_linked_ids From a0a2e2678e55f449201981b419d9a6a13f8b4a49 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Tue, 29 Mar 2022 14:07:57 +0200 Subject: [PATCH 04/21] OP-2951 - fixed typo --- openpype/plugins/load/add_site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index 0ddce6e160..59720eb5b6 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -1,6 +1,6 @@ from openpype.modules import ModulesManager from openpype.pipeline import load -:from openpype.lib.avalon_context import get_linked_ids_for_representations +from openpype.lib.avalon_context import get_linked_ids_for_representations from openpype.modules.sync_server.utils import SiteAlreadyPresentError From f6fb60bb49bed7a0c26825ef85b5d0f65c4aa6bb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 31 Mar 2022 11:53:58 +0200 Subject: [PATCH 05/21] Update openpype/plugins/load/add_site.py Co-authored-by: Roy Nieterau --- openpype/plugins/load/add_site.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index 59720eb5b6..e26ef586e0 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -38,23 +38,20 @@ class AddSyncSite(load.LoaderPlugin): family = context["representation"]["context"]["family"] project_name = data["project_name"] repre_id = data["_id"] + self.add_site_to_representation(project_name, + repre_id, + data["site_name"], + is_main=True) - add_ids = [repre_id] if family == "workfile": links = get_linked_ids_for_representations(project_name, add_ids, link_type="reference") - add_ids.extend(links) - - add_ids = set(add_ids) - self.log.info("Add to repre_ids {}".format(add_ids)) - is_main = True - for add_repre_id in add_ids: - self.add_site_to_representation(project_name, - add_repre_id, - data["site_name"], - is_main) - is_main = False + for link_repre_id in links: + self.add_site_to_representation(project_name, + link_repre_id, + data["site_name"], + is_main=False) self.log.debug("Site added.") From 6b6c466d8b6ca5b587c8ccf1a8e1dac5e9326bfe Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 31 Mar 2022 12:00:48 +0200 Subject: [PATCH 06/21] OP-2951 - fix wrong variable --- openpype/plugins/load/add_site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index e26ef586e0..22d3ebf24b 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -45,7 +45,7 @@ class AddSyncSite(load.LoaderPlugin): if family == "workfile": links = get_linked_ids_for_representations(project_name, - add_ids, + [repre_id], link_type="reference") for link_repre_id in links: self.add_site_to_representation(project_name, From af079897a884538a5af90bbf2301a004fef7233c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 31 Mar 2022 12:07:43 +0200 Subject: [PATCH 07/21] OP-2951 - refactor use better function --- openpype/lib/avalon_context.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index e8a365ec39..496b55a6f2 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -2040,14 +2040,14 @@ def get_linked_ids_for_representations(project, repre_ids, dbcon=None, referenced_version_ids = _process_referenced_pipeline_result(result, link_type) - representations = avalon.io.find( - { + ref_ids = avalon.io.distinct( + "_id", + filter={ "parent": {"$in": list(referenced_version_ids)}, "type": "representation" - }, - projection={"_id": True} + } ) - ref_ids = {representation["_id"] for representation in representations} + return list(ref_ids) From b826cfac4115f51d8387daab30e3475445256e0f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 31 Mar 2022 12:14:17 +0200 Subject: [PATCH 08/21] OP-2951 - change sort by depth Previous sorting by _id might not be deterministic, not reliable. The main logic is to have outputs sorted by how they were traversed, which should be denoted by 'depth' field. --- openpype/lib/avalon_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 496b55a6f2..9a5d382c98 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -2067,7 +2067,7 @@ def _process_referenced_pipeline_result(result, link_type): # outputs_recursive in random order, sort by _id outputs_recursive = sorted(item.get("outputs_recursive", []), - key=lambda d: d["_id"]) + key=lambda d: d["depth"]) # go from oldest to newest # only older _id can reference another newer _id for output in outputs_recursive[::-1]: From d8c56f0a67cacfc2e05b726efc0e3d8e392c0f78 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 1 Apr 2022 10:39:52 +0200 Subject: [PATCH 09/21] Update openpype/lib/avalon_context.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/lib/avalon_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 9a5d382c98..5ea472f11e 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -2001,7 +2001,7 @@ def get_linked_ids_for_representations(project, repre_ids, dbcon=None, if not isinstance(repre_ids, list): repre_ids = [repre_ids] - versions = avalon.io.find( + versions = dbcon.find( { "_id": {"$in": repre_ids}, "type": "representation" From 6f86f78860c795f027ac481b1f6494ddd5b6979c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 1 Apr 2022 10:40:00 +0200 Subject: [PATCH 10/21] Update openpype/lib/avalon_context.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/lib/avalon_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 5ea472f11e..68d38acf35 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -2040,7 +2040,7 @@ def get_linked_ids_for_representations(project, repre_ids, dbcon=None, referenced_version_ids = _process_referenced_pipeline_result(result, link_type) - ref_ids = avalon.io.distinct( + ref_ids = dbcon.distinct( "_id", filter={ "parent": {"$in": list(referenced_version_ids)}, From d14d739e1cfd312390d9ab880da0a589b3c6d567 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 1 Apr 2022 10:40:08 +0200 Subject: [PATCH 11/21] Update openpype/lib/avalon_context.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/lib/avalon_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 68d38acf35..7d562733fc 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -1974,7 +1974,7 @@ def get_last_workfile( @with_avalon -def get_linked_ids_for_representations(project, repre_ids, dbcon=None, +def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, link_type=None, max_depth=0): """Returns list of linked ids of particular type (if provided). From 44afe82d5a21f8ac4bf393fa35b2357df0c583a5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 1 Apr 2022 11:07:54 +0200 Subject: [PATCH 12/21] OP-2951 - refactored distinct version ids Fixed ordering of referenced versions --- openpype/lib/avalon_context.py | 37 +++++++++++++++------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 7d562733fc..65575493e0 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -1980,7 +1980,7 @@ def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, Goes from representations to version, back to representations Args: - project (str) + project_name (str) repre_ids (list) or (ObjectId) dbcon (avalon.mongodb.AvalonMongoDB, optional): Avalon Mongo connection with Session. @@ -1995,23 +1995,24 @@ def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, # Make sure is installed dbcon.install() - if dbcon.Session["AVALON_PROJECT"] != project: - dbcon.Session["AVALON_PROJECT"] = project + if dbcon.Session["AVALON_PROJECT"] != project_name: + dbcon.Session["AVALON_PROJECT"] = project_name if not isinstance(repre_ids, list): repre_ids = [repre_ids] - versions = dbcon.find( - { - "_id": {"$in": repre_ids}, - "type": "representation" - }, - projection={"parent": True} - ) - version_ids = [version["parent"] for version in versions] + version_ids = dbcon.distinct("parent", { + "_id": {"$in": repre_ids}, + "type": "representation" + }) + + match = { + "_id": {"$in": version_ids}, + "type": "version" + } graph_lookup = { - "from": project, + "from": project_name, "startWith": "$data.inputLinks.id", "connectFromField": "data.inputLinks.id", "connectToField": "_id", @@ -2024,11 +2025,6 @@ def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, # for outputs. graph_lookup["maxDepth"] = max_depth - 1 - match = { - "_id": {"$in": version_ids}, - "type": "version" - } - pipeline_ = [ # Match {"$match": match}, @@ -2065,12 +2061,11 @@ def _process_referenced_pipeline_result(result, link_type): link_type, correctly_linked_ids) - # outputs_recursive in random order, sort by _id + # outputs_recursive in random order, sort by depth outputs_recursive = sorted(item.get("outputs_recursive", []), key=lambda d: d["depth"]) - # go from oldest to newest - # only older _id can reference another newer _id - for output in outputs_recursive[::-1]: + + for output in outputs_recursive: if output["_id"] not in correctly_linked_ids: # leaf continue From 55246ce4a77e25b6d8f7479f741b64839213f5a2 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 1 Apr 2022 17:30:28 +0200 Subject: [PATCH 13/21] Update openpype/lib/avalon_context.py Co-authored-by: Roy Nieterau --- openpype/lib/avalon_context.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 65575493e0..224d8129a7 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -1995,8 +1995,7 @@ def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, # Make sure is installed dbcon.install() - if dbcon.Session["AVALON_PROJECT"] != project_name: - dbcon.Session["AVALON_PROJECT"] = project_name + dbcon.Session["AVALON_PROJECT"] = project_name if not isinstance(repre_ids, list): repre_ids = [repre_ids] From e150db350f972ac23a337fad14b6bcf8889be12c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 12:38:27 +0200 Subject: [PATCH 14/21] Docs - changed docstring --- openpype/plugins/load/add_site.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index 22d3ebf24b..cfb0ede328 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -63,9 +63,11 @@ class AddSyncSite(load.LoaderPlugin): project_name (str) representation_id (ObjectId): site_name (str) - is_main (bool): true for really downloaded, false for references, - force redownload main file always, for references only if - broken + is_main (bool): true for main representation, false for referenced + loaded repres. Drives if site state should be reset. + (it should be for main, not for referenced as they might be + shared from multiple workfiles). In necessary cases, referenced + repres should be reset (re-downloaded) manually. """ try: self.sync_server.add_site(project_name, representation_id, From 4b3405056982e7dea251e6a1d9b4d30d5a979bf6 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 8 Apr 2022 20:33:02 +0200 Subject: [PATCH 15/21] OP-2951 - clean up of unnecessary argument force and force_only_broken could be replaced only by force and explicitly catching SiteAlreadyPresentError --- .../modules/sync_server/sync_server_module.py | 48 ++++++++++--------- openpype/plugins/load/add_site.py | 41 +++++----------- 2 files changed, 36 insertions(+), 53 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 9895a6d430..af69e645d5 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -129,24 +129,27 @@ class SyncServerModule(OpenPypeModule, ITrayModule): """ Start of Public API """ def add_site(self, collection, representation_id, site_name=None, - force=False, force_only_broken=False): + force=False): """ - Adds new site to representation to be synced. + Adds new site to representation to be synced. - 'collection' must have synchronization enabled (globally or - project only) + 'collection' must have synchronization enabled (globally or + project only) - Used as a API endpoint from outside applications (Loader etc) + Used as a API endpoint from outside applications (Loader etc). - Args: - collection (string): project name (must match DB) - representation_id (string): MongoDB _id value - site_name (string): name of configured and active site - force (bool): reset site if exists - force_only_broken (bool): reset only if "error" present + Use 'force' to reset existing site. - Returns: - throws ValueError if any issue + Args: + collection (string): project name (must match DB) + representation_id (string): MongoDB _id value + site_name (string): name of configured and active site + force (bool): reset site if exists + + Throws: + SiteAlreadyPresentError - if adding already existing site and + not 'force' + ValueError - other errors (repre not found, misconfiguration) """ if not self.get_sync_project_setting(collection): raise ValueError("Project not configured") @@ -157,8 +160,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.reset_site_on_representation(collection, representation_id, site_name=site_name, - force=force, - force_only_broken=force_only_broken) + force=force) # public facing API def remove_site(self, collection, representation_id, site_name, @@ -1397,8 +1399,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): def reset_site_on_representation(self, collection, representation_id, side=None, file_id=None, site_name=None, - remove=False, pause=None, force=False, - force_only_broken=False): + remove=False, pause=None, force=False): """ Reset information about synchronization for particular 'file_id' and provider. @@ -1421,10 +1422,11 @@ class SyncServerModule(OpenPypeModule, ITrayModule): remove (bool): if True remove site altogether pause (bool or None): if True - pause, False - unpause force (bool): hard reset - currently only for add_site - force_only_broken(bool): reset site only if there is "error" field - Returns: - throws ValueError + Throws: + SiteAlreadyPresentError - if adding already existing site and + not 'force' + ValueError - other errors (repre not found, misconfiguration) """ query = { "_id": ObjectId(representation_id) @@ -1461,7 +1463,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation, site_name, pause) else: # add new site to all files for representation self._add_site(collection, query, representation, elem, site_name, - force=force, force_only_broken=force_only_broken) + force=force) def _update_site(self, collection, query, update, arr_filter): """ @@ -1569,7 +1571,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self._update_site(collection, query, update, arr_filter) def _add_site(self, collection, query, representation, elem, site_name, - force=False, file_id=None, force_only_broken=False): + force=False, file_id=None): """ Adds 'site_name' to 'representation' on 'collection' @@ -1591,7 +1593,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): for site in repre_file.get("sites"): if site["name"] == site_name: - if force or (force_only_broken and site.get("error")): + if force or site.get("error"): self._reset_site_for_file(collection, query, elem, repre_file["_id"], site_name) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index cfb0ede328..ae765c81b7 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -38,45 +38,26 @@ class AddSyncSite(load.LoaderPlugin): family = context["representation"]["context"]["family"] project_name = data["project_name"] repre_id = data["_id"] - self.add_site_to_representation(project_name, - repre_id, - data["site_name"], - is_main=True) + site_name = data["site_name"] + + self.sync_server.add_site(project_name, repre_id, site_name, + force=True) if family == "workfile": links = get_linked_ids_for_representations(project_name, [repre_id], link_type="reference") for link_repre_id in links: - self.add_site_to_representation(project_name, - link_repre_id, - data["site_name"], - is_main=False) + try: + self.sync_server.add_site(project_name, link_repre_id, + site_name, + force=False) + except SiteAlreadyPresentError: + # do not add/reset working site for references + self.log.debug("Site present", exc_info=True) self.log.debug("Site added.") - def add_site_to_representation(self, project_name, representation_id, - site_name, is_main): - """Adds new site to representation_id, resets if exists - - Args: - project_name (str) - representation_id (ObjectId): - site_name (str) - is_main (bool): true for main representation, false for referenced - loaded repres. Drives if site state should be reset. - (it should be for main, not for referenced as they might be - shared from multiple workfiles). In necessary cases, referenced - repres should be reset (re-downloaded) manually. - """ - try: - self.sync_server.add_site(project_name, representation_id, - site_name, - force=is_main, - force_only_broken=not is_main) - except SiteAlreadyPresentError: - self.log.debug("Site present", exc_info=True) - def filepath_from_context(self, context): """No real file loading""" return "" From ad3dfa872e8fd5b88b1581e0a32eccfe6c2e335c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 11 Apr 2022 12:30:33 +0200 Subject: [PATCH 16/21] OP-2951 - refactor sync loaders Changed structure of loaders bit, cannot use isinstance check though. --- openpype/plugins/load/add_site.py | 1 + openpype/plugins/load/remove_site.py | 26 +++++++++++++++----------- openpype/tools/utils/lib.py | 4 ++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/openpype/plugins/load/add_site.py b/openpype/plugins/load/add_site.py index ae765c81b7..55fda55d17 100644 --- a/openpype/plugins/load/add_site.py +++ b/openpype/plugins/load/add_site.py @@ -23,6 +23,7 @@ class AddSyncSite(load.LoaderPlugin): color = "#999999" _sync_server = None + is_add_site_loader = True @property def sync_server(self): diff --git a/openpype/plugins/load/remove_site.py b/openpype/plugins/load/remove_site.py index adffec9986..c5f442b2f5 100644 --- a/openpype/plugins/load/remove_site.py +++ b/openpype/plugins/load/remove_site.py @@ -12,22 +12,26 @@ class RemoveSyncSite(load.LoaderPlugin): icon = "download" color = "#999999" + _sync_server = None + is_remove_site_loader = True + + @property + def sync_server(self): + if not self._sync_server: + manager = ModulesManager() + self._sync_server = manager.modules_by_name["sync_server"] + + return self._sync_server + def load(self, context, name=None, namespace=None, data=None): self.log.info("Removing {} on representation: {}".format( data["site_name"], data["_id"])) - self.remove_site_on_representation(data["project_name"], - data["_id"], - data["site_name"]) + self.sync_server.remove_site(data["project_name"], + data["_id"], + data["site_name"], + True) self.log.debug("Site added.") - @staticmethod - def remove_site_on_representation(project_name, representation_id, - site_name): - manager = ModulesManager() - sync_server = manager.modules_by_name["sync_server"] - sync_server.remove_site(project_name, representation_id, - site_name, True) - def filepath_from_context(self, context): """No real file loading""" return "" diff --git a/openpype/tools/utils/lib.py b/openpype/tools/utils/lib.py index 422d0f5389..e5d978c3b2 100644 --- a/openpype/tools/utils/lib.py +++ b/openpype/tools/utils/lib.py @@ -719,11 +719,11 @@ def is_sync_loader(loader): def is_remove_site_loader(loader): - return hasattr(loader, "remove_site_on_representation") + return hasattr(loader, "is_remove_site_loader") def is_add_site_loader(loader): - return hasattr(loader, "add_site_to_representation") + return hasattr(loader, "is_add_site_loader") class WrappedCallbackItem: From b33667255813f009f32d63182a202412fe2a465f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 11 Apr 2022 12:31:08 +0200 Subject: [PATCH 17/21] OP-2951 - safer pulling of inputLinks from legacy records --- openpype/lib/avalon_context.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index d26011e522..7f35694e58 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -2057,7 +2057,8 @@ def _process_referenced_pipeline_result(result, link_type): referenced_version_ids = set() correctly_linked_ids = set() for item in result: - correctly_linked_ids = _filter_input_links(item["data"]["inputLinks"], + input_links = item["data"].get("inputLinks", []) + correctly_linked_ids = _filter_input_links(input_links, link_type, correctly_linked_ids) From fc4f7ed5d93b5bdf8c8738bdc7264bb1e7ae627a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Apr 2022 12:09:22 +0200 Subject: [PATCH 18/21] Update openpype/modules/sync_server/sync_server_module.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- openpype/modules/sync_server/sync_server_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index af69e645d5..fb81791da2 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -1423,7 +1423,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): pause (bool or None): if True - pause, False - unpause force (bool): hard reset - currently only for add_site - Throws: + Raises: SiteAlreadyPresentError - if adding already existing site and not 'force' ValueError - other errors (repre not found, misconfiguration) From eb1f72fccd192d5e6d6b326d4fac18206e454aa9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 20 Apr 2022 12:29:02 +0200 Subject: [PATCH 19/21] OP-2951 - refactored to use AvalonMongoDB --- openpype/lib/avalon_context.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/openpype/lib/avalon_context.py b/openpype/lib/avalon_context.py index 7f35694e58..35ccb1b68d 100644 --- a/openpype/lib/avalon_context.py +++ b/openpype/lib/avalon_context.py @@ -1990,13 +1990,14 @@ def get_linked_ids_for_representations(project_name, repre_ids, dbcon=None, Returns: (list) of ObjectId - linked representations """ + # Create new dbcon if not passed and use passed project name if not dbcon: - log.debug("Using `avalon.io` for query.") - dbcon = avalon.io - # Make sure is installed - dbcon.install() - - dbcon.Session["AVALON_PROJECT"] = project_name + from avalon.api import AvalonMongoDB + dbcon = AvalonMongoDB() + dbcon.Session["AVALON_PROJECT"] = project_name + # Validate that passed dbcon has same project + elif dbcon.Session["AVALON_PROJECT"] != project_name: + raise ValueError("Passed connection does not have right project") if not isinstance(repre_ids, list): repre_ids = [repre_ids] From 3f976f00d2ef46612a770a96bacde8d8d3df039d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Apr 2022 12:30:42 +0200 Subject: [PATCH 20/21] OP-2951 - refactored Validate projects Checks also if set in DB and not physically present --- .../modules/sync_server/sync_server_module.py | 83 ++++++++++--------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index fb81791da2..7afbdc2e9b 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -212,36 +212,38 @@ class SyncServerModule(OpenPypeModule, ITrayModule): def create_validate_project_task(self, collection, site_name): """Adds metadata about project files validation on a queue. - This process will loop through all representation and check if - their files actually exist on an active site. + This process will loop through all representation and check if + their files actually exist on an active site. - This might be useful for edge cases when artists is switching - between sites, remote site is actually physically mounted and - active site has same file urls etc. + It also checks if site is set in DB, but file is physically not + present - Task will run on a asyncio loop, shouldn't be blocking. + This might be useful for edge cases when artists is switching + between sites, remote site is actually physically mounted and + active site has same file urls etc. + + Task will run on a asyncio loop, shouldn't be blocking. """ task = { "type": "validate", "project_name": collection, - "func": lambda: self.validate_project(collection, site_name) + "func": lambda: self.validate_project(collection, site_name, + reset_missing=True) } self.projects_processed.add(collection) self.long_running_tasks.append(task) - def validate_project(self, collection, site_name, remove_missing=False): - """ - Validate 'collection' of 'site_name' and its local files + def validate_project(self, collection, site_name, reset_missing=False): + """Validate 'collection' of 'site_name' and its local files - If file present and not marked with a 'site_name' in DB, DB is - updated with site name and file modified date. + If file present and not marked with a 'site_name' in DB, DB is + updated with site name and file modified date. - Args: - module (SyncServerModule) - collection (string): project name - site_name (string): active site name - remove_missing (bool): if True remove sites in DB if missing - physically + Args: + collection (string): project name + site_name (string): active site name + reset_missing (bool): if True reset site in DB if missing + physically """ self.log.debug("Validation of {} for {} started".format(collection, site_name)) @@ -256,29 +258,32 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return sites_added = 0 - sites_removed = 0 + sites_reset = 0 for repre in representations: repre_id = repre["_id"] for repre_file in repre.get("files", []): try: - has_site = site_name in [site["name"] - for site in repre_file["sites"]] - except TypeError: + is_on_site = site_name in [site["name"] + for site in repre_file["sites"] + if (site.get("created_dt") and + not site.get("error"))] + except (TypeError, AttributeError): self.log.debug("Structure error in {}".format(repre_id)) continue - if has_site and not remove_missing: - continue - file_path = repre_file.get("path", "") local_file_path = self.get_local_file_path(collection, site_name, file_path) - if local_file_path and os.path.exists(local_file_path): - self.log.debug("Adding site {} for {}".format(site_name, - repre_id)) - if not has_site: + file_exists = (local_file_path and + os.path.exists(local_file_path)) + if not is_on_site: + if file_exists: + self.log.debug( + "Adding site {} for {}".format(site_name, + repre_id)) + query = { "_id": repre_id } @@ -288,25 +293,25 @@ class SyncServerModule(OpenPypeModule, ITrayModule): "created_dt": created_dt} self._add_site(collection, query, repre, elem, site_name=site_name, - file_id=repre_file["_id"]) + file_id=repre_file["_id"], + force=True) sites_added += 1 else: - if has_site and remove_missing: - self.log.debug("Removing site {} for {}". + if not file_exists and reset_missing: + self.log.debug("Resetting site {} for {}". format(site_name, repre_id)) - self.reset_provider_for_file(collection, - repre_id, - file_id=repre_file["_id"], - remove=True) - sites_removed += 1 + self.reset_site_on_representation( + collection, repre_id, site_name=site_name, + file_id=repre_file["_id"]) + sites_reset += 1 if sites_added % 100 == 0: self.log.debug("Sites added {}".format(sites_added)) self.log.debug("Validation of {} for {} ended".format(collection, site_name)) - self.log.info("Sites added {}, sites removed {}".format(sites_added, - sites_removed)) + self.log.info("Sites added {}, sites reset {}".format(sites_added, + reset_missing)) def pause_representation(self, collection, representation_id, site_name): """ From 581647a65cf912c09eb79684af40e49fffe60bde Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Apr 2022 13:33:23 +0200 Subject: [PATCH 21/21] OP-2951 - safer querying of tries --- openpype/modules/sync_server/sync_server_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/sync_server/sync_server_module.py b/openpype/modules/sync_server/sync_server_module.py index 7afbdc2e9b..ccd0fd111d 100644 --- a/openpype/modules/sync_server/sync_server_module.py +++ b/openpype/modules/sync_server/sync_server_module.py @@ -1772,7 +1772,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): (int) - number of failed attempts """ _, rec = self._get_site_rec(file.get("sites", []), provider) - return rec.get("tries", 0) + return self._get_tries_count_from_rec(rec) def _get_progress_dict(self, progress): """