From be57dd3e66eff6c63e089b81421a087bda1b5690 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 22 Jan 2021 17:16:17 +0100 Subject: [PATCH 01/28] SyncServer GUI - fix nondeterministic sorting Wrong placement of sort after limit lead to nondeterministic sorting --- pype/modules/sync_server/tray/app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index afd103f9d5..564d741adc 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -619,10 +619,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'failed_local': {'$sum': '$failed_local'}, 'updated_dt_local': {'$max': "$updated_dt_local"} }}, - {"$limit": limit}, - {"$skip": self._rec_loaded}, {"$project": self.projection}, - {"$sort": self.sort} + {"$sort": self.sort}, + {"$limit": limit}, + {"$skip": self._rec_loaded} ] def _get_match_part(self): @@ -1295,10 +1295,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): ]} ]}} }}, - {"$limit": limit}, - {"$skip": self._rec_loaded}, {"$project": self.projection}, - {"$sort": self.sort} + {"$sort": self.sort}, + {"$limit": limit}, + {"$skip": self._rec_loaded} ] def _get_match_part(self): From 976669ddc498f5234cfe5e53b42af2d35f0f1f11 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 22 Jan 2021 18:25:29 +0100 Subject: [PATCH 02/28] SyncServer GUI - fix wrong selection Only representations which have both 'local_site' and 'remote_site' are selected --- pype/modules/sync_server/tray/app.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 564d741adc..408ac8a961 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -637,14 +637,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if not self.filter: return { "type": "representation", - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } + 'files.sites.name': {'$all': [self.local_site, + self.remote_site]} } else: regex_str = '.*{}.*'.format(self.filter) @@ -655,14 +649,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): {'context.asset': {'$regex': regex_str, '$options': 'i'}}, {'context.representation': {'$regex': regex_str, '$options': 'i'}}], - 'files.sites': { - '$elemMatch': { - '$or': [ - {'name': self.local_site}, - {'name': self.remote_site} - ] - } - } + 'files.sites.name': {'$all': [self.local_site, + self.remote_site]} } def get_default_projection(self): From 6d160d8830dd6f22cdaf4b5fffdf9f28c7f899ed Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 22 Jan 2021 20:14:23 +0100 Subject: [PATCH 03/28] SyncServer GUI - fix pagination Used '$facet', return something like resultset --- pype/modules/sync_server/tray/app.py | 85 +++++++++++++++++++--------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 408ac8a961..48c93fc7e5 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -238,7 +238,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): class SyncRepresentationModel(QtCore.QAbstractTableModel): - PAGE_SIZE = 19 + PAGE_SIZE = 20 REFRESH_SEC = 5000 DEFAULT_SORT = { "updated_dt_remote": -1, @@ -288,7 +288,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._data = [] self._project = project self._rec_loaded = 0 - self._buffer = [] # stash one page worth of records (actually cursor) + self._total_records = 0 # how many documents query actually found self.filter = None self._initialized = False @@ -366,7 +366,22 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.endResetModel() def _add_page_records(self, local_site, remote_site, representations): - for repre in representations: + """ + Process all records from 'representation' and add them to storage. + + Args: + local_site (str): name of local site (mine) + remote_site (str): name of cloud provider (theirs) + representations (Mongo Cursor) - mimics result set, 1 object + with paginatedResults array and totalCount array + """ + result = representations.next() + count = 0 + total_count = result.get("totalCount") + if total_count: + count = total_count.pop().get('count') + self._total_records = count + for repre in result.get("paginatedResults"): context = repre.get("context").pop() files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary @@ -407,15 +422,13 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._data.append(item) self._rec_loaded += 1 + def canFetchMore(self, index): """ Check if there are more records than currently loaded """ # 'skip' might be suboptimal when representation hits 500k+ - self._buffer = list(self.dbcon.aggregate(self.query)) - # log.info("!!! canFetchMore _rec_loaded::{} - {}".format( - # self._rec_loaded, len(self._buffer))) - return len(self._buffer) > self._rec_loaded + return self._total_records > self._rec_loaded def fetchMore(self, index): """ @@ -423,17 +436,18 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): Called when 'canFetchMore' returns true, which means there are more records in DB than loaded. - 'self._buffer' is used to stash cursor to limit requery """ log.debug("fetchMore") - # cursor.count() returns always total number, not only skipped + limit - remainder = len(self._buffer) - self._rec_loaded - items_to_fetch = min(self.PAGE_SIZE, remainder) + items_to_fetch = min(self._total_records - self._rec_loaded, + self.PAGE_SIZE) + self.query = self.get_default_query(self._rec_loaded) + representations = self.dbcon.aggregate(self.query) self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) - self._add_page_records(self.local_site, self.remote_site, self._buffer) + self._add_page_records(self.local_site, self.remote_site, + representations) self.endInsertRows() @@ -621,8 +635,13 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): }}, {"$project": self.projection}, {"$sort": self.sort}, - {"$limit": limit}, - {"$skip": self._rec_loaded} + { + '$facet': { + 'paginatedResults': [{'$skip': self._rec_loaded}, + {'$limit': limit}], + 'totalCount': [{'$count': 'count'}] + } + } ] def _get_match_part(self): @@ -987,8 +1006,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self._data = [] self._project = project self._rec_loaded = 0 + self._total_records = 0 # how many documents query actually found self.filter = None - self._buffer = [] # stash one page worth of records (actually cursor) self._id = _id self._initialized = False @@ -1068,9 +1087,17 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): Args: local_site (str): name of local site (mine) remote_site (str): name of cloud provider (theirs) - representations (Mongo Cursor) + representations (Mongo Cursor) - mimics result set, 1 object + with paginatedResults array and totalCount array """ - for repre in representations: + # representations is a Cursor, get first + result = representations.next() + count = 0 + total_count = result.get("totalCount") + if total_count: + count = total_count.pop().get('count') + self._total_records = count + for repre in result.get("paginatedResults"): # log.info("!!! repre:: {}".format(repre)) files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary @@ -1118,8 +1145,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): Check if there are more records than currently loaded """ # 'skip' might be suboptimal when representation hits 500k+ - self._buffer = list(self.dbcon.aggregate(self.query)) - return len(self._buffer) > self._rec_loaded + return self._total_records > self._rec_loaded def fetchMore(self, index): """ @@ -1130,14 +1156,16 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'self._buffer' is used to stash cursor to limit requery """ log.debug("fetchMore") - # cursor.count() returns always total number, not only skipped + limit - remainder = len(self._buffer) - self._rec_loaded - items_to_fetch = min(self.PAGE_SIZE, remainder) - + items_to_fetch = min(self._total_records - self._rec_loaded, + self.PAGE_SIZE) + self.query = self.get_default_query(self._rec_loaded) + representations = self.dbcon.aggregate(self.query) self.beginInsertRows(index, self._rec_loaded, self._rec_loaded + items_to_fetch - 1) - self._add_page_records(self.local_site, self.remote_site, self._buffer) + + self._add_page_records(self.local_site, self.remote_site, + representations) self.endInsertRows() @@ -1285,8 +1313,13 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): }}, {"$project": self.projection}, {"$sort": self.sort}, - {"$limit": limit}, - {"$skip": self._rec_loaded} + { + '$facet': { + 'paginatedResults': [{'$skip': self._rec_loaded}, + {'$limit': limit}], + 'totalCount': [{'$count': 'count'}] + } + } ] def _get_match_part(self): From c0c12d0a9a5849a79393caa433f08d421fc954eb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Jan 2021 12:24:24 +0100 Subject: [PATCH 04/28] SyncServer - renamed active_site to publish_site publish_site meaning is 'a place I am publishing to'. Usually it will be 'studio', eg. publishing to shared drive, but it could be artist's local site. --- pype/modules/sync_server/README.md | 4 +- pype/modules/sync_server/sync_server.py | 52 ++++++++++--------- pype/plugins/global/publish/integrate_new.py | 2 +- .../defaults/project_settings/global.json | 2 +- .../schema_project_syncserver.json | 2 +- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/pype/modules/sync_server/README.md b/pype/modules/sync_server/README.md index 8ecf849a4e..d7d7f3718b 100644 --- a/pype/modules/sync_server/README.md +++ b/pype/modules/sync_server/README.md @@ -62,7 +62,7 @@ Needed configuration: - `"local_id": "local_0",` -- identifier of user pype - `"retry_cnt": 3,` -- how many times try to synch file in case of error - `"loop_delay": 60,` -- how many seconds between sync loops - - `"active_site": "studio",` -- which site user current, 'studio' by default, + - `"publish_site": "studio",` -- which site user current, 'studio' by default, could by same as 'local_id' if user is working from home without connection to studio infrastructure @@ -71,7 +71,7 @@ Needed configuration: Used in IntegrateNew to prepare skeleton for syncing in the representation record. Leave empty if no syncing is wanted. - This is a general configuration, 'local_id', 'active_site' and 'remote_site' + This is a general configuration, 'local_id', 'publish_site' and 'remote_site' will be set and changed by some GUI in the future. `pype/settings/defaults/project_settings/global.json`.`sync_server`.`sites`: diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 84637a1d62..8fb0dfe955 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -43,7 +43,7 @@ class SyncServer(PypeModule, ITrayModule): checks if 'created_dt' field is present denoting successful sync with provider destination. Sites structure is created during publish and by default it will - always contain 1 record with "name" == self.presets["active_site"] and + always contain 1 record with "name" == self.presets["publish_site"] and filled "created_dt" AND 1 or multiple records for all defined remote sites, where "created_dt" is not present. This highlights that file should be uploaded to @@ -73,8 +73,8 @@ class SyncServer(PypeModule, ITrayModule): Each Tray app has assigned its own self.presets["local_id"] used in sites as a name. Tray is searching only for records where name matches its - self.presets["active_site"] + self.presets["remote_site"]. - "active_site" could be storage in studio ('studio'), or specific + self.presets["publish_site"] + self.presets["remote_site"]. + "publish_site" could be storage in studio ('studio'), or specific "local_id" when user is working disconnected from home. If the local record has its "created_dt" filled, it is a source and process will try to upload the file to all defined remote sites. @@ -140,7 +140,7 @@ class SyncServer(PypeModule, ITrayModule): try: self.presets = self.get_synced_presets() - self.set_active_sites(self.presets) + self.set_publish_sites(self.presets) self.sync_server_thread = SyncServerThread(self) from .tray.app import SyncServerWindow self.widget = SyncServerWindow(self) @@ -166,7 +166,7 @@ class SyncServer(PypeModule, ITrayModule): Returns: None """ - if self.presets and self.active_sites: + if self.presets and self.publish_sites: self.sync_server_thread.start() else: log.info("No presets or active providers. " + @@ -229,7 +229,7 @@ class SyncServer(PypeModule, ITrayModule): sync_server_presets = settings["global"]["sync_server"]["config"] if settings["global"]["sync_server"]["enabled"]: - local_site = sync_server_presets.get("active_site", + local_site = sync_server_presets.get("publish_site", "studio").strip() remote_site = sync_server_presets.get("remote_site") @@ -244,6 +244,10 @@ class SyncServer(PypeModule, ITrayModule): (dict): of settings, keys are project names """ sync_presets = {} + if not self.connection: + self.connection = AvalonMongoDB() + self.connection.install() + for collection in self.connection.database.collection_names(False): sync_settings = self.get_synced_preset(collection) if sync_settings: @@ -274,9 +278,9 @@ class SyncServer(PypeModule, ITrayModule): return {} - def set_active_sites(self, settings): + def set_publish_sites(self, settings): """ - Sets 'self.active_sites' as a dictionary from provided 'settings' + Sets 'self.publish_sites' as a dictionary from provided 'settings' Format: { 'project_name' : ('provider_name', 'site_name') } @@ -284,23 +288,23 @@ class SyncServer(PypeModule, ITrayModule): settings (dict): all enabled project sync setting (sites labesl, retries count etc.) """ - self.active_sites = {} + self.publish_sites = {} for project_name, project_setting in settings.items(): for site_name, config in project_setting.get("sites").items(): handler = lib.factory.get_provider(config["provider"], site_name, presets=config) if handler.is_active(): - if not self.active_sites.get('project_name'): - self.active_sites[project_name] = [] + if not self.publish_sites.get('project_name'): + self.publish_sites[project_name] = [] - self.active_sites[project_name].append( + self.publish_sites[project_name].append( (config["provider"], site_name)) - if not self.active_sites: + if not self.publish_sites: log.info("No sync sites active, no working credentials provided") - def get_active_sites(self, project_name): + def get_publish_sites(self, project_name): """ Returns active sites (provider configured and able to connect) per project. @@ -313,10 +317,10 @@ class SyncServer(PypeModule, ITrayModule): Format: { 'project_name' : ('provider_name', 'site_name') } """ - return self.active_sites[project_name] + return self.publish_sites[project_name] @time_function - def get_sync_representations(self, collection, active_site, remote_site): + def get_sync_representations(self, collection, publish_site, remote_site): """ Get representations that should be synced, these could be recognised by presence of document in 'files.sites', where key is @@ -329,7 +333,7 @@ class SyncServer(PypeModule, ITrayModule): Args: collection (string): name of collection (in most cases matches project name - active_site (string): identifier of current active site (could be + publish_site (string): identifier of current active site (could be 'local_0' when working from home, 'studio' when working in the studio (default) remote_site (string): identifier of remote site I want to sync to @@ -348,7 +352,7 @@ class SyncServer(PypeModule, ITrayModule): { "files.sites": { "$elemMatch": { - "name": active_site, + "name": publish_site, "created_dt": {"$exists": True} } }}, { @@ -364,7 +368,7 @@ class SyncServer(PypeModule, ITrayModule): { "files.sites": { "$elemMatch": { - "name": active_site, + "name": publish_site, "created_dt": {"$exists": False}, "tries": {"$in": retries_arr} } @@ -392,7 +396,7 @@ class SyncServer(PypeModule, ITrayModule): (Eg. check if 'scene.ma' of lookdev.v10 should be synced to GDrive Always is comparing local record, eg. site with - 'name' == self.presets[PROJECT_NAME]['config']["active_site"] + 'name' == self.presets[PROJECT_NAME]['config']["publish_site"] Args: file (dictionary): of file from representation in Mongo @@ -416,7 +420,7 @@ class SyncServer(PypeModule, ITrayModule): else: _, local_rec = self._get_provider_rec( sites, - config_preset["active_site"]) or {} + config_preset["publish_site"]) or {} if not local_rec or not local_rec.get("created_dt"): tries = self._get_tries_count_from_rec(local_rec) @@ -883,11 +887,11 @@ class SyncServerThread(threading.Thread): start_time = time.time() sync_repres = self.module.get_sync_representations( collection, - preset.get('config')["active_site"], + preset.get('config')["publish_site"], preset.get('config')["remote_site"] ) - local = preset.get('config')["active_site"] + local = preset.get('config')["publish_site"] task_files_to_process = [] files_processed_info = [] # process only unique file paths in one batch @@ -896,7 +900,7 @@ class SyncServerThread(threading.Thread): # upload process can find already uploaded file and # reuse same id processed_file_path = set() - for check_site in self.module.get_active_sites(collection): + for check_site in self.module.get_publish_sites(collection): provider, site = check_site site_preset = preset.get('sites')[site] handler = lib.factory.get_provider(provider, diff --git a/pype/plugins/global/publish/integrate_new.py b/pype/plugins/global/publish/integrate_new.py index 5ba92435fd..19568c51ea 100644 --- a/pype/plugins/global/publish/integrate_new.py +++ b/pype/plugins/global/publish/integrate_new.py @@ -958,7 +958,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sync_server_presets["enabled"]: local_site = sync_server_presets["config"].\ - get("active_site", "studio").strip() + get("publish_site", "studio").strip() remote_site = sync_server_presets["config"].get("remote_site") rec = { diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 5913277df3..d73026f686 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -185,7 +185,7 @@ "local_id": "local_0", "retry_cnt": "3", "loop_delay": "60", - "active_site": "studio", + "publish_site": "studio", "remote_site": "gdrive" }, "sites": { diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index 396e4ca2dc..7a39f9cd4f 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -35,7 +35,7 @@ }, { "type": "text", - "key": "active_site", + "key": "publish_site", "label": "Active Site" }, { From e66e661c38ba68bbd95a261a51ca7dae626d7d76 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Feb 2021 16:19:09 +0100 Subject: [PATCH 05/28] SyncServer GUI - added reset site for whole representation Updated saving to DB with better approach --- pype/modules/sync_server/sync_server.py | 105 +++++++++++------------- pype/modules/sync_server/tray/app.py | 82 ++++++++++++++++-- 2 files changed, 122 insertions(+), 65 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 8fb0dfe955..db5180f0b1 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -557,34 +557,32 @@ class SyncServer(PypeModule, ITrayModule): representation_id = representation.get("_id") file_id = file.get("_id") query = { - "_id": representation_id, - "files._id": file_id + "_id": representation_id } - file_index, _ = self._get_file_info(representation.get('files', []), - file_id) - site_index, _ = self._get_provider_rec(file.get('sites', []), - site) + update = {} if new_file_id: - update["$set"] = self._get_success_dict(file_index, site_index, - new_file_id) + update["$set"] = self._get_success_dict(new_file_id) # reset previous errors if any - update["$unset"] = self._get_error_dict(file_index, site_index, - "", "", "") + update["$unset"] = self._get_error_dict("", "", "") elif progress is not None: - update["$set"] = self._get_progress_dict(file_index, site_index, - progress) + update["$set"] = self._get_progress_dict(progress) else: tries = self._get_tries_count(file, site) tries += 1 - update["$set"] = self._get_error_dict(file_index, site_index, - error, tries) + update["$set"] = self._get_error_dict(error, tries) - self.connection.Session["AVALON_PROJECT"] = collection - self.connection.update_one( + arr_filter = [ + {'s.name': site}, + {'f._id': ObjectId(file_id)} + ] + + self.connection.database[collection].update_one( query, - update + update, + upsert=True, + array_filters=arr_filter ) if progress is not None: @@ -642,7 +640,7 @@ class SyncServer(PypeModule, ITrayModule): return -1, None def reset_provider_for_file(self, collection, representation_id, - file_id, side): + side, file_id=None): """ Reset information about synchronization for particular 'file_id' and provider. @@ -671,24 +669,32 @@ class SyncServer(PypeModule, ITrayModule): else: site_name = remote_site - files = representation[0].get('files', []) - file_index, _ = self._get_file_info(files, - file_id) - site_index, _ = self._get_provider_rec(files[file_index]. - get('sites', []), - site_name) - if file_index >= 0 and site_index >= 0: - elem = {"name": site_name} + elem = {"name": site_name} + + if file_id: update = { - "$set": {"files.{}.sites.{}".format(file_index, site_index): - elem - } + "$set": {"files.$[f].sites.$[s]": elem} } - self.connection.database[collection].update_one( - query, - update - ) + arr_filter = [ + {'s.name': site_name}, + {'f._id': ObjectId(file_id)} + ] + else: + update = { + "$set": {"files.$[].sites.$[s]": elem} + } + + arr_filter = [ + {'s.name': site_name} + ] + + self.connection.database[collection].update_one( + query, + update, + upsert=True, + array_filters=arr_filter + ) def get_loop_delay(self, project_name): """ @@ -703,44 +709,35 @@ class SyncServer(PypeModule, ITrayModule): """Show dialog to enter credentials""" self.widget.show() - def _get_success_dict(self, file_index, site_index, new_file_id): + def _get_success_dict(self, new_file_id): """ Provide success metadata ("id", "created_dt") to be stored in Db. Used in $set: "DICT" part of query. Sites are array inside of array(file), so real indexes for both file and site are needed for upgrade in DB. Args: - file_index: (int) - index of modified file - site_index: (int) - index of modified site of modified file new_file_id: id of created file Returns: (dictionary) """ - val = {"files.{}.sites.{}.id".format(file_index, site_index): - new_file_id, - "files.{}.sites.{}.created_dt".format(file_index, site_index): - datetime.utcnow()} + val = {"files.$[f].sites.$[s].id": new_file_id, + "files.$[f].sites.$[s].created_dt": datetime.utcnow()} return val - def _get_error_dict(self, file_index, site_index, - error="", tries="", progress=""): + def _get_error_dict(self, error="", tries="", progress=""): """ Provide error metadata to be stored in Db. Used for set (error and tries provided) or unset mode. Args: - file_index: (int) - index of modified file - site_index: (int) - index of modified site of modified file error: (string) - message tries: how many times failed Returns: (dictionary) """ - val = {"files.{}.sites.{}.last_failed_dt". - format(file_index, site_index): datetime.utcnow(), - "files.{}.sites.{}.error".format(file_index, site_index): error, - "files.{}.sites.{}.tries".format(file_index, site_index): tries, - "files.{}.sites.{}.progress".format(file_index, site_index): - progress + val = {"files.$[f].sites.$[s].last_failed_dt": datetime.utcnow(), + "files.$[f].sites.$[s].error": error, + "files.$[f].sites.$[s].tries": tries, + "files.$[f].sites.$[s].progress": progress } return val @@ -768,20 +765,16 @@ class SyncServer(PypeModule, ITrayModule): _, rec = self._get_provider_rec(file.get("sites", []), provider) return rec.get("tries", 0) - def _get_progress_dict(self, file_index, site_index, progress): + def _get_progress_dict(self, progress): """ Provide progress metadata to be stored in Db. Used during upload/download for GUI to show. Args: - file_index: (int) - index of modified file - site_index: (int) - index of modified site of modified file progress: (float) - 0-1 progress of upload/download Returns: (dictionary) """ - val = {"files.{}.sites.{}.progress". - format(file_index, site_index): progress - } + val = {"files.$[f].sites.$[s].progress": progress} return val def _get_local_file_path(self, file, local_root): diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 48c93fc7e5..d0ff28ba22 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -139,6 +139,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.sync_server = sync_server self._selected_id = None # keep last selected _id + self.representation_id = None self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -202,7 +203,8 @@ class SyncRepresentationWidget(QtWidgets.QWidget): def _selection_changed(self, new_selection): index = self.selection_model.currentIndex() - self._selected_id = self.table_view.model().data(index, Qt.UserRole) + self.representation_id = \ + self.table_view.model().data(index, Qt.UserRole) def _set_selection(self): """ @@ -236,6 +238,66 @@ class SyncRepresentationWidget(QtWidgets.QWidget): if not point_index.isValid(): return + self.item = self.table_view.model()._data[point_index.row()] + self.representation_id = self.item._id + log.debug("menu representation _id:: {}". + format(self.representation_id)) + + menu = QtWidgets.QMenu() + actions_mapping = {} + + if self.item.state == STATUS[1]: + action = QtWidgets.QAction("Open error detail") + actions_mapping[action] = self._show_detail + menu.addAction(action) + + remote_site, remote_progress = self.item.remote_site.split() + if float(remote_progress) == 1.0: + action = QtWidgets.QAction("Reset local site") + actions_mapping[action] = self._reset_local_site + menu.addAction(action) + + local_site, local_progress = self.item.local_site.split() + if float(local_progress) == 1.0: + action = QtWidgets.QAction("Reset remote site") + actions_mapping[action] = self._reset_remote_site + menu.addAction(action) + + if not actions_mapping: + action = QtWidgets.QAction("< No action >") + actions_mapping[action] = None + menu.addAction(action) + + result = menu.exec_(QtGui.QCursor.pos()) + if result: + to_run = actions_mapping[result] + if to_run: + to_run() + + def _reset_local_site(self): + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + 'local' + ) + self.table_view.model().refresh() + + def _reset_remote_site(self): + """ + Removes errors or success metadata for particular file >> forces + redo of upload/download + """ + self.sync_server.reset_provider_for_file( + self.table_view.model()._project, + self.representation_id, + 'remote' + ) + self.table_view.model().refresh() + class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 20 @@ -517,7 +579,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): value = self.data(index, Qt.UserRole) if value == id: return index - return index + return None def get_default_query(self, limit=0): """ @@ -916,13 +978,13 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): menu.addAction(action) remote_site, remote_progress = self.item.remote_site.split() - if remote_progress == '1': + if float(remote_progress) == 1.0: action = QtWidgets.QAction("Reset local site") actions_mapping[action] = self._reset_local_site menu.addAction(action) local_site, local_progress = self.item.local_site.split() - if local_progress == '1': + if float(local_progress) == 1.0: action = QtWidgets.QAction("Reset remote site") actions_mapping[action] = self._reset_remote_site menu.addAction(action) @@ -946,8 +1008,9 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, - self.item._id, - 'local') + 'local', + self.item._id) + self.table_view.model().refresh() def _reset_remote_site(self): """ @@ -957,8 +1020,9 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.sync_server.reset_provider_for_file( self.table_view.model()._project, self.representation_id, - self.item._id, - 'remote') + 'remote', + self.item._id) + self.table_view.model().refresh() class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): @@ -1208,7 +1272,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): value = self.data(index, Qt.UserRole) if value == id: return index - return index + return None def get_default_query(self, limit=0): """ From a70e1f8a7b111585d75814ce8c6ad84baa2794d5 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Feb 2021 18:46:16 +0100 Subject: [PATCH 06/28] SyncServer GUI - added add_site and remove_site as public facing api app.py contains redundant items in menu for easier testing. Both will be removed later. --- pype/modules/sync_server/sync_server.py | 192 +++++++++++++++++++++--- pype/modules/sync_server/tray/app.py | 25 +++ 2 files changed, 192 insertions(+), 25 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index db5180f0b1..758cf63eaf 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -42,8 +42,11 @@ class SyncServer(PypeModule, ITrayModule): that are marked to be in different location than 'studio' (temporary), checks if 'created_dt' field is present denoting successful sync with provider destination. - Sites structure is created during publish and by default it will - always contain 1 record with "name" == self.presets["publish_site"] and + Sites structure is created during publish OR by calling 'add_site' + method. + + By default it will always contain 1 record with + "name" == self.presets["publish_site"] and filled "created_dt" AND 1 or multiple records for all defined remote sites, where "created_dt" is not present. This highlights that file should be uploaded to @@ -119,6 +122,56 @@ class SyncServer(PypeModule, ITrayModule): self.action_show_widget = None + # public facing API + def add_site(self, collection, representation_id, site_name=None): + """ + Adds new site to representation to be synced. + + 'collection' must have synchronization enabled (globally or + project only) + + Used as a API endpoint from outside applications (Loader etc) + + Args: + collection (string): project name (must match DB) + representation_id (string): MongoDB _id value + site_name (string): name of configured and active site + + Returns: + throws ValueError if any issue + """ + if not self.get_synced_preset(collection): + raise ValueError("Project not configured") + + if not site_name: + site_name = self.DEFAULT_SITE + + self.reset_provider_for_file(collection, + representation_id, + site_name=site_name) + + # public facing API + def remove_site(self, collection, representation_id, site_name): + """ + Removes 'site_name' for particular 'representation_id' on + 'collection' + + Args: + collection (string): project name (must match DB) + representation_id (string): MongoDB _id value + site_name (string): name of configured and active site + + Returns: + throws ValueError if any issue + """ + if not self.get_synced_preset(collection): + raise ValueError("Project not configured") + + self.reset_provider_for_file(collection, + representation_id, + site_name=site_name, + remove=True) + def connect_with_modules(self, *_a, **kw): return @@ -640,20 +693,32 @@ class SyncServer(PypeModule, ITrayModule): return -1, None def reset_provider_for_file(self, collection, representation_id, - side, file_id=None): + side=None, file_id=None, site_name=None, + remove=False): """ Reset information about synchronization for particular 'file_id' and provider. Useful for testing or forcing file to be reuploaded. + + 'side' and 'site_name' are disjunctive. + + 'side' is used for resetting local or remote side for + current user for repre. + + 'site_name' is used to set synchronization for particular site. + Should be used when repre should be synced to new site. + Args: collection (string): name of project (eg. collection) in DB representation_id(string): _id of representation file_id (string): file _id in representation side (string): local or remote side + site_name (string): for adding new site + remove (bool): if True remove site altogether + Returns: - None + throws ValueError """ - # TODO - implement reset for ALL files or ALL sites query = { "_id": ObjectId(representation_id) } @@ -662,33 +727,35 @@ class SyncServer(PypeModule, ITrayModule): if not representation: raise ValueError("Representation {} not found in {}". format(representation_id, collection)) + if side and site_name: + raise ValueError("Misconfiguration, only one of side and " + + "site_name arguments should be passed.") local_site, remote_site = self.get_sites_for_project(collection) - if side == 'local': - site_name = local_site - else: - site_name = remote_site + if side: + if side == 'local': + site_name = local_site + else: + site_name = remote_site elem = {"name": site_name} - if file_id: - update = { - "$set": {"files.$[f].sites.$[s]": elem} - } + if file_id: # reset site for particular file + self._reset_site_for_file(collection, query, + elem, file_id, site_name) + elif side: # reset site for whole representation + self._reset_site(collection, query, elem, site_name) + elif remove: # remove site for whole representation + self._remove_site(collection, query, representation, site_name) + else: # add new site to all files for representation + self._add_site(collection, query, representation, elem, site_name) - arr_filter = [ - {'s.name': site_name}, - {'f._id': ObjectId(file_id)} - ] - else: - update = { - "$set": {"files.$[].sites.$[s]": elem} - } - - arr_filter = [ - {'s.name': site_name} - ] + def _update_site(self, collection, query, update, arr_filter): + """ + Auxiliary method to call update_one function on DB + Used for refactoring ugly reset_provider_for_file + """ self.connection.database[collection].update_one( query, update, @@ -696,6 +763,81 @@ class SyncServer(PypeModule, ITrayModule): array_filters=arr_filter ) + def _reset_site_for_file(self, collection, query, + elem, file_id, site_name): + """ + Resets 'site_name' for 'file_id' on representation in 'query' on + 'collection' + """ + update = { + "$set": {"files.$[f].sites.$[s]": elem} + } + arr_filter = [ + {'s.name': site_name}, + {'f._id': ObjectId(file_id)} + ] + + self._update_site(collection, query, update, arr_filter) + + def _reset_site(self, collection, query, elem, site_name): + """ + Resets 'site_name' for all files of representation in 'query' + """ + update = { + "$set": {"files.$[].sites.$[s]": elem} + } + + arr_filter = [ + {'s.name': site_name} + ] + + self._update_site(collection, query, update, arr_filter) + + def _remove_site(self, collection, query, representation, site_name): + """ + Removes 'site_name' for 'representation' in 'query' + + Throws ValueError if 'site_name' not found on 'representation' + """ + found = False + for file in representation.pop().get("files"): + for site in file.get("sites"): + if site["name"] == site_name: + found = True + break + if not found: + msg = "Site {} not found".format(site_name) + log.info(msg) + raise ValueError(msg) + + update = { + "$pull": {"files.$[].sites": {"name": site_name}} + } + arr_filter = [] + + self._update_site(collection, query, update, arr_filter) + + def _add_site(self, collection, query, representation, elem, site_name): + """ + Adds 'site_name' to 'representation' on 'collection' + + Throws ValueError if already present + """ + for file in representation.pop().get("files"): + for site in file.get("sites"): + if site["name"] == site_name: + msg = "Site {} already present".format(site_name) + log.info(msg) + raise ValueError(msg) + + update = { + "$push": {"files.$[].sites": elem} + } + + arr_filter = [] + + self._update_site(collection, query, update, arr_filter) + def get_loop_delay(self, project_name): """ Return count of seconds before next synchronization loop starts diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index d0ff28ba22..5ed8b6fa84 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -263,6 +263,14 @@ class SyncRepresentationWidget(QtWidgets.QWidget): actions_mapping[action] = self._reset_remote_site menu.addAction(action) + action = QtWidgets.QAction("Add site site TEMP") + actions_mapping[action] = self._add_site + menu.addAction(action) + + action = QtWidgets.QAction("Remove site site TEMP") + actions_mapping[action] = self._remove_site + menu.addAction(action) + if not actions_mapping: action = QtWidgets.QAction("< No action >") actions_mapping[action] = None @@ -274,6 +282,23 @@ class SyncRepresentationWidget(QtWidgets.QWidget): if to_run: to_run() + # temporary here for testing, will be removed TODO + def _add_site(self): + log.info(self.representation_id) + self.sync_server.add_site( + self.table_view.model()._project, + self.representation_id, + 'new_site' + ) + + def _remove_site(self): + log.info(self.representation_id) + self.sync_server.remove_site( + self.table_view.model()._project, + self.representation_id, + 'new_site' + ) + def _reset_local_site(self): """ Removes errors or success metadata for particular file >> forces From b56e9cfb439de601c717e8aed9a8f05232842a39 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Feb 2021 18:59:27 +0100 Subject: [PATCH 07/28] SyncServer GUI - fix - selection kept disappearing --- pype/modules/sync_server/tray/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 5ed8b6fa84..c8d9350848 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -203,7 +203,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): def _selection_changed(self, new_selection): index = self.selection_model.currentIndex() - self.representation_id = \ + self._selected_id = \ self.table_view.model().data(index, Qt.UserRole) def _set_selection(self): From 2aea417e4095cbbabaa2c8b67b3300b003304c70 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Feb 2021 19:31:40 +0100 Subject: [PATCH 08/28] SyncServer GUI - added Open in explorer to menu --- pype/modules/sync_server/tray/app.py | 64 ++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index c8d9350848..ec20ee1025 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1,8 +1,10 @@ from Qt import QtWidgets, QtCore, QtGui from Qt.QtCore import Qt -from pype.tools.settings.settings.widgets.base import ProjectListWidget import attr import os +import sys +import subprocess +from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.tools.settings.settings import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp @@ -246,6 +248,10 @@ class SyncRepresentationWidget(QtWidgets.QWidget): menu = QtWidgets.QMenu() actions_mapping = {} + action = QtWidgets.QAction("Open in explorer") + actions_mapping[action] = self._open_in_explorer + menu.addAction(action) + if self.item.state == STATUS[1]: action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail @@ -323,6 +329,24 @@ class SyncRepresentationWidget(QtWidgets.QWidget): ) self.table_view.model().refresh() + def _open_in_explorer(self): + if not self.item: + return + + fpath = self.item.path + fpath = os.path.normpath(os.path.dirname(fpath)) + + if os.path.isdir(fpath): + if 'win' in sys.platform: # windows + subprocess.Popen('explorer "%s"' % fpath) + elif sys.platform == 'darwin': # macOS + subprocess.Popen(['open', fpath]) + else: # linux + try: + subprocess.Popen(['xdg-open', fpath]) + except OSError: + raise OSError('unsupported xdg-open call??') + class SyncRepresentationModel(QtCore.QAbstractTableModel): PAGE_SIZE = 20 @@ -368,6 +392,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): files_size = attr.ib(default=None) priority = attr.ib(default=None) state = attr.ib(default=None) + path = attr.ib(default=None) def __init__(self, sync_server, header, project=None): super(SyncRepresentationModel, self).__init__() @@ -470,6 +495,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self._total_records = count for repre in result.get("paginatedResults"): context = repre.get("context").pop() + data = repre.get("data").pop() files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] @@ -503,7 +529,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): repre.get("files_count", 1), repre.get("files_size", 0), 1, - STATUS[repre.get("status", -1)] + STATUS[repre.get("status", -1)], + data.get("path") ) self._data.append(item) @@ -706,6 +733,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): '_id': '$_id', # pass through context - same for representation 'context': {'$addToSet': '$context'}, + 'data': {'$addToSet': '$data'}, # pass through files as a list 'files': {'$addToSet': '$files'}, # count how many files @@ -773,6 +801,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "context.asset": 1, "context.version": 1, "context.representation": 1, + "data.path": 1, "files": 1, 'files_count': 1, "files_size": 1, @@ -997,6 +1026,10 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): menu = QtWidgets.QMenu() actions_mapping = {} + action = QtWidgets.QAction("Open in explorer") + actions_mapping[action] = self._open_in_explorer + menu.addAction(action) + if self.item.state == STATUS[1]: action = QtWidgets.QAction("Open error detail") actions_mapping[action] = self._show_detail @@ -1049,6 +1082,24 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): self.item._id) self.table_view.model().refresh() + def _open_in_explorer(self): + if not self.item: + return + + fpath = self.item.path + fpath = os.path.normpath(os.path.dirname(fpath)) + + if os.path.isdir(fpath): + if 'win' in sys.platform: # windows + subprocess.Popen('explorer "%s"' % fpath) + elif sys.platform == 'darwin': # macOS + subprocess.Popen(['open', fpath]) + else: # linux + try: + subprocess.Popen(['xdg-open', fpath]) + except OSError: + raise OSError('unsupported xdg-open call??') + class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ @@ -1088,6 +1139,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): state = attr.ib(default=None) tries = attr.ib(default=None) error = attr.ib(default=None) + path = attr.ib(default=None) def __init__(self, sync_server, header, _id, project=None): super(SyncRepresentationDetailModel, self).__init__() @@ -1189,6 +1241,7 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): for repre in result.get("paginatedResults"): # log.info("!!! repre:: {}".format(repre)) files = repre.get("files", []) + data = repre.get("data") if isinstance(files, dict): # aggregate returns dictionary files = [files] @@ -1224,7 +1277,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 1, STATUS[repre.get("status", -1)], repre.get("tries"), - '\n'.join(errors) + '\n'.join(errors), + data.get("path") + ) self._data.append(item) self._rec_loaded += 1 @@ -1490,7 +1545,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): ], 'default': -1 } - } + }, + 'data.path': 1 } From 4195549ff926bd8ab3192741c449f2cd5bac527a Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 Feb 2021 18:51:55 +0100 Subject: [PATCH 09/28] SyncServer GUI - added Pause functionality Allows pausing on representation, project or server level. --- pype/modules/sync_server/sync_server.py | 157 ++++++++++++++++- pype/modules/sync_server/tray/app.py | 215 ++++++++++++++++++++---- 2 files changed, 339 insertions(+), 33 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 758cf63eaf..7efee838d9 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -121,6 +121,9 @@ class SyncServer(PypeModule, ITrayModule): self.sync_server_thread = None # asyncio requires new thread self.action_show_widget = None + self._paused = False + self._paused_projects = set() + self._paused_representations = set() # public facing API def add_site(self, collection, representation_id, site_name=None): @@ -172,6 +175,109 @@ class SyncServer(PypeModule, ITrayModule): site_name=site_name, remove=True) + def pause_representation(self, collection, representation_id, site_name): + """ + Sets 'representation_id' as paused, eg. no syncing should be + happening on it. + + Args: + collection (string): project name + representation_id (string): MongoDB objectId value + site_name (string): 'gdrive', 'studio' etc. + """ + log.info("Pausing SyncServer for {}".format(representation_id)) + self._paused_representations.add(representation_id) + self.reset_provider_for_file(collection, representation_id, + site_name=site_name, pause=True) + + def unpause_representation(self, collection, representation_id, site_name): + """ + Sets 'representation_id' as unpaused. + + Does not fail or warn if repre wasn't paused. + + Args: + collection (string): project name + representation_id (string): MongoDB objectId value + site_name (string): 'gdrive', 'studio' etc. + """ + log.info("Unpausing SyncServer for {}".format(representation_id)) + try: + self._paused_representations.remove(representation_id) + except KeyError: + pass + # self.paused_representations is not persistent + self.reset_provider_for_file(collection, representation_id, + site_name=site_name, pause=False) + + def is_representation_paused(self, representation_id): + """ + Returns if 'representation_id' is paused or not. + + Args: + representation_id (string): MongoDB objectId value + Returns: + (bool) + """ + return representation_id in self._paused_representations + + def pause_project(self, project_name): + """ + Sets 'project_name' as paused, eg. no syncing should be + happening on all representation inside. + + Args: + project_name (string): collection name + """ + log.info("Pausing SyncServer for {}".format(project_name)) + self._paused_projects.add(project_name) + + def unpause_project(self, project_name): + """ + Sets 'project_name' as unpaused + + Does not fail or warn if project wasn't paused. + + Args: + project_name (string): collection name + """ + log.info("Unpausing SyncServer for {}".format(project_name)) + try: + self._paused_projects.remove(project_name) + except KeyError: + pass + + def is_project_paused(self, project_name): + """ + Returns if 'project_name' is paused or not. + + Args: + project_name (string): collection name + Returns: + (bool) + """ + return project_name in self._paused_projects + + def pause_server(self): + """ + Pause sync server + + It won't check anything, not uploading/downloading... + """ + log.info("Pausing SyncServer") + self._paused = True + + def unpause_server(self): + """ + Unpause server + """ + log.info("Unpausing SyncServer") + self._paused = False + + def is_paused(self): + """ Is server paused """ + return self._paused + def connect_with_modules(self, *_a, **kw): return @@ -694,7 +800,7 @@ class SyncServer(PypeModule, ITrayModule): def reset_provider_for_file(self, collection, representation_id, side=None, file_id=None, site_name=None, - remove=False): + remove=False, pause=None): """ Reset information about synchronization for particular 'file_id' and provider. @@ -715,6 +821,7 @@ class SyncServer(PypeModule, ITrayModule): side (string): local or remote side site_name (string): for adding new site remove (bool): if True remove site altogether + pause (bool or None): if True - pause, False - unpause Returns: throws ValueError @@ -747,6 +854,9 @@ class SyncServer(PypeModule, ITrayModule): self._reset_site(collection, query, elem, site_name) elif remove: # remove site for whole representation self._remove_site(collection, query, representation, site_name) + elif pause is not None: + self._pause_unpause_site(collection, query, + representation, site_name, pause) else: # add new site to all files for representation self._add_site(collection, query, representation, elem, site_name) @@ -817,6 +927,42 @@ class SyncServer(PypeModule, ITrayModule): self._update_site(collection, query, update, arr_filter) + + def _pause_unpause_site(self, collection, query, + representation, site_name, pause): + """ + Pauses/unpauses all files for 'representation' based on 'pause' + + Throws ValueError if 'site_name' not found on 'representation' + """ + found = False + site = None + for file in representation.pop().get("files"): + for site in file.get("sites"): + if site["name"] == site_name: + found = True + break + if not found: + msg = "Site {} not found".format(site_name) + log.info(msg) + raise ValueError(msg) + + if pause: + site['paused'] = pause + else: + if site.get('paused'): + site.pop('paused') + + update = { + "$set": {"files.$[].sites.$[s]": site} + } + + arr_filter = [ + {'s.name': site_name} + ] + + self._update_site(collection, query, update, arr_filter) + def _add_site(self, collection, query, representation, elem, site_name): """ Adds 'site_name' to 'representation' on 'collection' @@ -1014,11 +1160,14 @@ class SyncServerThread(threading.Thread): """ try: - while self.is_running: + while self.is_running and not self.module.is_paused(): import time start_time = None for collection, preset in self.module.get_synced_presets().\ items(): + if self.module.is_project_paused(collection): + continue + start_time = time.time() sync_repres = self.module.get_sync_representations( collection, @@ -1046,6 +1195,10 @@ class SyncServerThread(threading.Thread): # building folder tree structure in memory # call only if needed, eg. DO_UPLOAD or DO_DOWNLOAD for sync in sync_repres: + if self.module.\ + is_representation_paused(sync['_id']): + continue + if limit <= 0: continue files = sync.get("files") or [] diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index ec20ee1025..fe5ecaa492 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -32,6 +32,7 @@ class SyncServerWindow(QtWidgets.QDialog): def __init__(self, sync_server, parent=None): super(SyncServerWindow, self).__init__(parent) + self.sync_server = sync_server self.setWindowFlags(QtCore.Qt.Window) self.setFocusPolicy(QtCore.Qt.StrongFocus) @@ -43,17 +44,25 @@ class SyncServerWindow(QtWidgets.QDialog): footer = QtWidgets.QWidget(self) footer.setFixedHeight(20) - container = QtWidgets.QWidget() + left_column = QtWidgets.QWidget(body) + left_column_layout = QtWidgets.QVBoxLayout(left_column) + projects = SyncProjectListWidget(sync_server, self) projects.refresh() # force selection of default + left_column_layout.addWidget(projects) + self.pause_btn = QtWidgets.QPushButton("Pause server") + + left_column_layout.addWidget(self.pause_btn) + left_column.setLayout(left_column_layout) + repres = SyncRepresentationWidget(sync_server, project=projects.current_project, parent=self) - + container = QtWidgets.QWidget() container_layout = QtWidgets.QHBoxLayout(container) container_layout.setContentsMargins(0, 0, 0, 0) split = QtWidgets.QSplitter() - split.addWidget(projects) + split.addWidget(left_column) split.addWidget(repres) split.setSizes([180, 950, 200]) container_layout.addWidget(split) @@ -82,6 +91,15 @@ class SyncServerWindow(QtWidgets.QDialog): lambda: repres.table_view.model().set_project( projects.current_project)) + self.pause_btn.clicked.connect(self._pause) + + def _pause(self): + if self.sync_server.is_paused(): + self.sync_server.unpause_server() + self.pause_btn.setText("Pause server") + else: + self.sync_server.pause_server() + self.pause_btn.setText("Unpause server") class SyncProjectListWidget(ProjectListWidget): """ @@ -91,6 +109,10 @@ class SyncProjectListWidget(ProjectListWidget): def __init__(self, sync_server, parent): super(SyncProjectListWidget, self).__init__(parent) self.sync_server = sync_server + self.project_list.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) + self.project_list.customContextMenuRequested.connect( + self._on_context_menu) + self.project_name = None def validate_context_change(self): return True @@ -112,6 +134,40 @@ class SyncProjectListWidget(ProjectListWidget): self.current_project = self.project_list.model().item(0). \ data(QtCore.Qt.DisplayRole) + def _on_context_menu(self, point): + point_index = self.project_list.indexAt(point) + if not point_index.isValid(): + return + + self.project_name = point_index.data(QtCore.Qt.DisplayRole) + + menu = QtWidgets.QMenu() + actions_mapping = {} + + action = None + if self.sync_server.is_project_paused(self.project_name): + action = QtWidgets.QAction("Unpause") + actions_mapping[action] = self._unpause + else: + action = QtWidgets.QAction("Pause") + actions_mapping[action] = self._pause + menu.addAction(action) + + result = menu.exec_(QtGui.QCursor.pos()) + if result: + to_run = actions_mapping[result] + if to_run: + to_run() + + def _pause(self): + if self.project_name: + self.sync_server.pause_project(self.project_name) + self.project_name = None + + def _unpause(self): + if self.project_name: + self.sync_server.unpause_project(self.project_name) + self.project_name = None class SyncRepresentationWidget(QtWidgets.QWidget): """ @@ -142,6 +198,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self._selected_id = None # keep last selected _id self.representation_id = None + self.site_name = None # to pause/unpause representation self.filter = QtWidgets.QLineEdit() self.filter.setPlaceholderText("Filter representations..") @@ -252,19 +309,38 @@ class SyncRepresentationWidget(QtWidgets.QWidget): actions_mapping[action] = self._open_in_explorer menu.addAction(action) - if self.item.state == STATUS[1]: - action = QtWidgets.QAction("Open error detail") - actions_mapping[action] = self._show_detail + local_site, local_progress = self.item.local_site.split() + remote_site, remote_progress = self.item.remote_site.split() + local_progress = float(local_progress) + remote_progress = float(remote_progress) + + # progress smaller then 1.0 --> in progress or queued + if local_progress < 1.0: + self.site_name = local_site + else: + self.site_name = remote_site + + if self.item.state in [STATUS[0], STATUS[2]]: + action = QtWidgets.QAction("Pause") + actions_mapping[action] = self._pause menu.addAction(action) - remote_site, remote_progress = self.item.remote_site.split() - if float(remote_progress) == 1.0: + if self.item.state == STATUS[3]: + action = QtWidgets.QAction("Unpause") + actions_mapping[action] = self._unpause + menu.addAction(action) + + # if self.item.state == STATUS[1]: + # action = QtWidgets.QAction("Open error detail") + # actions_mapping[action] = self._show_detail + # menu.addAction(action) + + if remote_progress == 1.0: action = QtWidgets.QAction("Reset local site") actions_mapping[action] = self._reset_local_site menu.addAction(action) - local_site, local_progress = self.item.local_site.split() - if float(local_progress) == 1.0: + if local_progress == 1.0: action = QtWidgets.QAction("Reset remote site") actions_mapping[action] = self._reset_remote_site menu.addAction(action) @@ -288,6 +364,21 @@ class SyncRepresentationWidget(QtWidgets.QWidget): if to_run: to_run() + self.table_view.model().refresh() + + def _pause(self): + self.sync_server.pause_representation(self.table_view.model()._project, + self.representation_id, + self.site_name) + self.site_name = None + + def _unpause(self): + self.sync_server.unpause_representation( + self.table_view.model()._project, + self.representation_id, + self.site_name) + self.site_name = None + # temporary here for testing, will be removed TODO def _add_site(self): log.info(self.representation_id) @@ -315,7 +406,6 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.representation_id, 'local' ) - self.table_view.model().refresh() def _reset_remote_site(self): """ @@ -327,7 +417,6 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.representation_id, 'remote' ) - self.table_view.model().refresh() def _open_in_explorer(self): if not self.item: @@ -465,6 +554,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): return self._header.index(value) def refresh(self, representations=None, load_records=0): + if self.sync_server.is_paused() or \ + self.sync_server.is_project_paused(self._project): + return + self.beginResetModel() self._data = [] self._rec_loaded = 0 @@ -513,8 +606,10 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): remote_updated = \ repre.get('updated_dt_remote').strftime("%Y%m%dT%H%M%SZ") - avg_progress_remote = repre.get('avg_progress_remote', '') - avg_progress_local = repre.get('avg_progress_local', '') + avg_progress_remote = _convert_progress( + repre.get('avg_progress_remote', '0')) + avg_progress_local = _convert_progress( + repre.get('avg_progress_local', '0')) item = self.SyncRepresentation( repre.get("_id"), @@ -727,7 +822,23 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'failed_local': { '$cond': [{'$size': "$order_local.last_failed_dt"}, 1, - 0]} + 0]}, + 'failed_local_tries': { + '$cond': [{'$size': '$order_local.tries'}, + {'$first': '$order_local.tries'}, + 0]}, + 'failed_remote_tries': { + '$cond': [{'$size': '$order_remote.tries'}, + {'$first': '$order_local.tries'}, + 0]}, + 'paused_remote': { + '$cond': [{'$size': "$order_remote.paused"}, + 1, + 0]}, + 'paused_local': { + '$cond': [{'$size': "$order_local.paused"}, + 1, + 0]}, }}, {'$group': { '_id': '$_id', @@ -745,7 +856,11 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): # select last touch of file 'updated_dt_remote': {'$max': "$updated_dt_remote"}, 'failed_remote': {'$sum': '$failed_remote'}, - 'failed_local': {'$sum': '$failed_local'}, + 'failed_local': {'$sum': '$paused_remote'}, + 'failed_local_tries': {'$sum': '$failed_local_tries'}, + 'failed_remote_tries': {'$sum': '$failed_remote_tries'}, + 'paused_remote': {'$sum': '$paused_remote'}, + 'paused_local': {'$sum': '$paused_local'}, 'updated_dt_local': {'$max': "$updated_dt_local"} }}, {"$project": self.projection}, @@ -809,19 +924,28 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 'avg_progress_local': 1, 'updated_dt_remote': 1, 'updated_dt_local': 1, + 'paused_remote': 1, + 'paused_local': 1, 'status': { '$switch': { 'branches': [ { 'case': { - '$or': [{'$eq': ['$avg_progress_remote', 0]}, - {'$eq': ['$avg_progress_local', 0]}]}, - 'then': 2 # Queued + '$or': ['$paused_remote', '$paused_local']}, + 'then': 3 # Paused }, { 'case': { - '$or': ['$failed_remote', '$failed_local']}, - 'then': 1 # Failed + '$or': [ + {'$gte': ['$failed_local_tries', 3]}, + {'$gte': ['$failed_remote_tries', 3]} + ]}, + 'then': 1}, + { + 'case': { + '$or': [{'$eq': ['$avg_progress_remote', 0]}, + {'$eq': ['$avg_progress_local', 0]}]}, + 'then': 2 # Queued }, { 'case': {'$or': [{'$and': [ @@ -835,10 +959,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): ]}, 'then': 0 # In progress }, - { - 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 # Paused - }, { 'case': {'$and': [ {'$eq': ['$avg_progress_remote', 1]}, @@ -1209,6 +1329,9 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): return str(self._header[section]) def refresh(self, representations=None, load_records=0): + if self.sync_server.is_paused(): + return + self.beginResetModel() self._data = [] self._rec_loaded = 0 @@ -1257,8 +1380,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): repre.get('updated_dt_remote').strftime( "%Y%m%dT%H%M%SZ") - progress_remote = repre.get('progress_remote', '') - progress_local = repre.get('progress_local', '') + progress_remote = _convert_progress( + repre.get('progress_remote', '0')) + progress_local = _convert_progress( + repre.get('progress_local', '0')) errors = [] if repre.get('failed_remote_error'): @@ -1429,6 +1554,14 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): } ] }}, + 'paused_remote': { + '$cond': [{'$size': "$order_remote.paused"}, + 1, + 0]}, + 'paused_local': { + '$cond': [{'$size': "$order_local.paused"}, + 1, + 0]}, 'failed_remote': { '$cond': [{'$size': "$order_remote.last_failed_dt"}, 1, @@ -1502,12 +1635,27 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'progress_local': 1, 'updated_dt_remote': 1, 'updated_dt_local': 1, + 'paused_remote': 1, + 'paused_local': 1, 'failed_remote_error': 1, 'failed_local_error': 1, 'tries': 1, 'status': { '$switch': { 'branches': [ + { + 'case': { + '$or': ['$paused_remote', '$paused_local']}, + 'then': 3 # Paused + }, + { + 'case': {'$and': [ + {'$or': ['$failed_remote', + '$failed_local']}, + {'$eq': ['$tries', 3]} + ]}, + 'then': 1 # Failed (3 tries) + }, { 'case': { '$or': [{'$eq': ['$progress_remote', 0]}, @@ -1531,10 +1679,6 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): ]}, 'then': 0 # In Progress }, - { - 'case': {'$eq': ['dummy_placeholder', 'paused']}, - 'then': 3 - }, { 'case': {'$and': [ {'$eq': ['$progress_remote', 1]}, @@ -1673,3 +1817,12 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): return "%3.1f%s%s" % (value, unit, suffix) value /= 1024.0 return "%.1f%s%s" % (value, 'Yi', suffix) + + +def _convert_progress(value): + try: + progress = float(value) + except (ValueError, TypeError) as _: + progress = 0.0 + + return progress From dc436645cf7828cad4cd9180372fad093e3380d8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 5 Feb 2021 19:06:33 +0100 Subject: [PATCH 10/28] SyncServer GUI - fix - download was broken Fixed order of arguments in method call --- pype/modules/sync_server/sync_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 7efee838d9..cebb666d4e 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -685,7 +685,6 @@ class SyncServer(PypeModule, ITrayModule): handler.download_file, remote_file, local_file, - False, self, collection, file, From 2c7862de999f51ab6287bc5492c6e241bcc74481 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 8 Feb 2021 12:53:25 +0100 Subject: [PATCH 11/28] SyncServer GUI - added some documentation --- pype/modules/sync_server/tray/app.py | 66 +++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index fe5ecaa492..196c94dd85 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -438,8 +438,26 @@ class SyncRepresentationWidget(QtWidgets.QWidget): class SyncRepresentationModel(QtCore.QAbstractTableModel): - PAGE_SIZE = 20 - REFRESH_SEC = 5000 + """ + Model for summary of representations. + + Groups files information per representation. Allows sorting and + full text filtering. + + Allows pagination, most of heavy lifting is being done on DB side. + Single model matches to single collection. When project is changed, + model is reset and refreshed. + + Args: + sync_server (SyncServer) - object to call server operations (update + db status, set site status...) + header (list) - names of visible columns + project (string) - collection name, all queries must be called on + a specific collection + + """ + PAGE_SIZE = 20 # default page size to query for + REFRESH_SEC = 5000 # in seconds, requery DB for new status DEFAULT_SORT = { "updated_dt_remote": -1, "_id": 1 @@ -459,8 +477,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): "status" # state ] - numberPopulated = QtCore.Signal(int) - @attr.s class SyncRepresentation: """ @@ -517,6 +533,12 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): @property def dbcon(self): + """ + Database object with preselected project (collection) to run DB + operations (find, aggregate). + + All queries should go through this (because of collection). + """ return self.sync_server.connection.database[self._project] def data(self, index, role): @@ -539,6 +561,12 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): return str(self._header[section]) def tick(self): + """ + Triggers refresh of model. + + Because of pagination, prepared (sorting, filtering) query needs + to be run on DB every X seconds. + """ self.refresh(representations=None, load_records=self._rec_loaded) self.timer.start(self.REFRESH_SEC) @@ -554,6 +582,21 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): return self._header.index(value) def refresh(self, representations=None, load_records=0): + """ + Reloads representations from DB if necessary, adds them to model. + + Runs periodically (every X seconds) or by demand (change of + sorting, filtering etc.) + + Emits 'modelReset' signal. + + Args: + representations (PaginationResult object): pass result of + aggregate query from outside - mostly for testing only + load_records (int) - enforces how many records should be + actually queried (scrolled a couple of times to list more + than single page of records) + """ if self.sync_server.is_paused() or \ self.sync_server.is_project_paused(self._project): return @@ -660,8 +703,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.endInsertRows() - self.numberPopulated.emit(items_to_fetch) # ?? - def sort(self, index, order): """ Summary sort per representation. @@ -744,7 +785,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): 0 - in progress 1 - failed 2 - queued - 3 - paused (not implemented yet) + 3 - paused 4 - finished on both sides are calculated and must be calculated in DB because of @@ -1224,6 +1265,17 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): """ List of all syncronizable files per single representation. + + Used in detail window accessible after clicking on single repre in the + summary. + + Args: + sync_server (SyncServer) - object to call server operations (update + db status, set site status...) + header (list) - names of visible columns + _id (string) - MongoDB _id of representation + project (string) - collection name, all queries must be called on + a specific collection """ PAGE_SIZE = 30 # TODO add filter filename From 1c938fc11ed1e4db599d6b0cfe238a17ba56335e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 8 Feb 2021 13:32:16 +0100 Subject: [PATCH 12/28] SyncServer GUI - updated search in summary model Added possibility of searching by _id --- pype/modules/sync_server/tray/app.py | 29 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 196c94dd85..f00e0b5f79 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -7,6 +7,7 @@ import subprocess from pype.tools.settings.settings.widgets.base import ProjectListWidget from pype.tools.settings.settings import style from avalon.tools.delegates import PrettyTimeDelegate, pretty_timestamp +from bson.objectid import ObjectId from pype.lib import PypeLogger @@ -607,6 +608,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if not representations: self.query = self.get_default_query(load_records) + from pprint import pformat + log.info(pformat(self.query)) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, @@ -923,25 +926,31 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): checked. If performance issues are found, '$text' and text indexes should be investigated. + + Fulltext searches in: + context.subset + context.asset + context.representation names AND _id (ObjectId) """ - if not self.filter: - return { + base_match = { "type": "representation", 'files.sites.name': {'$all': [self.local_site, self.remote_site]} - } + } + if not self.filter: + return base_match else: regex_str = '.*{}.*'.format(self.filter) - return { - "type": "representation", - '$or': [ + base_match['$or'] = [ {'context.subset': {'$regex': regex_str, '$options': 'i'}}, {'context.asset': {'$regex': regex_str, '$options': 'i'}}, {'context.representation': {'$regex': regex_str, - '$options': 'i'}}], - 'files.sites.name': {'$all': [self.local_site, - self.remote_site]} - } + '$options': 'i'}}] + + if ObjectId.is_valid(self.filter): + base_match['$or'] = [{'_id': ObjectId(self.filter)}] + + return base_match def get_default_projection(self): """ From 64935bbfa2d2e1905e4bb27079e0bc2b35737c50 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 8 Feb 2021 22:57:10 +0100 Subject: [PATCH 13/28] Create CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..567bb92773 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at info@pype.club. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq From 0562ab614d45bc97d2e3d1d99eeaf398147aedb8 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Mon, 8 Feb 2021 23:23:41 +0100 Subject: [PATCH 14/28] Create CONTRIBUTING.md add simple contribution guidelines. --- CONTRIBUTING.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..0e08bb516f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,49 @@ +## How to contribute to Pype + +#### **Did you find a bug?** + +1. Check in the issues and our [bug triage[(https://github.com/pypeclub/pype/projects/2) to make sure it wasn't reported already. +2. Ask on our [discord](http://pype.community/chat) Often, what appears as a bug, might be the intended behaviour for someone else. +3. Create a new issue. +4. Use the issue template for you PR please. + + +#### **Did you write a patch that fixes a bug?** + +- Open a new GitHub pull request with the patch. +- Ensure the PR description clearly describes the problem and solution. Include the relevant issue number if applicable. + + +#### **Do you intend to add a new feature or change an existing one?** + +- Open a new thread in the [github discussions](https://github.com/pypeclub/pype/discussions/new) +- Do not open issue untill the suggestion is discussed. We will convert accepted suggestions into backlog and point them to the relevant discussion thread to keep the context. + +#### **Do you have questions about the source code?** + +Open a new question on [github discussions](https://github.com/pypeclub/pype/discussions/new) + +## Branching Strategy + +As we move to 3.x as the primary supported version of pype and only keep 2.15 on bug bugfixes and client sponsored feature requests, we need to be very careful with merging strategy. + +We also use this opportunity to switch the branch naming. 3.0 production branch will no longer be called MASTER, but will be renamed to MAIN. Develop will stay as it is. + +A few important notes about 2.x and 3.x development: + +- 3.x features are not backported to 2.x unless specifically requested +- 3.x bugs and hotfixes can be ported to 2.x if they are relevant or severe +- 2.x features and bugs MUST be ported to 3.x at the same time + +## Pull Requests + +- Each 2.x PR MUST have a corresponding 3.x PR in github. Without 3.x PR, 2.x features will not be merged! Luckily most of the code is compatible, albeit sometimes in a different place after refactor. Porting from 2.x to 3.x should be really easy. +- Please keep the corresponding 2 and 3 PR names the same so they can be easily identified from the PR list page. +- Each 2.x PR should be labeled with `2.x-dev` label. + +Inside each PR, put a link to the corresponding PR + + + + +If a PR is targeted at 2.x release it must be labelled with 2x-dev label in Github. From 77a8cc34bf57af3923a30ace034657ef5b0d01b9 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Feb 2021 11:12:19 +0100 Subject: [PATCH 15/28] Fix for master version Master version document contained files.path with original version, not pointing to master --- .../publish/integrate_master_version.py | 132 ++++++++++++------ 1 file changed, 92 insertions(+), 40 deletions(-) diff --git a/pype/plugins/global/publish/integrate_master_version.py b/pype/plugins/global/publish/integrate_master_version.py index d82c3be075..7d72bb26d4 100644 --- a/pype/plugins/global/publish/integrate_master_version.py +++ b/pype/plugins/global/publish/integrate_master_version.py @@ -298,6 +298,62 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): repre["data"] = repre_data repre.pop("_id", None) + # Prepare paths of source and destination files + if len(published_files) == 1: + src_to_dst_file_paths.append( + (published_files[0], template_filled) + ) + else: + collections, remainders = clique.assemble(published_files) + if remainders or not collections or len(collections) > 1: + raise Exception(( + "Integrity error. Files of published representation " + "is combination of frame collections and single files." + "Collections: `{}` Single files: `{}`" + ).format(str(collections), + str(remainders))) + + src_col = collections[0] + + # Get head and tail for collection + frame_splitter = "_-_FRAME_SPLIT_-_" + anatomy_data["frame"] = frame_splitter + _anatomy_filled = anatomy.format(anatomy_data) + _template_filled = _anatomy_filled["master"]["path"] + head, tail = _template_filled.split(frame_splitter) + padding = int( + anatomy.templates["render"].get( + "frame_padding", + anatomy.templates["render"].get("padding") + ) + ) + + dst_col = clique.Collection( + head=head, padding=padding, tail=tail + ) + dst_col.indexes.clear() + dst_col.indexes.update(src_col.indexes) + for src_file, dst_file in zip(src_col, dst_col): + src_to_dst_file_paths.append( + (src_file, dst_file) + ) + + # replace original file name with master name in repre doc + for index in range(len(repre.get("files"))): + file = repre.get("files")[index] + file_name = os.path.basename(file.get('path')) + for src_file, dst_file in src_to_dst_file_paths: + src_file_name = os.path.basename(src_file) + if src_file_name == file_name: + repre["files"][index]["path"] = self._update_path( + anatomy, repre["files"][index]["path"], + src_file, dst_file) + + repre["files"][index]["hash"] = self._update_hash( + repre["files"][index]["hash"], + src_file_name, dst_file + ) + schema.validate(repre) repre_name_low = repre["name"].lower() @@ -333,46 +389,6 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): InsertOne(repre) ) - # Prepare paths of source and destination files - if len(published_files) == 1: - src_to_dst_file_paths.append( - (published_files[0], template_filled) - ) - continue - - collections, remainders = clique.assemble(published_files) - if remainders or not collections or len(collections) > 1: - raise Exception(( - "Integrity error. Files of published representation " - "is combination of frame collections and single files." - "Collections: `{}` Single files: `{}`" - ).format(str(collections), str(remainders))) - - src_col = collections[0] - - # Get head and tail for collection - frame_splitter = "_-_FRAME_SPLIT_-_" - anatomy_data["frame"] = frame_splitter - _anatomy_filled = anatomy.format(anatomy_data) - _template_filled = _anatomy_filled["master"]["path"] - head, tail = _template_filled.split(frame_splitter) - padding = int( - anatomy.templates["render"].get( - "frame_padding", - anatomy.templates["render"].get("padding") - ) - ) - - dst_col = clique.Collection( - head=head, padding=padding, tail=tail - ) - dst_col.indexes.clear() - dst_col.indexes.update(src_col.indexes) - for src_file, dst_file in zip(src_col, dst_col): - src_to_dst_file_paths.append( - (src_file, dst_file) - ) - self.path_checks = [] # Copy(hardlink) paths of source and destination files @@ -533,3 +549,39 @@ class IntegrateMasterVersion(pyblish.api.InstancePlugin): "type": "representation" })) return (master_version, master_repres) + + def _update_path(self, anatomy, path, src_file, dst_file): + """ + Replaces source path with new master path + + 'path' contains original path with version, must be replaced with + 'master' path (with 'master' label and without version) + + Args: + anatomy (Anatomy) - to get rootless style of path + path (string) - path from DB + src_file (string) - original file path + dst_file (string) - master file path + """ + _, rootless = anatomy.find_root_template_from_path( + dst_file + ) + _, rtls_src = anatomy.find_root_template_from_path( + src_file + ) + return path.replace(rtls_src, rootless) + + def _update_hash(self, hash, src_file_name, dst_file): + """ + Updates hash value with proper master name + """ + src_file_name = self._get_name_without_ext( + src_file_name) + master_file_name = self._get_name_without_ext( + dst_file) + return hash.replace(src_file_name, master_file_name) + + def _get_name_without_ext(self, value): + file_name = os.path.basename(value) + file_name, _ = os.path.splitext(file_name) + return file_name From 9079c12ed65162160867f0d6557c215ad1cce54b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Feb 2021 11:21:20 +0100 Subject: [PATCH 16/28] SyncServer GUI - fix proper usage of master version --- pype/modules/sync_server/tray/app.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index f00e0b5f79..b669592d0e 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -608,8 +608,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if not representations: self.query = self.get_default_query(load_records) - from pprint import pformat - log.info(pformat(self.query)) representations = self.dbcon.aggregate(self.query) self._add_page_records(self.local_site, self.remote_site, @@ -657,11 +655,16 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): avg_progress_local = _convert_progress( repre.get('avg_progress_local', '0')) + if context.get("version"): + version = "v{:0>3d}".format(context.get("version")) + else: + version = "master" + item = self.SyncRepresentation( repre.get("_id"), context.get("asset"), context.get("subset"), - "v{:0>3d}".format(context.get("version", 1)), + version, context.get("representation"), local_updated, remote_updated, From d05b25a038c04d85aea8c59222d5705cd68cfd49 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Feb 2021 11:24:03 +0100 Subject: [PATCH 17/28] SyncServer GUI - set download to overwrite local file Overwrite should be default for both upload/download --- pype/modules/sync_server/sync_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index cebb666d4e..127b86a285 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -689,7 +689,8 @@ class SyncServer(PypeModule, ITrayModule): collection, file, representation, - local_site + local_site, + True ) return file_id From 64ef2866536c6966cd2a0f9b4b6a53f0d9902d6b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Feb 2021 20:06:46 +0100 Subject: [PATCH 18/28] SyncServer GUI - added local_drive provider Added possibility to remove local file(s) for repre or whole project Updated icons Update menus --- .../providers/abstract_provider.py | 10 +- pype/modules/sync_server/providers/lib.py | 7 +- .../sync_server/providers/local_drive.py | 59 ++++ .../providers/resources/local_drive.png | Bin 0 -> 766 bytes .../providers/resources/studio.png | Bin 557 -> 1246 bytes pype/modules/sync_server/sync_server.py | 286 +++++++++++++----- pype/modules/sync_server/tray/app.py | 147 +++++++-- .../defaults/project_settings/global.json | 5 + 8 files changed, 399 insertions(+), 115 deletions(-) create mode 100644 pype/modules/sync_server/providers/local_drive.py create mode 100644 pype/modules/sync_server/providers/resources/local_drive.png diff --git a/pype/modules/sync_server/providers/abstract_provider.py b/pype/modules/sync_server/providers/abstract_provider.py index 6931373561..9130a06d94 100644 --- a/pype/modules/sync_server/providers/abstract_provider.py +++ b/pype/modules/sync_server/providers/abstract_provider.py @@ -3,6 +3,13 @@ from abc import ABCMeta, abstractmethod class AbstractProvider(metaclass=ABCMeta): + def __init__(self, site_name, tree=None, presets=None): + self.presets = None + self.active = False + self.site_name = site_name + + self.presets = presets + @abstractmethod def is_active(self): """ @@ -27,13 +34,14 @@ class AbstractProvider(metaclass=ABCMeta): pass @abstractmethod - def download_file(self, source_path, local_path): + def download_file(self, source_path, local_path, overwrite=True): """ Download file from provider into local system Args: source_path (string): absolute path on provider local_path (string): absolute path on local + overwrite (bool): default set to True Returns: None """ diff --git a/pype/modules/sync_server/providers/lib.py b/pype/modules/sync_server/providers/lib.py index a6a52f0624..144594ecbe 100644 --- a/pype/modules/sync_server/providers/lib.py +++ b/pype/modules/sync_server/providers/lib.py @@ -1,10 +1,6 @@ from enum import Enum from .gdrive import GDriveHandler - - -class Providers(Enum): - LOCAL = 'studio' - GDRIVE = 'gdrive' +from .local_drive import LocalDriveHandler class ProviderFactory: @@ -94,3 +90,4 @@ factory = ProviderFactory() # 7 denotes number of files that could be synced in single loop - learned by # trial and error factory.register_provider('gdrive', GDriveHandler, 7) +factory.register_provider('local_drive', LocalDriveHandler, 10) diff --git a/pype/modules/sync_server/providers/local_drive.py b/pype/modules/sync_server/providers/local_drive.py new file mode 100644 index 0000000000..a21dfa2c71 --- /dev/null +++ b/pype/modules/sync_server/providers/local_drive.py @@ -0,0 +1,59 @@ +from __future__ import print_function +import os.path +import shutil + +from pype.api import Logger +from .abstract_provider import AbstractProvider + +log = Logger().get_logger("SyncServer") + + +class LocalDriveHandler(AbstractProvider): + """ Handles required operations on mounted disks with OS """ + def is_active(self): + return True + + def upload_file(self, source_path, target_path, overwrite=True): + """ + Copies file from 'source_path' to 'target_path' + """ + if os.path.exists(source_path): + if overwrite: + shutil.copy(source_path, target_path) + else: + if os.path.exists(target_path): + raise ValueError("File {} exists, set overwrite". + format(target_path)) + + def download_file(self, source_path, local_path, overwrite=True): + """ + Download a file form 'source_path' to 'local_path' + """ + if os.path.exists(source_path): + if overwrite: + shutil.copy(source_path, local_path) + else: + if os.path.exists(local_path): + raise ValueError("File {} exists, set overwrite". + format(local_path)) + + def delete_file(self, path): + """ + Deletes a file at 'path' + """ + if os.path.exists(path): + os.remove(path) + + def list_folder(self, folder_path): + """ + Returns list of files and subfolder in a 'folder_path'. Non recurs + """ + lst = [] + if os.path.isdir(folder_path): + for (dir_path, dir_names, file_names) in os.walk(folder_path): + for name in file_names: + lst.append(os.path.join(dir_path, name)) + for name in dir_names: + lst.append(os.path.join(dir_path, name)) + + return lst \ No newline at end of file diff --git a/pype/modules/sync_server/providers/resources/local_drive.png b/pype/modules/sync_server/providers/resources/local_drive.png new file mode 100644 index 0000000000000000000000000000000000000000..b53bdccac9c52901da0bec2a85fc6fda59b669bb GIT binary patch literal 766 zcmVynN&WC%p?+)F?PMp*UQZ`=ce4l!r=Y8HggK*!V{x2c~J}XGvBQ z;9FZ;FDjKv>;YZZQ79Ddnx^@5RsyTtZlhQ%#vb6DBaujK_!y8}z@@%aD&cEy^>GY-URu;_HleDj^jZKN{mvqL z_4-6r9%P-? zBtc6ORI62_QYpBuD>qY_CDhskt*(oxy*&8v0B$%R9uVWCrsCIGL#!N1f=;Jn3H5@& zp$W_{Yv>UQCe;kIj;f6Ui~V3uJc^7l59_?yI@;ar*T8X{pPHu08nnn9meU5h#E(@s z_>Pe=&OtmTdxbNU>zzY{@1S0u7h=8W6ay+wOL|sE|b%ON*%4ZeRT*I3Bmxk4D2rujhOS9x+#i zZTtKCrKa7(8NGppOa>P|1MwN$poy_4An|y=`5bQ0$XF|$*HO;De+wWLW*|rsg+H10 zg^^x7waU{ynG%#zw92P%&X3f-jzn1m>eCxBy1`2PifzpOyi;WxA%I{*Lx07*qoM6N<$f;{MIFaQ7m literal 0 HcmV?d00001 diff --git a/pype/modules/sync_server/providers/resources/studio.png b/pype/modules/sync_server/providers/resources/studio.png index e95e9762f89cde6869c8e4fea1470e599d6a845e..d61b7832bddd14606f6f0c3757041ae0d36e37c9 100644 GIT binary patch delta 1227 zcmV;+1T_1t1l|ddBYy;?NklUyE`-H zd)=0Wt~W2f_RE`2XTINizxR9Z!~fnCnZVJbZvrXc`R>bEBY*o5=L|v!q-pw4e)92$ z?{)9XcRHO2^Nq$Q^?UaY$8mzUk00*>^z6kK_I>dBYp))@d-qSMOa%x)p>K*5iu@=F zk|d$)`<4QV0?&0Zb@7|&ciw&Psor0FJ%D9d6(egvXj(VWUcd@y(_A(SUDvVDYGHYK z8LF!EGmyb3ihl~nvF-JHs_TDvp64bBz!(SPyzf<&!75w${f}Gt^s~Xax0mE8DUul!3Pm`s>kZVvwryXcE2>Hl zx7!^&_V^ffKffE>w?Bac2VO#w#8l8nM@De3K8I>`7*FrmfgY_w<1n&WDVAUWz;ztI zP$+;gm5Q6hF?`R*&YjP|G)+uAH-Y@n5M_A!^nV%5&COx|{{3h)n-H2HisaJu48kA` z2Pm=~$1@CrWJkS~0t*W*jBeeEp?n@EPktyPqlR)E2YdGJp{8p#8qhQWRZ*#CLTK;< z-yNjLvK&bb7Xm#myG*KPa&i)-l8GDFZ(!4=O+;55$7r|P$YgW~u2FMpT82P9*Rux! z9Dm1=)ZAZ!Uh1^lC=?5L<={bV8{3B4w{PLjrIzHz|BZKuLlSMKUj_9J^(Go zX-dYcF~Uj(VHl#-YEh#}LC9hWF3`(Z)_+1ODhw+Hj|#R{WX724xt=F^Wn4f>(+7rf zYjF`HBO^F+shmTqN=kXq zQYu%pvZW$Oo*db17XO8XVOZ@0)M~ZPrAwD^ z=T41gx-688ygT%CI&3zwUIudVO3I|dWG4>~S1~g))9eGx&d&aI?AS3}|K(?}Aq&YP z<4}M59t`Bn}S$Vk+UB}u|K=SiBTZ^dzZ^B-MXd^+Vx1`_}P002ovPDHLkV1ncVQ0@Q# delta 533 zcmV+w0_y$V39ST>BYyw^b5ch_0Itp)=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi z!vFvd!vV){sAK>D0m4Z{K~zXf-IXCv13?glPk}@tk(J1rHpr&eD<^UA52&M(goH{= z0S69AgQ~2fvO!fME0MM3dz)FZZSSskIC#m+&b*!3*}dDji+}zf(lkBjcDp@zU6H+Z zyM1WOf8`b_f`SOH zDZ!+Of+tzzaUN-2h2l6qE?vOPEihNt!sc$_*9GkOOU5aq9*s`MBlZ?%+R>1mU5$>GDPSMWIb-dZial~7z`jI#f>+otCQBcnxhw5+3*m$G@~{_91dT>x z5B~waak|)K#gDWZv&$C{NK^;6%ymGkp)IAW(HQ6cJ%5F{xd?;o6mwWN$pK%x{$hL# z8!ybw#b%Ip4Ao#G=s#@tj31``oDjQu0S^|{OZQ(-*JQA+llYFtFIT`ic") @@ -372,6 +417,7 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.representation_id, self.site_name) self.site_name = None + self.message_generated.emit("Paused {}".format(self.representation_id)) def _unpause(self): self.sync_server.unpause_representation( @@ -379,23 +425,43 @@ class SyncRepresentationWidget(QtWidgets.QWidget): self.representation_id, self.site_name) self.site_name = None + self.message_generated.emit("Unpaused {}".format( + self.representation_id)) # temporary here for testing, will be removed TODO def _add_site(self): log.info(self.representation_id) - self.sync_server.add_site( - self.table_view.model()._project, - self.representation_id, - 'new_site' - ) + try: + self.sync_server.add_site( + self.table_view.model()._project, + self.representation_id, + 'local_0' + ) + self.message_generated.emit("Site local_0 added") + except ValueError as exp: + self.message_generated.emit("Error {}".format(str(exp))) def _remove_site(self): - log.info(self.representation_id) - self.sync_server.remove_site( - self.table_view.model()._project, - self.representation_id, - 'new_site' - ) + """ + Removes site record AND files. + + This is ONLY for representations stored on local site, which + cannot be same as SyncServer.DEFAULT_SITE. + + This could only happen when artist work on local machine, not + connected to studio mounted drives. + """ + log.info("Removing {}".format(self.representation_id)) + try: + self.sync_server.remove_site( + self.table_view.model()._project, + self.representation_id, + 'local_0', + True + ) + self.message_generated.emit("Site local_0 removed") + except ValueError as exp: + self.message_generated.emit("Error {}".format(str(exp))) def _reset_local_site(self): """ @@ -630,6 +696,14 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): if total_count: count = total_count.pop().get('count') self._total_records = count + + local_provider = _translate_provider_for_icon(self.sync_server, + self._project, + local_site) + remote_provider = _translate_provider_for_icon(self.sync_server, + self._project, + remote_site) + for repre in result.get("paginatedResults"): context = repre.get("context").pop() data = repre.get("data").pop() @@ -668,8 +742,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): context.get("representation"), local_updated, remote_updated, - '{} {}'.format(local_site, avg_progress_local), - '{} {}'.format(remote_site, avg_progress_remote), + '{} {}'.format(local_provider, avg_progress_local), + '{} {}'.format(remote_provider, avg_progress_remote), repre.get("files_count", 1), repre.get("files_size", 0), 1, @@ -754,6 +828,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): project (str): name of project """ self._project = project + self.local_site, self.remote_site = \ + self.sync_server.get_sites_for_project(self._project) self.refresh() def get_index(self, id): @@ -1048,11 +1124,11 @@ class SyncServerDetailWindow(QtWidgets.QDialog): body_layout.addWidget(container) body_layout.setContentsMargins(0, 0, 0, 0) - message = QtWidgets.QLabel() - message.hide() + self.message = QtWidgets.QLabel() + self.message.hide() footer_layout = QtWidgets.QVBoxLayout(footer) - footer_layout.addWidget(message) + footer_layout.addWidget(self.message) footer_layout.setContentsMargins(0, 0, 0, 0) layout = QtWidgets.QVBoxLayout(self) @@ -1088,6 +1164,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): def __init__(self, sync_server, _id=None, project=None, parent=None): super(SyncRepresentationDetailWidget, self).__init__(parent) + log.debug("Representation_id:{}".format(_id)) self.representation_id = _id self.item = None # set to item that mouse was clicked over @@ -1425,6 +1502,14 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): if total_count: count = total_count.pop().get('count') self._total_records = count + + local_provider = _translate_provider_for_icon(self.sync_server, + self._project, + local_site) + remote_provider = _translate_provider_for_icon(self.sync_server, + self._project, + remote_site) + for repre in result.get("paginatedResults"): # log.info("!!! repre:: {}".format(repre)) files = repre.get("files", []) @@ -1460,8 +1545,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): os.path.basename(file["path"]), local_updated, remote_updated, - '{} {}'.format(local_site, progress_local), - '{} {}'.format(remote_site, progress_remote), + '{} {}'.format(local_provider, progress_local), + '{} {}'.format(remote_provider, progress_remote), file.get('size', 0), 1, STATUS[repre.get("status", -1)], @@ -1890,3 +1975,15 @@ def _convert_progress(value): progress = 0.0 return progress + +def _translate_provider_for_icon(sync_server, project, site): + """ + Get provider for 'site' + + This is used for getting icon, 'studio' should have different icon + then local sites, even the provider 'local_drive' is same + + """ + if site == sync_server.DEFAULT_SITE: + return sync_server.DEFAULT_SITE + return sync_server.get_provider_for_site(project, site) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index d73026f686..9fef50aaf9 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -193,6 +193,11 @@ "provider": "gdrive", "credentials_url": "", "root": "/sync_testing/test" + }, + "studio": { + "provider": "local_drive", + "credentials_url": "", + "root": "" } } } From 854fece76dcf361cadb0011030b6c422fa305c6f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Feb 2021 11:19:25 +0100 Subject: [PATCH 19/28] SyncServer GUI - modified label for add site --- pype/modules/sync_server/tray/app.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 0c2f677bb2..6c2cd0feed 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -393,11 +393,10 @@ class SyncRepresentationWidget(QtWidgets.QWidget): action = QtWidgets.QAction("Completely remove from local") actions_mapping[action] = self._remove_site menu.addAction(action) - - action = QtWidgets.QAction("Add site site TEMP") - actions_mapping[action] = self._add_site - menu.addAction(action) - + else: + action = QtWidgets.QAction("Mark for sync to local") + actions_mapping[action] = self._add_site + menu.addAction(action) if not actions_mapping: action = QtWidgets.QAction("< No action >") @@ -431,13 +430,17 @@ class SyncRepresentationWidget(QtWidgets.QWidget): # temporary here for testing, will be removed TODO def _add_site(self): log.info(self.representation_id) + project_name = self.table_view.model()._project + local_site_name = self.sync_server.get_my_local_site(project_name) try: self.sync_server.add_site( self.table_view.model()._project, self.representation_id, - 'local_0' + local_site_name ) - self.message_generated.emit("Site local_0 added") + self.message_generated.emit( + "Site {} added for {}".format(local_site_name, + self.representation_id)) except ValueError as exp: self.message_generated.emit("Error {}".format(str(exp))) From 5afa8174725cc2031757f1f4865cdbbd204cb445 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Feb 2021 12:30:31 +0100 Subject: [PATCH 20/28] SyncServer GUI - added icons to project list --- pype/modules/sync_server/resources/paused.png | Bin 0 -> 692 bytes pype/modules/sync_server/resources/synced.png | Bin 0 -> 561 bytes pype/modules/sync_server/tray/app.py | 57 +++++++++++++++--- 3 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 pype/modules/sync_server/resources/paused.png create mode 100644 pype/modules/sync_server/resources/synced.png diff --git a/pype/modules/sync_server/resources/paused.png b/pype/modules/sync_server/resources/paused.png new file mode 100644 index 0000000000000000000000000000000000000000..c18d25d2f1010a1c2c6848db04d3eee0811f0257 GIT binary patch literal 692 zcmV;l0!#ggP)`(+AZ3CH>OJxzE1+0^iK zKVE)*Qdhdaa*paiG`U8rt<9djB#83pBR~%)aNCY|R~9vJM=BG?e8>Y|XdL0h634>HkMM`b zF&}z{R3^r;3UdS8#Vy@@h~))tSFSutD2fbrusjnClFTuqRDe?XFa#=de@@CxHAgvt zL6W82J?cS(iV&!nwIM<%1cA73AqVO~-_q*{CRg}8vU7=XdvkkZha@Y3*y4hqYmRd%skY=)huKgHfl5d# zNf;zEs;sRrN`)YxNGWmH3+mxCH-ZuYtDCbJT^PK@8@l|sTBsE&^X+V-GGs9xRP~qA apy4NU?fJ=s0t}x30000ZagM zTs5ND#JCU=(9$aLgAgTE2cjbP`ra@Gp@Ih<^WK|#?#H|5`p|&b($)t6BLM7%Un>CS zOUd}0_tJF`TiSXKU=cuHqtgcD0i;UF_;1&NAm;$+(zf4d!P|Bn?dp@;@W(a$D4(Ks zc}2$)02Vn>g#DtIoG~L9?hM_DOAgarA!G`BDApWZa4eswc~%qk0hOwOwgGs)z@3qfq4X*&b#JlTeiw82&JR?k~f+yuioAXPB*F zw%e$l6J>-P*rujp>QxssrI{T~tbfJJa4+7O@0s|$xWL&Q{QHaU))@MYpmbm>MDmjP zg`5fopK`W7s_{@jc;F_f!5(?;NAa)ckkIgf}mSIDIcD16mTpod)lk?rZdR zrGPKl45to3&h9UBSrRg+vQa%>DLZvr_M4!qA*g2o Date: Thu, 11 Feb 2021 12:37:17 +0100 Subject: [PATCH 21/28] SyncServer GUI - fix error view in detail --- pype/modules/sync_server/tray/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index ec33215314..387223940c 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -1208,6 +1208,7 @@ class SyncRepresentationDetailWidget(QtWidgets.QWidget): log.debug("Representation_id:{}".format(_id)) self.representation_id = _id self.item = None # set to item that mouse was clicked over + self.project = project self.sync_server = sync_server From 37b9b23e4b81eb09fcc4711d4cdec744babe0d09 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Feb 2021 13:04:50 +0100 Subject: [PATCH 22/28] SyncServer GUI - fix - removed unwanted UTC mongo driver automatically converts local date to UTC, pretty_date works with it fine --- pype/modules/sync_server/sync_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index e3b2445305..6caa1026a6 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1126,7 +1126,7 @@ class SyncServer(PypeModule, ITrayModule): (dictionary) """ val = {"files.$[f].sites.$[s].id": new_file_id, - "files.$[f].sites.$[s].created_dt": datetime.utcnow()} + "files.$[f].sites.$[s].created_dt": datetime.now()} return val def _get_error_dict(self, error="", tries="", progress=""): @@ -1139,7 +1139,7 @@ class SyncServer(PypeModule, ITrayModule): Returns: (dictionary) """ - val = {"files.$[f].sites.$[s].last_failed_dt": datetime.utcnow(), + val = {"files.$[f].sites.$[s].last_failed_dt": datetime.now(), "files.$[f].sites.$[s].error": error, "files.$[f].sites.$[s].tries": tries, "files.$[f].sites.$[s].progress": progress From cc58f364b8954a9e62e48126e8844b2b2b6dfff1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Feb 2021 13:36:00 +0100 Subject: [PATCH 23/28] SyncServer GUI - added functionality of pausing during upload/download to Gdrive Triggers error, process need to be reset for now for that repre --- pype/modules/sync_server/providers/gdrive.py | 8 +++++++ pype/modules/sync_server/sync_server.py | 23 ++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index 5bc6f21b38..fa6b3f82cd 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -351,6 +351,10 @@ class GDriveHandler(AbstractProvider): last_tick = status = response = None status_val = 0 while response is None: + if server.is_representation_paused(representation['_id'], + check_parents=True, + project_name=collection): + raise ValueError("Paused during process, please redo.") if status: status_val = float(status.progress()) if not last_tick or \ @@ -433,6 +437,10 @@ class GDriveHandler(AbstractProvider): last_tick = status = response = None status_val = 0 while response is None: + if server.is_representation_paused(representation['_id'], + check_parents=True, + project_name=collection): + raise ValueError("Paused during process, please redo.") if status: status_val = float(status.progress()) if not last_tick or \ diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 6caa1026a6..d3cee71ce0 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -236,16 +236,26 @@ class SyncServer(PypeModule, ITrayModule): self.reset_provider_for_file(collection, representation_id, site_name=site_name, pause=False) - def is_representation_paused(self, representation_id): + def is_representation_paused(self, representation_id, + check_parents=False, project_name=None): """ Returns if 'representation_id' is paused or not. Args: representation_id (string): MongoDB objectId value + check_parents (bool): check if parent project or server itself + are not paused + project_name (string): project to check if paused + + if 'check_parents', 'project_name' should be set too Returns: (bool) """ - return representation_id in self._paused_representations + condition = representation_id in self._paused_representations + if check_parents and project_name: + condition = condition or self.is_project_paused(project_name) \ + or self.is_paused() + return condition def pause_project(self, project_name): """ @@ -273,16 +283,21 @@ class SyncServer(PypeModule, ITrayModule): except KeyError: pass - def is_project_paused(self, project_name): + def is_project_paused(self, project_name, check_parents=False): """ Returns if 'project_name' is paused or not. Args: project_name (string): collection name + check_parents (bool): check if server itself + is not paused Returns: (bool) """ - return project_name in self._paused_projects + condition = project_name in self._paused_projects + if check_parents: + condition = condition or self.is_paused() + return condition def pause_server(self): """ From c8dc7a3fd4ee8d9a6167e91243ad9655de800513 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Feb 2021 17:12:45 +0100 Subject: [PATCH 24/28] SyncServer GUI - rename publish_site to active_site publish_site is too specific, active denotes we are syncing from, but also we are publishing too --- pype/modules/sync_server/sync_server.py | 46 +++++++++---------- pype/modules/sync_server/tray/app.py | 2 +- .../defaults/project_settings/global.json | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index d3cee71ce0..78fc20a055 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -47,7 +47,7 @@ class SyncServer(PypeModule, ITrayModule): method. By default it will always contain 1 record with - "name" == self.presets["publish_site"] and + "name" == self.presets["active_site"] and filled "created_dt" AND 1 or multiple records for all defined remote sites, where "created_dt" is not present. This highlights that file should be uploaded to @@ -77,8 +77,8 @@ class SyncServer(PypeModule, ITrayModule): Each Tray app has assigned its own self.presets["local_id"] used in sites as a name. Tray is searching only for records where name matches its - self.presets["publish_site"] + self.presets["remote_site"]. - "publish_site" could be storage in studio ('studio'), or specific + self.presets["active_site"] + self.presets["remote_site"]. + "active_site" could be storage in studio ('studio'), or specific "local_id" when user is working disconnected from home. If the local record has its "created_dt" filled, it is a source and process will try to upload the file to all defined remote sites. @@ -340,7 +340,7 @@ class SyncServer(PypeModule, ITrayModule): try: self.presets = self.get_synced_presets() - self.set_publish_sites(self.presets) + self.set_active_sites(self.presets) self.sync_server_thread = SyncServerThread(self) from .tray.app import SyncServerWindow self.widget = SyncServerWindow(self) @@ -366,7 +366,7 @@ class SyncServer(PypeModule, ITrayModule): Returns: None """ - if self.presets and self.publish_sites: + if self.presets and self.active_sites: self.sync_server_thread.start() else: log.info("No presets or active providers. " + @@ -429,7 +429,7 @@ class SyncServer(PypeModule, ITrayModule): sync_server_presets = settings["global"]["sync_server"]["config"] if settings["global"]["sync_server"]["enabled"]: - local_site = sync_server_presets.get("publish_site", + local_site = sync_server_presets.get("active_site", "studio").strip() remote_site = sync_server_presets.get("remote_site") @@ -478,9 +478,9 @@ class SyncServer(PypeModule, ITrayModule): return {} - def set_publish_sites(self, settings): + def set_active_sites(self, settings): """ - Sets 'self.publish_sites' as a dictionary from provided 'settings' + Sets 'self.active_sites' as a dictionary from provided 'settings' Format: { 'project_name' : ('provider_name', 'site_name') } @@ -488,7 +488,7 @@ class SyncServer(PypeModule, ITrayModule): settings (dict): all enabled project sync setting (sites labesl, retries count etc.) """ - self.publish_sites = {} + self.active_sites = {} initiated_handlers = {} for project_name, project_setting in settings.items(): for site_name, config in project_setting.get("sites").items(): @@ -500,16 +500,16 @@ class SyncServer(PypeModule, ITrayModule): initiated_handlers[config["provider"]] = handler if handler.is_active(): - if not self.publish_sites.get('project_name'): - self.publish_sites[project_name] = [] + if not self.active_sites.get('project_name'): + self.active_sites[project_name] = [] - self.publish_sites[project_name].append( + self.active_sites[project_name].append( (config["provider"], site_name)) - if not self.publish_sites: + if not self.active_sites: log.info("No sync sites active, no working credentials provided") - def get_publish_sites(self, project_name): + def get_active_sites(self, project_name): """ Returns active sites (provider configured and able to connect) per project. @@ -522,7 +522,7 @@ class SyncServer(PypeModule, ITrayModule): Format: { 'project_name' : ('provider_name', 'site_name') } """ - return self.publish_sites[project_name] + return self.active_sites[project_name] def get_provider_for_site(self, project_name, site): """ @@ -535,7 +535,7 @@ class SyncServer(PypeModule, ITrayModule): return "NA" @time_function - def get_sync_representations(self, collection, publish_site, remote_site): + def get_sync_representations(self, collection, active_site, remote_site): """ Get representations that should be synced, these could be recognised by presence of document in 'files.sites', where key is @@ -548,7 +548,7 @@ class SyncServer(PypeModule, ITrayModule): Args: collection (string): name of collection (in most cases matches project name - publish_site (string): identifier of current active site (could be + active_site (string): identifier of current active site (could be 'local_0' when working from home, 'studio' when working in the studio (default) remote_site (string): identifier of remote site I want to sync to @@ -567,7 +567,7 @@ class SyncServer(PypeModule, ITrayModule): { "files.sites": { "$elemMatch": { - "name": publish_site, + "name": active_site, "created_dt": {"$exists": True} } }}, { @@ -583,7 +583,7 @@ class SyncServer(PypeModule, ITrayModule): { "files.sites": { "$elemMatch": { - "name": publish_site, + "name": active_site, "created_dt": {"$exists": False}, "tries": {"$in": retries_arr} } @@ -611,7 +611,7 @@ class SyncServer(PypeModule, ITrayModule): (Eg. check if 'scene.ma' of lookdev.v10 should be synced to GDrive Always is comparing local record, eg. site with - 'name' == self.presets[PROJECT_NAME]['config']["publish_site"] + 'name' == self.presets[PROJECT_NAME]['config']["active_site"] Args: file (dictionary): of file from representation in Mongo @@ -1062,7 +1062,7 @@ class SyncServer(PypeModule, ITrayModule): return handler = None - sites = self.get_publish_sites(collection) + sites = self.get_active_sites(collection) for provider_name, provider_site_name in sites: if provider_site_name == site_name: handler = lib.factory.get_provider(provider_name, @@ -1303,11 +1303,11 @@ class SyncServerThread(threading.Thread): start_time = time.time() sync_repres = self.module.get_sync_representations( collection, - preset.get('config')["publish_site"], + preset.get('config')["active_site"], preset.get('config')["remote_site"] ) - local_site = preset.get('config')["publish_site"] + local_site = preset.get('config')["active_site"] remote_site = preset.get('config')["remote_site"] task_files_to_process = [] files_processed_info = [] diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index 387223940c..bcaf028c7b 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -172,7 +172,7 @@ class SyncProjectListWidget(ProjectListWidget): data(QtCore.Qt.DisplayRole) self.local_site = self.sync_server.get_synced_preset(project_name)\ - ['config']["publish_site"] + ['config']["active_site"] def _get_icon(self, status): if not self.icons.get(status): diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 9fef50aaf9..598ab09372 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -185,7 +185,7 @@ "local_id": "local_0", "retry_cnt": "3", "loop_delay": "60", - "publish_site": "studio", + "active_site": "studio", "remote_site": "gdrive" }, "sites": { From 92f8f1c409900dbfd25eb1f9dc571c97e3700c94 Mon Sep 17 00:00:00 2001 From: Milan Kolar Date: Fri, 12 Feb 2021 13:15:41 +0100 Subject: [PATCH 25/28] Update CONTRIBUTING.md --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e08bb516f..e9473eb4e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,10 @@ A few important notes about 2.x and 3.x development: Inside each PR, put a link to the corresponding PR +Of course if you want to contribute, feel free to make a PR to only 2.x/develop or develop, based on what you are using. While reviewing the PRs, we might convert the code to corresponding PR for the other release ourselves. + +We might also change the target of you PR to and intermediate branch, rather than `develop` if we feel it requires some extra work on our end. That way, we preserve all your commits so you don't loos out on the contribution credits. + From d55060d7dd96221ffbbe7d277919d337d6fed837 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Feb 2021 16:42:58 +0100 Subject: [PATCH 26/28] SyncServer GUI - implemented pulling roots from settings Updated defaults (empty) for Settings Updated schema for syncserver Small assorted fixes --- pype/modules/sync_server/sync_server.py | 136 +++++++++++------- pype/modules/sync_server/tray/app.py | 22 +-- .../defaults/project_settings/global.json | 17 ++- .../schema_project_syncserver.json | 12 +- .../settings/settings/widgets/item_types.py | 2 + 5 files changed, 123 insertions(+), 66 deletions(-) diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 78fc20a055..7e9f6f8dfd 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -1,4 +1,5 @@ from pype.api import ( + Anatomy, get_project_settings, get_current_project_settings) @@ -125,6 +126,7 @@ class SyncServer(PypeModule, ITrayModule): self._paused = False self._paused_projects = set() self._paused_representations = set() + self._anatomies = {} # public facing API def add_site(self, collection, representation_id, site_name=None): @@ -410,32 +412,17 @@ class SyncServer(PypeModule, ITrayModule): def is_running(self): return self.sync_server_thread.is_running - def get_sites_for_project(self, project_name=None): + def get_anatomy(self, project_name): """ - Checks if sync is enabled globally and on project. - In that case return local and remote site + Get already created or newly created anatomy for project Args: - project_name (str): + project_name (string): - Returns: - (tuple): of strings, labels for (local_site, remote_site) + Return: + (Anatomy) """ - if self.enabled: - if project_name: - settings = get_project_settings(project_name) - else: - settings = get_current_project_settings() - - sync_server_presets = settings["global"]["sync_server"]["config"] - if settings["global"]["sync_server"]["enabled"]: - local_site = sync_server_presets.get("active_site", - "studio").strip() - remote_site = sync_server_presets.get("remote_site") - - return local_site, remote_site - - return self.DEFAULT_SITE, None + return self._anatomies.get('project_name') or Anatomy(project_name) def get_synced_presets(self): """ @@ -443,6 +430,9 @@ class SyncServer(PypeModule, ITrayModule): Returns: (dict): of settings, keys are project names """ + if self.presets: # presets set already, do not call again and again + return self.presets + sync_presets = {} if not self.connection: self.connection = AvalonMongoDB() @@ -467,6 +457,11 @@ class SyncServer(PypeModule, ITrayModule): (dict): settings dictionary for the enabled project, empty if no settings or sync is disabled """ + # presets set already, do not call again and again + # self.log.debug("project preset {}".format(self.presets)) + if self.presets and self.presets.get(project_name): + return self.presets.get(project_name) + settings = get_project_settings(project_name) sync_settings = settings.get("global")["sync_server"] if not sync_settings: @@ -524,6 +519,18 @@ class SyncServer(PypeModule, ITrayModule): """ return self.active_sites[project_name] + def get_local_site(self, project_name): + """ + Returns active (mine) site for 'project_name' from settings + """ + return self.get_synced_preset(project_name)['config']['active_site'] + + def get_remote_site(self, project_name): + """ + Returns remote (theirs) site for 'project_name' from settings + """ + return self.get_synced_preset(project_name)['config']['remote_site'] + def get_provider_for_site(self, project_name, site): """ Return provider name for site. @@ -681,8 +688,9 @@ class SyncServer(PypeModule, ITrayModule): remote_file = self._get_remote_file_path(file, handler.get_roots_config() ) - local_root = representation.get("context", {}).get("root") - local_file = self._get_local_file_path(file, local_root) + + local_file = self.get_local_file_path(collection, + file.get("path", "")) target_folder = os.path.dirname(remote_file) folder_id = handler.create_folder(target_folder) @@ -691,7 +699,8 @@ class SyncServer(PypeModule, ITrayModule): err = "Folder {} wasn't created. Check permissions.".\ format(target_folder) raise NotADirectoryError(err) - _, remote_site = self.get_sites_for_project(collection) + + remote_site = self.get_remote_site(collection) loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.upload_file, @@ -727,22 +736,21 @@ class SyncServer(PypeModule, ITrayModule): with self.lock: handler = lib.factory.get_provider(provider_name, site_name, tree=tree, presets=preset) - remote_file = self._get_remote_file_path(file, - handler.get_roots_config() - ) - local_root = representation.get("context", {}).get("root") - local_file = self._get_local_file_path(file, local_root) + remote_file_path = self._get_remote_file_path( + file, handler.get_roots_config()) - local_folder = os.path.dirname(local_file) + local_file_path = self.get_local_file_path(collection, + file.get("path", "")) + local_folder = os.path.dirname(local_file_path) os.makedirs(local_folder, exist_ok=True) - local_site, _ = self.get_sites_for_project(collection) + local_site = self.get_local_site(collection) loop = asyncio.get_running_loop() file_id = await loop.run_in_executor(None, handler.download_file, - remote_file, - local_file, + remote_file_path, + local_file_path, self, collection, file, @@ -812,8 +820,9 @@ class SyncServer(PypeModule, ITrayModule): error_str = '' source_file = file.get("path", "") - log.debug("File {source_file} process {status} {error_str}". - format(status=status, + log.debug("File for {} - {source_file} process {status} {error_str}". + format(representation_id, + status=status, source_file=source_file, error_str=error_str)) @@ -897,7 +906,9 @@ class SyncServer(PypeModule, ITrayModule): raise ValueError("Misconfiguration, only one of side and " + "site_name arguments should be passed.") - local_site, remote_site = self.get_sites_for_project(collection) + local_site = self.get_local_site(collection) + remote_site = self.get_remote_site(collection) + if side: if side == 'local': site_name = local_site @@ -1082,12 +1093,11 @@ class SyncServer(PypeModule, ITrayModule): return representation = representation.pop() - local_root = representation.get("context", {}).get("root") for file in representation.get("files"): try: self.log.debug("Removing {}".format(file["path"])) - local_file = self._get_local_file_path(file, - local_root) + local_file = self.get_local_file_path(collection, + file.get("path", "")) os.remove(local_file) except IndexError: msg = "No file set for {}".format(representation_id) @@ -1098,6 +1108,14 @@ class SyncServer(PypeModule, ITrayModule): self.log.warning(msg) raise ValueError(msg) + try: + folder = os.path.dirname(local_file) + os.rmdir(folder) + except OSError: + msg = "folder {} cannot be removed".format(folder) + self.log.warning(msg) + raise ValueError(msg) + def get_my_local_site(self, project_name=None): """ Returns name of current user local_site @@ -1197,22 +1215,40 @@ class SyncServer(PypeModule, ITrayModule): val = {"files.$[f].sites.$[s].progress": progress} return val - def _get_local_file_path(self, file, local_root): + def get_local_file_path(self, collection, path): """ Auxiliary function for replacing rootless path with real path + Works with multi roots. + If root definition is not found in Settings, anatomy is used + Args: - file (dictionary): file info, get 'path' to file with {root} - local_root (string): value of {root} for local projects + collection (string): project name + path (dictionary): 'path' to file with {root} Returns: (string) - absolute path on local system """ - if not local_root: - raise ValueError("Unknown local root for file {}") - path = file.get("path", "") + local_active_site = self.get_local_site(collection) + root_config = \ + self.get_synced_preset(collection) \ + ["sites"][local_active_site]["root"] - return path.format(**{"root": local_root}) + if not root_config.get("root"): + root_config = {"root": root_config} + + try: + path = path.format(**root_config) + except KeyError: + try: + anatomy = self.get_anatomy(collection) + path = anatomy.fill_root(path) + except: + msg = "Error in resolving local root from anatomy" + self.log.error(msg) + raise ValueError(msg) + + return path def _get_remote_file_path(self, file, root_config): """ @@ -1227,8 +1263,12 @@ class SyncServer(PypeModule, ITrayModule): path = file.get("path", "") if not root_config.get("root"): root_config = {"root": root_config} - path = path.format(**root_config) - return path + + try: + return path.format(**root_config) + except KeyError: + msg = "Error in resolving remote root, unknown key" + self.log.error(msg) def _get_retries_arr(self, project_name): """ diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index bcaf028c7b..cf85d106f7 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -171,8 +171,7 @@ class SyncProjectListWidget(ProjectListWidget): self.current_project = self.project_list.model().item(0). \ data(QtCore.Qt.DisplayRole) - self.local_site = self.sync_server.get_synced_preset(project_name)\ - ['config']["active_site"] + self.local_site = self.sync_server.get_local_site(project_name) def _get_icon(self, status): if not self.icons.get(status): @@ -532,7 +531,6 @@ class SyncRepresentationWidget(QtWidgets.QWidget): fpath = self.item.path fpath = os.path.normpath(os.path.dirname(fpath)) - if os.path.isdir(fpath): if 'win' in sys.platform: # windows subprocess.Popen('explorer "%s"' % fpath) @@ -621,8 +619,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote - self.local_site, self.remote_site = \ - self.sync_server.get_sites_for_project(self._project) + self.local_site = self.sync_server.get_local_site(self._project) + self.remote_site = self.sync_server.get_remote_site(self._project) self.projection = self.get_default_projection() @@ -789,7 +787,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): repre.get("files_size", 0), 1, STATUS[repre.get("status", -1)], - data.get("path") + self.sync_server.get_local_file_path(self._project, + files[0].get('path')) ) self._data.append(item) @@ -869,8 +868,8 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): project (str): name of project """ self._project = project - self.local_site, self.remote_site = \ - self.sync_server.get_sites_for_project(self._project) + self.local_site = self.sync_server.get_local_site(self._project) + self.remote_site = self.sync_server.get_remote_site(self._project) self.refresh() def get_index(self, id): @@ -1458,8 +1457,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): self.sync_server = sync_server # TODO think about admin mode # this is for regular user, always only single local and single remote - self.local_site, self.remote_site = \ - self.sync_server.get_sites_for_project(self._project) + self.local_site = self.sync_server.get_local_site(self._project) + self.remote_site = self.sync_server.get_remote_site(self._project) self.sort = self.DEFAULT_SORT @@ -1594,7 +1593,8 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): STATUS[repre.get("status", -1)], repre.get("tries"), '\n'.join(errors), - data.get("path") + self.sync_server.get_local_file_path(self._project, + file.get('path')) ) self._data.append(item) diff --git a/pype/settings/defaults/project_settings/global.json b/pype/settings/defaults/project_settings/global.json index 598ab09372..c74cc9a2c5 100644 --- a/pype/settings/defaults/project_settings/global.json +++ b/pype/settings/defaults/project_settings/global.json @@ -180,7 +180,7 @@ } }, "sync_server": { - "enabled": false, + "enabled": true, "config": { "local_id": "local_0", "retry_cnt": "3", @@ -192,12 +192,23 @@ "gdrive": { "provider": "gdrive", "credentials_url": "", - "root": "/sync_testing/test" + "root": { + "work": "" + } }, "studio": { "provider": "local_drive", "credentials_url": "", - "root": "" + "root": { + "work": "" + } + }, + "local_0": { + "provider": "local_drive", + "credentials_url": "", + "root": { + "work": "" + } } } } diff --git a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json index 7a39f9cd4f..ef5167f2ea 100644 --- a/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json +++ b/pype/tools/settings/settings/gui_schemas/projects_schema/schema_project_syncserver.json @@ -35,7 +35,7 @@ }, { "type": "text", - "key": "publish_site", + "key": "active_site", "label": "Active Site" }, { @@ -66,10 +66,14 @@ "label": "Credentials url" }, { - "type": "text", + "type": "dict-modifiable", "key": "root", - "label": "Root" - }] + "label": "Roots", + "collapsable": false, + "collapsable_key": false, + "object_type": "text" + } + ] } } ] diff --git a/pype/tools/settings/settings/widgets/item_types.py b/pype/tools/settings/settings/widgets/item_types.py index 2e40a627d9..2942ba6683 100644 --- a/pype/tools/settings/settings/widgets/item_types.py +++ b/pype/tools/settings/settings/widgets/item_types.py @@ -1213,6 +1213,8 @@ class EnumeratorWidget(QtWidgets.QWidget, InputObject): def set_value(self, value): # Ignore value change because if `self.isChecked()` has same # value as `value` the `_on_value_change` is not triggered + if value is NOT_SET: + value = [] self.input_field.set_value(value) def update_style(self): From 8fe2544d1fa5bbf63a41cf65eaa6f2be7b26ecbc Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Feb 2021 17:06:28 +0100 Subject: [PATCH 27/28] SyncServer GUI - Hound --- pype/modules/sync_server/providers/gdrive.py | 2 +- .../sync_server/providers/local_drive.py | 2 +- pype/modules/sync_server/sync_server.py | 15 +++++++-------- pype/modules/sync_server/tray/app.py | 19 ++++++++----------- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pype/modules/sync_server/providers/gdrive.py b/pype/modules/sync_server/providers/gdrive.py index fa6b3f82cd..b141131203 100644 --- a/pype/modules/sync_server/providers/gdrive.py +++ b/pype/modules/sync_server/providers/gdrive.py @@ -439,7 +439,7 @@ class GDriveHandler(AbstractProvider): while response is None: if server.is_representation_paused(representation['_id'], check_parents=True, - project_name=collection): + project_name=collection): raise ValueError("Paused during process, please redo.") if status: status_val = float(status.progress()) diff --git a/pype/modules/sync_server/providers/local_drive.py b/pype/modules/sync_server/providers/local_drive.py index a21dfa2c71..4d16b8b930 100644 --- a/pype/modules/sync_server/providers/local_drive.py +++ b/pype/modules/sync_server/providers/local_drive.py @@ -56,4 +56,4 @@ class LocalDriveHandler(AbstractProvider): for name in dir_names: lst.append(os.path.join(dir_path, name)) - return lst \ No newline at end of file + return lst diff --git a/pype/modules/sync_server/sync_server.py b/pype/modules/sync_server/sync_server.py index 7e9f6f8dfd..22dede66d8 100644 --- a/pype/modules/sync_server/sync_server.py +++ b/pype/modules/sync_server/sync_server.py @@ -255,8 +255,9 @@ class SyncServer(PypeModule, ITrayModule): """ condition = representation_id in self._paused_representations if check_parents and project_name: - condition = condition or self.is_project_paused(project_name) \ - or self.is_paused() + condition = condition or \ + self.is_project_paused(project_name) or \ + self.is_paused() return condition def pause_project(self, project_name): @@ -641,7 +642,7 @@ class SyncServer(PypeModule, ITrayModule): if tries < int(config_preset["retry_cnt"]): return SyncStatus.DO_UPLOAD else: - _, local_rec = self._get_site_rec(sites,local_site) or {} + _, local_rec = self._get_site_rec(sites, local_site) or {} if not local_rec or not local_rec.get("created_dt"): tries = self._get_tries_count_from_rec(local_rec) # file will be skipped if unsuccessfully tried over @@ -997,7 +998,6 @@ class SyncServer(PypeModule, ITrayModule): self._update_site(collection, query, update, arr_filter) - def _pause_unpause_site(self, collection, query, representation, site_name, pause): """ @@ -1230,9 +1230,8 @@ class SyncServer(PypeModule, ITrayModule): (string) - absolute path on local system """ local_active_site = self.get_local_site(collection) - root_config = \ - self.get_synced_preset(collection) \ - ["sites"][local_active_site]["root"] + sites = self.get_synced_preset(collection)["sites"] + root_config = sites[local_active_site]["root"] if not root_config.get("root"): root_config = {"root": root_config} @@ -1243,7 +1242,7 @@ class SyncServer(PypeModule, ITrayModule): try: anatomy = self.get_anatomy(collection) path = anatomy.fill_root(path) - except: + except KeyError: msg = "Error in resolving local root from anatomy" self.log.error(msg) raise ValueError(msg) diff --git a/pype/modules/sync_server/tray/app.py b/pype/modules/sync_server/tray/app.py index cf85d106f7..e511db050e 100644 --- a/pype/modules/sync_server/tray/app.py +++ b/pype/modules/sync_server/tray/app.py @@ -628,8 +628,7 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): self.query = self.get_default_query() self.default_query = list(self.get_default_query()) - log.debug("!!! init query: {}".format(json.dumps(self.query, - indent=4))) + representations = self.dbcon.aggregate(self.query) self.refresh(representations) @@ -745,7 +744,6 @@ class SyncRepresentationModel(QtCore.QAbstractTableModel): for repre in result.get("paginatedResults"): context = repre.get("context").pop() - data = repre.get("data").pop() files = repre.get("files", []) if isinstance(files, dict): # aggregate returns dictionary files = [files] @@ -1550,11 +1548,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): remote_provider = _translate_provider_for_icon(self.sync_server, self._project, remote_site) - + for repre in result.get("paginatedResults"): # log.info("!!! repre:: {}".format(repre)) files = repre.get("files", []) - data = repre.get("data") if isinstance(files, dict): # aggregate returns dictionary files = [files] @@ -1840,11 +1837,10 @@ class SyncRepresentationDetailModel(QtCore.QAbstractTableModel): 'then': 3 # Paused }, { - 'case': {'$and': [ - {'$or': ['$failed_remote', - '$failed_local']}, - {'$eq': ['$tries', 3]} - ]}, + 'case': { + '$and': [{'$or': ['$failed_remote', + '$failed_local']}, + {'$eq': ['$tries', 3]}]}, 'then': 1 # Failed (3 tries) }, { @@ -2012,11 +2008,12 @@ class SizeDelegate(QtWidgets.QStyledItemDelegate): def _convert_progress(value): try: progress = float(value) - except (ValueError, TypeError) as _: + except (ValueError, TypeError): progress = 0.0 return progress + def _translate_provider_for_icon(sync_server, project, site): """ Get provider for 'site' From 1e0e6cee11bd36282a9e6708a0ae7f1300064224 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 17 Feb 2021 15:43:40 +0100 Subject: [PATCH 28/28] SyncServer GUI - fix local site name --- pype/plugins/publish/integrate_new.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pype/plugins/publish/integrate_new.py b/pype/plugins/publish/integrate_new.py index 618d7e97a3..cf267a84cf 100644 --- a/pype/plugins/publish/integrate_new.py +++ b/pype/plugins/publish/integrate_new.py @@ -967,7 +967,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sync_server_presets["enabled"]: local_site = sync_server_presets["config"].\ - get("publish_site", "studio").strip() + get("active_site", "studio").strip() remote_site = sync_server_presets["config"].get("remote_site") rec = {