From 8a37b9065e340be28a1a7c85a4be68ac8308e9a1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Nov 2021 16:22:20 +0100 Subject: [PATCH 01/12] OP-1937 - added alternative_sites to System Setting for a site --- openpype/settings/entities/dict_conditional.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openpype/settings/entities/dict_conditional.py b/openpype/settings/entities/dict_conditional.py index 6f27760570..3621c1319a 100644 --- a/openpype/settings/entities/dict_conditional.py +++ b/openpype/settings/entities/dict_conditional.py @@ -762,6 +762,17 @@ class SyncServerProviders(DictConditionalEntity): enum_children = [] for provider_code, configurables in system_settings_schema.items(): + # any site could be exposed or vendorized by different site + # eg studio site content could be mapped on sftp site, single file + # accessible via 2 different protocols (sites) + configurables.append( + { + "type": "list", + "key": "alternative_sites", + "label": "Alternative sites", + "object_type": "text" + } + ) label = provider_code_to_label.get(provider_code) or provider_code enum_children.append({ From 26b9bbaedae7e21b6f4346b403160a495eb53d83 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:03:57 +0100 Subject: [PATCH 02/12] OP-1937 - added alternative_sites integrate_new --- openpype/plugins/publish/integrate_new.py | 68 +++++++++++++++-------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index fe780480c2..2f90cd7d66 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1029,31 +1029,25 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] - sync_server_presets = None + alternate_sites = set() + system_sync_server_presets = instance.context.data["system_settings"]\ + ["modules"]\ + ["sync_server"] + log.debug("system_sett:: {}".format(system_sync_server_presets)) + if system_sync_server_presets["enabled"]: + sync_project_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) - if (instance.context.data["system_settings"] - ["modules"] - ["sync_server"] - ["enabled"]): - sync_server_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) + if sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) - local_site_id = openpype.api.get_local_site_id() - if sync_server_presets["enabled"]: - local_site = sync_server_presets["config"].\ - get("active_site", "studio").strip() - always_accesible = sync_server_presets["config"].\ - get("always_accessible_on", []) - if local_site == 'local': - local_site = local_site_id - - remote_site = sync_server_presets["config"].get("remote_site") - if remote_site == local_site: - remote_site = None - - if remote_site == 'local': - remote_site = local_site_id + sites = system_sync_server_presets.get("sites", {}) + log.debug("sites:: {}".format(sites)) + for site_name, site_info in sites.items(): + for added_site in [local_site, remote_site]: + if added_site in site_info.get("alternative_sites",[]): + alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1068,21 +1062,49 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sites: rec["sites"] = sites else: + already_attached_sites = set() meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] + already_attached_sites.add(meta["name"]) if remote_site: meta = {"name": remote_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) # add skeleton for site where it should be always synced to for always_on_site in always_accesible: if always_on_site not in [local_site, remote_site]: meta = {"name": always_on_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) + + log.debug("alternate_sites:: {}".format(alternate_sites)) + for alt_site in alternate_sites: + if alt_site not in already_attached_sites: + meta = {"name": local_site, "created_dt": datetime.now()} + rec["sites"].append(meta) return rec + def _get_sites(self, sync_project_presets): + local_site_id = openpype.api.get_local_site_id() + local_site = sync_project_presets["config"]. \ + get("active_site", "studio").strip() + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + if local_site == 'local': + local_site = local_site_id + + remote_site = sync_project_presets["config"].get("remote_site") + if remote_site == local_site: + remote_site = None + + if remote_site == 'local': + remote_site = local_site_id + + return local_site, remote_site + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 691faaf70d7abbd191d5ce7c22d188256c577066 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:20 +0100 Subject: [PATCH 03/12] OP-1937 - added alternative_sites to upload/download --- .../sync_server/sync_server.py | 8 +++ .../sync_server/sync_server_module.py | 53 ++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 66d4e46db7..6eaede048c 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name, remote_site_name, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id @@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name, local_site, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index a2cfd6f6b9..af672e7a6f 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -109,6 +109,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): # some parts of code need to run sequentially, not in async self.lock = None + self._sync_system_settings = None # settings for all enabled projects for sync self._sync_project_settings = None self.sync_server_thread = None # asyncio requires new thread @@ -769,6 +770,38 @@ class SyncServerModule(OpenPypeModule, ITrayModule): enabled_projects.append(project_name) return enabled_projects + + def handle_alternate_site(self, collection, representation, processed_site, + file_id): + """ + For special use cases where one site vendors another. + + Current use case is sftp site vendoring (exposing) same data as + regular site (studio). Each site is accessible for different + audience. 'studio' for artists in a studio, 'sftp' for externals. + + Change of file status on one site actually means same change on + 'alternate' site. (eg. artists publish to 'studio', 'sftp' is using + same location >> file is accesible on 'sftp' site right away. + + Args: + collection (str): name of project + representation (dict) + processed_site (str): real site_name of published/uploaded file + file_id (ObjectId): DB id of file handled + """ + sites = self.sync_system_settings.get("sites", {}) + for site_name, site_info in sites.items(): + if processed_site in site_info.get("alternative_sites", []): + query = { + "_id": representation["_id"] + } + elem = {"name": "sftp", "created_dt": datetime.now()} + self.log.debug("Adding alternate {} to {}".format( + site_name, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): @@ -919,6 +952,14 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return self._connection + @property + def sync_system_settings(self): + if self._sync_system_settings is None: + self._sync_system_settings = get_system_settings()["modules"].\ + get("sync_server") + + return self._sync_system_settings + @property def sync_project_settings(self): if self._sync_project_settings is None: @@ -1004,9 +1045,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): (dict): {'studio': {'provider':'local_drive'...}, 'MY_LOCAL': {'provider':....}} """ - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") - + sync_sett = self.sync_system_settings project_enabled = True if project_name: project_enabled = project_name in self.get_enabled_projects() @@ -1064,8 +1103,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if provider: return provider - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") + sync_sett = self.sync_system_settings for site, detail in sync_sett.get("sites", {}).items(): sites[site] = detail.get("provider") @@ -1434,9 +1472,12 @@ class SyncServerModule(OpenPypeModule, ITrayModule): update = { "$set": {"files.$[f].sites.$[s]": elem} } + if not isinstance(file_id, ObjectId): + file_id = ObjectId(file_id) + arr_filter = [ {'s.name': site_name}, - {'f._id': ObjectId(file_id)} + {'f._id': file_id} ] self._update_site(collection, query, update, arr_filter) From f19b39185d47b4666a8065bb155823dbcaefd4dd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:54 +0100 Subject: [PATCH 04/12] OP-1937 - fixes to stabilize sftp provider if wrong settings --- .../sync_server/providers/sftp.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 07450265e2..8549c1c981 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,8 +1,7 @@ import os import os.path import time -import sys -import six +import paramiko import threading import platform @@ -37,7 +36,6 @@ class SFTPHandler(AbstractProvider): def __init__(self, project_name, site_name, tree=None, presets=None): self.presets = None - self.active = False self.project_name = project_name self.site_name = site_name self.root = None @@ -64,7 +62,6 @@ class SFTPHandler(AbstractProvider): self.sftp_key_pass = provider_presets["sftp_key_pass"] self._tree = None - self.active = True @property def conn(self): @@ -80,7 +77,9 @@ class SFTPHandler(AbstractProvider): Returns: (boolean) """ - return self.conn is not None + return self.presets.get(self.CODE) and \ + self.presets[self.CODE].get("sftp_host") and \ + self.conn is not None @classmethod def get_system_settings_schema(cls): @@ -108,7 +107,7 @@ class SFTPHandler(AbstractProvider): editable = [ # credentials could be overriden on Project or User level { - 'key': "sftp_server", + 'key': "sftp_host", 'label': "SFTP host name", 'type': 'text' }, @@ -421,7 +420,10 @@ class SFTPHandler(AbstractProvider): if self.sftp_key_pass: conn_params['private_key_pass'] = self.sftp_key_pass - return pysftp.Connection(**conn_params) + try: + return pysftp.Connection(**conn_params) + except paramiko.ssh_exception.SSHException: + log.warning("Couldn't connect", exc_info=True) def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From ab36fdfe711400059e103e6475c43f60a6de754c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:00:03 +0100 Subject: [PATCH 05/12] OP-1937 - added documentation --- .../docs/assets/site_sync_system_sites.png | Bin 0 -> 13118 bytes website/docs/module_site_sync.md | 47 ++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 website/docs/assets/site_sync_system_sites.png diff --git a/website/docs/assets/site_sync_system_sites.png b/website/docs/assets/site_sync_system_sites.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f895c74347ac8f311e81ad3f1dd6a388723695 GIT binary patch literal 13118 zcma*OcU)6V_b&>fB2p9)QK}#yf`HO{5vBJk9Yl~08fxgGpi~i2>Ai*o1VSKmP>|jU z20{d+210Lvz}@kA?>+Z-@AG-j`9p`xcZa}e7Uvo@&$qU$moZ1 zAH#c&Q%KO#8GqhnP)N@ntgm&?xNS5IC=~DRn4dLGpk>!vBPcus1qEF_Jxv`P z@_9|P4V3U7ZhirYU=4Bv@9&rrYIh3PBr*f z&{FE?=t!^{hvM~4Ds#cl~NyTJB;1#?u(1ypd z&XW4!PozkV9X%bXdHRNe)RNegBLa$?SLEgVzMST}d^JVxki%sJ&kb z_evkRSiAbUmwFdvmaylyMPm(2yq)r0ynOk&QEXb%!X{qDRR*jHy{lKcK1K!4l;g}) zSP?8P5G@{mMTiR}3`bjR4F}=1NkW!#!Mumi=(5qf%;V^h#q6XNqGH2y^;*C_ z`RR7GN&~!jnELO+3{`_?bD-lw+vw9C&7F3oG@*XBoEARe;qM>qL)9y_iqKUe<8v3H zC3QJA_3%H@()1PEFe~Kfx_T5mjbY#1#U({^wd=y;;4bMI6Q7QO!ZJBq6fH@Ox>mC} zg-{(q7Y~~bEcHe6nR_#aQa(BOlAHNw@_QEX>}ej6%D1_{hF2D^f0C8GIO^Q&lAbj= zJB}O2PSmQ1n%QRk3D>7TX=Kw=r*1|XYk0g!?>x~N5!gQt>`j2cn>u{#&S?kCw_x_a z9$8>(ZFaiqJVcLv*;N)25t{8OHQTpYlP4;n&Dm{T$m4bijlqm@to-Q z!mK6zQIQhDtdn-X1vkRL#yerPPe5fe6S+mEOmtNz(wDOG;v)9FjAbEkaEEo;OmnWB zQ<{^h`>5ZPoxguSDlJ#+v@+wJ%>~)-MMQ5l|ACPq{j7l?uEre0)FSEG_uqG2jni!2 zg6g9l7!?PRKaXf;US}SVK?P6ffGt>Re!r@4AIR{VD%#)7dZXLy_LP{S^(88EDGb{x zO^;;(gk*vrwkumeEJV=-%{C8~-OfEq)2)@jt`jbx>b)(l}_KCKMFoK*Xr2 zy|3*XC2Es7{{D3)qYq9KcFWm5I&_i0W$ye^G11rhp}?$Q<2*~}jP!=C|AN1ZN?g{v zOs$suVgV;t^jAQ4t=3>9zt0%D3{WS<&Vs*dMz8pJ?eWW!R{T|*?CaCLO%Arh))BdS z0;V-n2@?xfaSdq%Cg*8^P~xB~oJk}Dp6J?EXLbT=8HK~B{u%(JtqjY5vgput))qAvWEzC5R=Q#o{F*WXt?YYeu)t&^OdW6JXtIVT<@%P{R?q@Jw&U|Qs1wTt~+VKKk zukFuR1k<8%o#Fv)bHDELw}bfx76=Tjx~>vURJI9M6{77c6i3$S;zEU_d-C8qkrD8|B|*&ICx2i@97*lwr`^WN{~G&$MpmG$MDuVDu1;btIOyY(5le%N|Th_nlKcwBY>3nHQhG>pb~H#Lw; zXUrel+=H)ZW~(3QxXXUkV0FC#YIg)qUAUm@r2^|DDl2{pY`(ng95J9 zXy&&%y1KV)CwlzGuAfLCg3dw_eysN$qN3 zLBF7o1CzL+_@h8VXUQ{@=S%e?#=oF8k@!6FPKIx1EjF+fYU z8+uRb6v!*cltixnzvJ4C{}S@hL+#IIfXo80udKg8P@$a^*2rSLN&mNUv67m~XS(VG z>UyYf^2kpqsf9jX@{BdLwPmIC8h7>B&0-NGX17BPEQ!1~|NQw=>XqGmkAa2;1YJz) zho^6lC;)GR=q_IAsf}MV_AA{`Rgd;0pc-`m-J#fhQZU{h9#z##AD$F!8CVxRav?}~-JGe|qg zHhvD19S`yX-hHqD@3jVte&S0hBmq;oj zD=$eDVM~+jgTjsEjIIban|EM}m~AFL9-Fx(PQ~ecD^fg>9N9iLHo?Vw{L`0eV&HlaDZKoX5x%IX?UtZv-#oO0)Zt);oy354Dymiw{A=M!7j*z8L zm`UEPMB(wcYX~Y}s+HZ=0qdkqD=`u^1?{+4!o$7@rm*vX(a<+ z4uj}i4_sP!SG7cAtd*6ZWj2JS&M(pUTw5~*gRp>Zx6L5VS;z&OdF;Me`H-o2$?9nf zN|;4Ea%NAd&Rzr6e@2<&c^cHRM!83rn@(k3Fk4D3F$}@3?{{x@J^bAeo3j; zIY&g`)+IpBI&b8Ia_6SciicHV>l`&FU5ej&r{h%pf%2mlJPjSx6MUdb9w~(Xz_XFYy z4aE@q`d|B>7ktye2-LeUVh|E!kiz8qh@VLzkIDfp=ZEvDVPlv}b$Kc@B)|5C z_Qap+MJjCr^IyCs6|d$axe|5`puz5$7!v1QooAB_Yi_VSBloj50in_td-cRB-)Dx>sfJ%hXy`Ii8NEwK=7(iUcnP6u%yYL?Q>Zb|9nHGd-Ws_Fs>iRc=IuTLR!GLAjB=RF#M|v-1fiEUbSk^_{8b{gH^OB%%E5gM@B0|utwq%w-NW2sgB~j6vn&*KY zciYhw$_@oc#q^f>-pLXtkAAm@#rB~GzhC;pA3>y;^x^zgG{tC1_L*cFlU32GCPJV# zx2(?LGrv$7H<^#hgWI2-JLk2;^S`G6+%qV*6nnWGkG4yol z3{g9AmUMkxMwSqAXLYh5rMkIzZ}EfQ$DJuK*304T!O<^tuHwXA1h)Iqsj2)DNgK-{ z*F=_n&(lI`+*b~W3$~J4$)}GQI~0bavdbrmREoAX_AfA>)I0V}&Gv)_K3yAj@9)D(8-xn-1j1)|? zU1(Pmn}r4q!lvAQ4+!lt#}oxm7@?mf;-4Zy6RQ;}blu7tvCp@!`-l5OlY8Ot9`S8z z!2g$HdtNk;jv^oe2$4=AEvyY&aDl^-t zH*>@3%wyMfZX8(deGOv*YDhDgC2#KqU&kJz_ZMvJRs6g)+V>~4>y>kLD3jFIXyj$Z z6XKWlM~@|Iyng$Xwi($LGDbAOg7(YWrcMKY!0|zm|79#nt^X ztJFxRI7Yh1^z=xS%6QYx`l>xG7JrDRL!hA8J3m9R>a(hF#xA`lZnU8&=a!TO4#f61 zU(W`}IV?OG%X_jjh#Zjd_%-=MX70lfwpJz6%75tS)E{t6;8>xk$ZN#X^bhXQr;Rn3 zlw;7IVmF$*HHS(`mu6+*36K6A$Ypv8#k$?;l3 zXnOS_%G_z7VOAz(qNP+)C%sJi1|2Xmcq5ltRRo8;4Pcg>&`o#qK8Q1S@JA7ONv3Z>tUY!jMoIHM1ae@`Y zX+bAdKZGYreZZ66U!{DA2k%Y;akeENdHct*bv{lsd&8Q@Gv9CjVmp<1VR=@hSW^=4 z*xq^Bhza-3EH8BGD|qa$Ff2UJ2ihtdISuXJ4OjO1m{%_SW?+;11ZmWoc4F;&^Cz%$ zll=`T%52~@S!Vri;HSr!(1VQ4-1ph9RO ziJR@r-#i~FgJNJD1Dyh3Quxe0WrmNExQ@Z#HV#uUalPnvFx?Ec&b;1sH=3*PalWFb zxW&%%lN3Y3ddRQRSiC}zWmWWnA8y*Rdk3Z6J^%F2&w=)c?h^({1gSvwM` zf8?9gdWS&3_eFvv)=oG`_n9-w1C}-POSTQm03RNQ2t{ysgeQbzv+{f%BJIxE=#(z zJ>C+g_`iaXn79P4^)!-Sh`n^0Q+hS|SJyfz6&_-{Ct1!nUHE0uWhjSKe$-4wUZkrG zoA*8WuJQj1dOm;g#1B%y98!L&^y=TDbIKT1w?kKgpW8??A<}bw+R>>9!l!5kRZA4m z@sPowpN;lc#?izN_83MSNPdoVagkXQ3~Ca^Sy-3^{a*W0@+8@8-;sC(m-g(P%*QHJ z>oy3b=lwEL)qvyR+mego%{+wm%oP6pE8DEYQzpVLM!(%C6Rt>A=_pR6+$=wIBYUck_ z#r_u>|Mp6g#p3Do;zVTBA}z=bBN}R7XqoH>4d{aUVmI4a9rt?mP2JK)(yZ<8rj3<# z7CEsgf>rP8iX#DZA3*c(G13wC`$yi{|B`t)GNrC(1X3&&?8P#TCpp2_WqH`VfO=JM zv{7leILx*%Q*$&zFQK%)ew>YDJ+}+n+g_;JkXeyv4=8?NBcl|_ za=hBnl#OhE`pP5}CuW^KF9DS&tc&7&O4fP_pz3_TCeza*)h{8Zewe3g9~_A6U>3vmU&9x*pLMq@WED~O(7I~$sC9PBjOw>Nu2_xI^b>kPxpKjS!9b;br| z3$7?`5A0n}-0$6qdOp>nrcg$E;eCcXTWEjqK3JGLARU$PW!>k}m23Ox6CxmoQXTa* zx*Ib@9mI`$ruD<4TJ67l=8E#=#CW)tswc@O0;!a0uQ zFHl}NZn`e55;fUEFLTrwnT|N3CCqVEv}HpE<0Qm6hSkJjFIuIl)Hu)u-cO@FKL0d@ z`Qsh)iR_VRXaxIvW@4HRY0uPv+7?-BuAF1LKrug-%IlaR=XT!MCsJfu)BP5)%BSkv ze6CtU=scKwZSpLf)mpXEKcc|&%!dNjm+bdJ`Xf`9Dh{(%JEB= zxX?h_SmF5k9EOh+5joX%b{SH`B{dXVzH9~Y%U;=<*OL*#kTBa!DL*u0hG2!eW=Zt# z-=;*d2ls!z8?H;_5gYp{I19R*5uo6OQVUvh+JYy^Un6OY%WAWj@dtnx9nV!ILp3Yb z4?SB_l1ppm$cHIjn@!wBw2bH#uly-ZSnZjBk9wh)Uh7@HOtV16k;U-pCr4L4r^=%p zDnE~{AhH-OEfxFrK9x&-ci&y@T3RPWo(pW#4uHMiIVx(?r*BNKN(pNogOR)z`7Mfz zR|0=RDQTCn!*>+sB^&d8LIdNM`YET;^|F}B0w4xBmDq?ozVOn6KJunBn@HW#mdh0qOWDJx2M7oZW@?$R=sL&v1y`no zaiOgFA$tQ@-6i$XFF6;!9xyyFD2CW#wT~s)bzJw_pH%Cz%H}5*F>&#v`g0nWJ>?Z^ z{2oj)8E#|EsK{8;a(_;sLq?_aj|aEjr0OR1!59Cniu^a1`5(COpI7At&3{9H(rV7% z{rRtFGJC+(UD-gS$u z1WT#xAxDi zwE6)BY;!`xv-mngo(mfgRnIfobhyhIW2o?>^DWW#E&eBeyyC~n%$S#59-M2URkl9{ zNQ7r`FSNHg=qW5r4AgGoy^wIJ3+>8E$5BCcVQL*Q=h|A?M1RsgkvYO)SFKYhQP|+4 zlCZTe2`f5(zZf{@6N~9}0Wu%yn>W*EMuKKDuc1Dzbl&yc`mAW$eF<)(92j}Y=(zF$*$yedUVz4&cDW=k5rb_yS#VxYXiYX zeR9`La;Md88j)>3L@zq_Gb*n2+2VsbnSQ^V==NPMU>=#gfuN*QIoiiS}v$ z86`bXQp?h9v@BhpgO069d@A#h=cDakU0nedNwb+dSv_3Y#$%_Y>ZGAp+O>vAte-L+ z{qx89e@o&1?cDgkn4u|vug^&!*u})WzFLwr;{ENvY=#>QA0y5;?2c%O1q_inK6k1E z9Ax+yup6%x=w&M~i}Tyk9xD#7jJ_BfnV#DRs1;m{VkD;$DdV#}z7kx{(4}{c23UzE zunno~qOOUH1?SCo>DK?Hq5@E9?R)q@znWUFv!5P&LKGKih6*&2 zO^oHxDFDxz7;iFeB`-Qk$(ciGaS4f4I4X2q6CmyNqmqk^joHLl`vDo*Z?6BOdh~SR zGesrFM$MLG?j%(Es&G5`hx2u0Qvs#FLv?X1EFi$@E`b#oY6IRzz%Q4+8_?GRF~+Nz|6HP@pj&u zxSPpd6BK)3$T}<+-4Ny6{HhDJ+re?)-_l;wbMzz6s3Syt6;gAUwPZ5YJpJG}>(gS5 z@D+4HO6u9gHI*0Mb@MhID|BqsM?w=CQnkCA+axnF=h?Tl2hq$_+wCylk49w>48Z*);IZ#q82~inlg6pB?oXo zz$ylk>BA&+BWv;dLG*D^9%%MuT0VZ$=Js02wTaUc08Fjl8{v|)&3$~KP;Gp3g0sOf z6|{3uqTua$9(2OK9^A&pVN*dLN^5N(GSV=kgFZn{?U#q$|K@+>CfQsX^?I`vaK|6x z0aF;~OF>d~7amRRb|#C%8Z(omb2*hK?AhFYPS{O}zzm2J2K*(PrUEfyjrNGu{>mIO zkY@)y8z`MV=9SokHRd8{`It8XV(GeHro50QAuyH?P&VY4B~vdOdh=&Bf%nRWeBS(# zH1nE7At!T2m~q!W!SYz*SS7ZK`c+K}(prAe3IYC%QS;$CVP2bCWcBHl5wtYs!EC?F zGKY5x?Ra`Ol)vitTktcN`jOIAu|2WUgKbQb26EI4wKo(wcPFhOQ5R9Nb~PKEl`65Y z;WzKq2G|vzpHrID&t21aV&8N%w4&pyB?wcS8u=kdtt@Jg^J%k2A?S;%YUQ0=Oh#na zT;}N2%Z%03kQyoc4zLrruF{3G&wfyN`b~%8MZ>IObs>~qp0+3{8>(z$cWJ=n_IwC5E0J-|nGY zW)rLO-ik)eV!h0fS_F0ipXh3AYg?Sj5&Sy=pZ0treylYE@L@wNR@Y~9<_IjiU#d?v zaW=&Gw6wIE;@SxbbTufdr0#i5jOrIY0RgSZs_MN+$KJ8YvE_j^D=yqXpEl~lw3e7% ztCHF4ti|Qwd{k}6znUduBV5R4@0OQ zvoP-p3xq9eI_s+f)_S|boBmYMn5X$(L2&kvj`pK!#D z`My+9!ftY+waW$_rGrp|_WWJ#u@?C=Vb`5gU@71V8#cNdp~vJ+2iT1=9F0QlgK zg9GUHLB0FhWF3B0O*9B^<#F60OTZA$Uh!~tm!*4ukP#{(ArV6CP1`O{oL*O^QhLMU z{T&vz7t;1pWXBM;ie__KPMEFKyub*=h)~8{4=_3$k4u?tm6QIdeH9P$t!OHvO6IK> zv=zZ6s4cX6@t^$&@?9#wh1~guN!vO!?B{za3v8@#&t=ji!!bGOl7=)EmXt64@29qT zs__FUC6$%BmJ$>HT%`f7M@T)`V1AHzeO-S`TYh&_JN{Mr@K~sE{@id9Vqv~amTq{} zv*J}Qi^3m4EvJ-#sYGX2<=i5nhmL3EPkP%CO$&fKvU-*NIUwxC`N6^$0B^Fb6@Yi+ zo4!W3oCa4nSF!yv$nhju zOz$+hyWjf?do4)>Fgs17jr~WF_kcZ2CNm_^5fLc+Es)CGpSHVyyfpbPhD5ZhR@cqc zm-EAz*~$`d1pp2?Rk5VzbrWeKO=;K)D38n^}sF&Zve_E)dc$Ue_+>v&VRC(+C$qlDFZ)(w zR;8PE0W~3{VW5!~)3`g-IQ3nweNwKpPwqo&pR7m+H4cZX*@?U|J#3xaha@LkPOxF2 zi%T!kL#3G6K)X0><}~N3|8SgXoE^o&!Rv})3z`F;SV_n@W1%h`py74S$>0z7QDIJH^GC2ON$|)TS9bqM>v*af=oic3%`$g2e-2vP9 zfKsJ*ZeFf^M&~E|@Wc-_G2S}1!1ALtoY!SqE1A=nqu{sWxUYj&oI2j9Q3OHS1O0p7 zM0>c#MNOu;h);Doq(O6iuWm2|ayZ>wu~d#c*JA&=u(%kP)nZvdq@^W%Nm2AE{FXZV z7W-9FucbWHyu#R|eq{Nh!9l!asBV0GJa`>$+W=&s$YNA9#e8+jnM!r@tLTX)LRnfm z^ZL~L{^Zwv4Hx_FJn2u4tP(jc@j43y6G|NoqObk<_!BwW*r{SO!^14)og<*E2il!9 z5YJLzpr?tOoV~p!RDVwXg6E{}U3tnPWmX!7Sy~^^Oys=JvqHC41UA9BFZx>-WBp7l zpZ^90$|y4**Xsu%Xq?fT&7$ac($)yIXXx$y>$Tr^J6o9_NXF)-rCcQs1ja zx!8IiGqZ`(9)IP$)9I&wxa5D8t~unfLu5AFetm;%T~VcKGozqX2JV3Em+~cz+oID; zRo@)ukr0mlwR#YI{mRC&KklqIL57GU_e zu$jUq2lt1{UWAp?R#f0j=jBs<@4y}P_(yV1ULr#m*3)W7Xi~dPPYbLN2#P1t%q0f= z@TJs3!ww7#CZZC1B7>&oi78dxvs5e1X}5&(jci6?$`JNN)To--E9lbJgJ$c6B3Oct zvA-EkhVuUjaUC?mVomQBos{dIPh|Qy(HZ@|=M3CtYk$H(7rrBGc7dTVGdA03&mD1p zhF9l4{$#7LA#wT5zO&nJxrWE}D%jMRd&twGJFqO*b$9QD%t=IGV$_!uzJ}Xs#}EwL zv5@sBXUL|<*;h#$>9W~TnZW06(Z1Q%W_hW1DO0K1F>pNGOx%8i8?gXH*Gz%oxUMf3 zv8f}1=Pv+v5rHJ``8^hN3_Zc=67Wj6@1d-jS6cJFxN9UJQA+j9Q9ab`pXVKT6h{e#3UQ!5h z9&XO|8g2dMsU9En5Z!ziGXCIY8jBOCL9;1mHnA%aJIpk*#e9?!$_9Gn!&MGcw=bd& zQPL%x+oJPLMfrOpUn#9I)lI>3ypohY3&(z@W?WzWX}_w)_>}|{v4aDOWCA$rLvBZ; z`d=4|#Ad4VfwYdq-&8y$RHdEe4*Fxq~c}Dvg0zziH-f~p) zE6Xzgh`?-K>IWKbqb^}(6}JkzrX?ZK(%d0vo^kR{M3KH*!Qtb|f6Kg*DDW{twcm6q!MockT zVRO1?yB-G%k=-6229f>FdoiQyZu4ludXc_!QNwZeqq)C3uF=k$PX0a*W12p2OsbQw zcmotyw!72%u)=F>;&peL_69lG?q%SPH&c^`SBnQ$H~7Y=<>q|0bG`WuZ29bwBv5A^ z&!*yYu8OX8$+b&a^&vA2M4OjTORI6>?)NM(^ftCocp8inu^(eku-WQ9%6yu_{`Z{P{G+ALFH zsE3u|WoXjn%d5s5IAEvWdrRVl(zh)x@4Jui#fwx|xm%MIQa5LQaN+Jn6n-ZGG6k^^ zelRxvqyb`E^2pzP0cxL|;HKT7DE7GI5ZS#Brjiq9sVQu>a*W7&&jK@YA*O>xw|2gh zlQEIalC?hZ>Z0Uu*4M4=v=*I7d2A1+m#M2(3~fKZheO$N;NJzW*jxGfbD3ghT@UW< z8gq%hBz*I&4JJ~X^kn0D`bVQ&vm}L!>DVo6EBMP1cA1|AE2#0LBS%fWNAVMKw@izV z?(^QEP^xGfjR`;h+vQ1Vg`O#gJb}33b}HG7*F-LsrYMe_G&I9Lf4*UM?PEFe4uB7k zDV-Yvt}JGC7O9d-01W7i%j{YU&)K0Kcou8=g!ZCp)#^+$F~r(;eB6^7$f1E97QUlt zk^-dl;y`9@Hu4Z~qNOiF3v7wcpf<;Yw*N$|Vvex#k@pLlq|!i%>p}S?b+RbNJ)vyq z2?r@fufdO-jZcp`V|CrLu0Nj@xkb*o?z@RhpRfIGUuPlAl5;V zpOGvab{jxrl%$qJ2>ZC=_Y29Ng1UKXhZTUlcungk5c;$onsZ^g9QZhS<+?lFAE%%pdH&*rXLJM_vL>qHgc~ z2Lq!KAe|ij&!0>DPuPoR&*jNHlPOZ%1r_7}_h%U4S?~V2Qu;p!{*Rwr{Et_^lTH;a Y)~G@38KWNnZe$uybRWYW*}eY10Psf-VgLXD literal 0 HcmV?d00001 diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index 31854e2729..4e3c8e4ed9 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -27,6 +27,38 @@ To use synchronization, *Site Sync* needs to be enabled globally in **OpenPype S ![Configure module](assets/site_sync_system.png) +### Sites + +By default there are two sites created for each OpenPype installation: +- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. +- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. + +Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. + +Many different sites can be created and configured on the system level, and some or all can be assigned to each project. + +Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). + +Sites could be configured differently per project basis. + +Each new site needs to be created first in `System Settings`. Most important feature of site is its Provider, select one from already prepared Providers. + +#### Alternative sites + +This attribute is meant for special use cases only. + +One of the use cases is sftp site vendoring (exposing) same data as regular site (studio). Each site is accessible for different audience. 'studio' for artists in a studio via shared disk, 'sftp' for externals via sftp server with mounted 'studio' drive. + +Change of file status on one site actually means same change on 'alternate' site occured too. (eg. artists publish to 'studio', 'sftp' is using +same location >> file is accessible on 'sftp' site right away, no need to sync it anyhow.) + +##### Example +![Configure module](assets/site_sync_system_sites.png) +Admin created new `sftp` site which is handled by `SFTP` provider. Somewhere in the studio SFTP server is deployed on a machine that has access to `studio` drive. + +Alternative sites work both way: +- everything published to `studio` is accessible on a `sftp` site too +- everything published to `sftp` (most probably via artist's local disk - artists publishes locally, representation is marked to be synced to `sftp`. Immediately after it is synced, it is marked to be available on `studio` too for artists in the studio to use.) ## Project Settings @@ -45,21 +77,6 @@ Artists can also override which site they use as active and remote if need be. ![Local overrides](assets/site_sync_local_setting.png) -## Sites - -By default there are two sites created for each OpenPype installation: -- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. -- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. - -Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. - -Many different sites can be created and configured on the system level, and some or all can be assigned to each project. - -Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). - -Sites could be configured differently per project basis. - - ## Providers Each site implements a so called `provider` which handles most common operations (list files, copy files etc.) and provides interface with a particular type of storage. (disk, gdrive, aws, etc.) From db02c03394f1d5d159152e0873059c6c7146b387 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:01:51 +0100 Subject: [PATCH 06/12] OP-1937 - fix broken import in Python2 Not imported exception in that case shouldnt happen, as sync process is not running in Python2. --- openpype/modules/default_modules/sync_server/providers/sftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 8549c1c981..267e23f8fb 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,7 +1,6 @@ import os import os.path import time -import paramiko import threading import platform @@ -13,6 +12,7 @@ log = Logger().get_logger("SyncServer") pysftp = None try: import pysftp + import paramiko except (ImportError, SyntaxError): pass From 1e1b9b0416b804e815785fdfa83e7acf38e8128b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:04 +0100 Subject: [PATCH 07/12] OP-1937 - added file id to DB --- .../modules/default_modules/sync_server/sync_server.py | 4 ++-- .../default_modules/sync_server/sync_server_module.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 6eaede048c..8518c4a301 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -82,7 +82,7 @@ async def upload(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id @@ -137,7 +137,7 @@ async def download(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index af672e7a6f..9ff706d6dd 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -772,7 +772,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return enabled_projects def handle_alternate_site(self, collection, representation, processed_site, - file_id): + file_id, synced_file_id): """ For special use cases where one site vendors another. @@ -789,6 +789,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation (dict) processed_site (str): real site_name of published/uploaded file file_id (ObjectId): DB id of file handled + synced_file_id (str): id of the created file returned + by provider """ sites = self.sync_system_settings.get("sites", {}) for site_name, site_info in sites.items(): @@ -796,7 +798,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule): query = { "_id": representation["_id"] } - elem = {"name": "sftp", "created_dt": datetime.now()} + elem = {"name": "sftp", + "created_dt": datetime.now(), + "id": synced_file_id} + self.log.debug("Adding alternate {} to {}".format( site_name, representation["_id"])) self._add_site(collection, query, From 8ba0e606711c0bc440dbaf43750bbfeb3b685e9f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:23 +0100 Subject: [PATCH 08/12] OP-1937 - fix - integrate_new --- openpype/plugins/publish/integrate_new.py | 98 ++++++++++++++--------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index ab79d95fb7..c0a90ee78e 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,25 +1030,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] - alternate_sites = set() - system_sync_server_presets = instance.context.data["system_settings"]\ - ["modules"]\ - ["sync_server"] - log.debug("system_sett:: {}".format(system_sync_server_presets)) - if system_sync_server_presets["enabled"]: - sync_project_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) - - if sync_project_presets["enabled"]: - local_site, remote_site = self._get_sites(sync_project_presets) - - sites = system_sync_server_presets.get("sites", {}) - log.debug("sites:: {}".format(sites)) - for site_name, site_info in sites.items(): - for added_site in [local_site, remote_site]: - if added_site in site_info.get("alternative_sites",[]): - alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1063,28 +1044,48 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sites: rec["sites"] = sites else: - already_attached_sites = set() + system_sync_server_presets = ( + instance.context.data["system_settings"] + ["modules"] + ["sync_server"]) + log.debug("system_sett:: {}".format(system_sync_server_presets)) + + if system_sync_server_presets["enabled"]: + sync_project_presets = ( + instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_project_presets and sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) + + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + + already_attached_sites = {} meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = meta["created_dt"] - if remote_site: + if sync_project_presets and sync_project_presets["enabled"]: + # add remote meta = {"name": remote_site.strip()} rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = None - # add skeleton for site where it should be always synced to - for always_on_site in always_accesible: - if always_on_site not in [local_site, remote_site]: - meta = {"name": always_on_site.strip()} - rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + # add skeleton for site where it should be always synced to + for always_on_site in always_accesible: + if always_on_site not in already_attached_sites.keys(): + meta = {"name": always_on_site.strip()} + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = None - log.debug("alternate_sites:: {}".format(alternate_sites)) - for alt_site in alternate_sites: - if alt_site not in already_attached_sites: - meta = {"name": local_site, "created_dt": datetime.now()} - rec["sites"].append(meta) + # add alternative sites + rec = self._add_alternative_sites(system_sync_server_presets, + already_attached_sites, + rec) + + log.debug("final sites:: {}".format(rec["sites"])) return rec @@ -1092,8 +1093,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() - always_accesible = sync_project_presets["config"]. \ - get("always_accessible_on", []) + if local_site == 'local': local_site = local_site_id @@ -1106,6 +1106,32 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return local_site, remote_site + def _add_alternative_sites(self, + system_sync_server_presets, + already_attached_sites, + rec): + """Loop through all configured sites and add alternatives. + + See SyncServerModule.handle_alternate_site + """ + conf_sites = system_sync_server_presets.get("sites", {}) + + for site_name, site_info in conf_sites.items(): + alt_sites = set(site_info.get("alternative_sites", [])) + for added_site in already_attached_sites.keys(): + if added_site in alt_sites: + if site_name in already_attached_sites.keys(): + continue + meta = {"name": site_name} + real_created = already_attached_sites[added_site] + # alt site inherits state of 'created_dt' + if real_created: + meta["created_dt"] = real_created + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = real_created + + return rec + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 828c601d144dc0d24cfacdbe1a4ba29157349e99 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:38:29 +0100 Subject: [PATCH 09/12] OP-1937 - fix - failure when Site Sync is not enabled --- openpype/plugins/publish/integrate_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index c0a90ee78e..6c51f640fb 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,6 +1030,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] + sync_project_presets = None rec = { "_id": io.ObjectId(), @@ -1090,6 +1091,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return rec def _get_sites(self, sync_project_presets): + """Returns tuple (local_site, remote_site)""" local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() From 199717a0df976d21e23c5f2566a843ed3562771e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:10:19 +0100 Subject: [PATCH 10/12] OP-1937 - fix - wrong icon used Icon of provider from last configured site was used --- .../modules/default_modules/sync_server/sync_server_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 9ff706d6dd..bfc7c75c83 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -1109,8 +1109,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return provider sync_sett = self.sync_system_settings - for site, detail in sync_sett.get("sites", {}).items(): - sites[site] = detail.get("provider") + for conf_site, detail in sync_sett.get("sites", {}).items(): + sites[conf_site] = detail.get("provider") return sites.get(site, 'N/A') From 8d00883a04d22bb19454e6e18d57e8afe3e9bd91 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:26:34 +0100 Subject: [PATCH 11/12] Hound --- .../sync_server/tray/widgets.py | 2 +- openpype/tests/mongo_performance.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py index 01cc0d46d2..15da1b5d19 100644 --- a/openpype/modules/default_modules/sync_server/tray/widgets.py +++ b/openpype/modules/default_modules/sync_server/tray/widgets.py @@ -140,7 +140,7 @@ class SyncProjectListWidget(QtWidgets.QWidget): selected_index.isValid() and \ not self._selection_changed: mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.project_list.selectionModel().select(selected_index, mode) if self.current_project: diff --git a/openpype/tests/mongo_performance.py b/openpype/tests/mongo_performance.py index 9220c6c730..2df3363f4b 100644 --- a/openpype/tests/mongo_performance.py +++ b/openpype/tests/mongo_performance.py @@ -104,8 +104,8 @@ class TestPerformance(): "name": "mb", "parent": {"oid": '{}'.format(id)}, "data": { - "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa - "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa + "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa: E501 + "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa: E501 }, "type": "representation", "schema": "openpype:representation-2.0" @@ -188,21 +188,21 @@ class TestPerformance(): create_files=False): ret = [ { - "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id2), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id3), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), @@ -223,8 +223,8 @@ class TestPerformance(): ret = {} ret['{}'.format(file_id)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 @@ -232,16 +232,16 @@ class TestPerformance(): ret['{}'.format(file_id2)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 } ret['{}'.format(file_id3)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 From 87191d8d300338defa81c3894af849d98562e5d3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 12:08:04 +0100 Subject: [PATCH 12/12] OP-1937 - fix alternate sites for default studio site --- .../sync_server/sync_server_module.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index ca322b9f64..c5649afec4 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -793,20 +793,34 @@ class SyncServerModule(OpenPypeModule, ITrayModule): by provider """ sites = self.sync_system_settings.get("sites", {}) - for site_name, site_info in sites.items(): - if processed_site in site_info.get("alternative_sites", []): - query = { - "_id": representation["_id"] - } - elem = {"name": "sftp", - "created_dt": datetime.now(), - "id": synced_file_id} + sites[self.DEFAULT_SITE] = {"provider": "local_drive", + "alternative_sites": []} - self.log.debug("Adding alternate {} to {}".format( - site_name, representation["_id"])) - self._add_site(collection, query, - [representation], elem, - site_name, file_id=file_id, force=True) + alternate_sites = [] + for site_name, site_info in sites.items(): + conf_alternative_sites = site_info.get("alternative_sites", []) + if processed_site in conf_alternative_sites: + alternate_sites.append(site_name) + continue + if processed_site == site_name and conf_alternative_sites: + alternate_sites.extend(conf_alternative_sites) + continue + + alternate_sites = set(alternate_sites) + + for alt_site in alternate_sites: + query = { + "_id": representation["_id"] + } + elem = {"name": alt_site, + "created_dt": datetime.now(), + "id": synced_file_id} + + self.log.debug("Adding alternate {} to {}".format( + alt_site, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path):