From 96c2cc0f9099474fc1664d85710b95e36193ac52 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 14:00:16 +0200 Subject: [PATCH 01/57] OP-1920 - create command for background running of Site Sync server WIP --- openpype/cli.py | 22 +++++++++++++++++++ .../sync_server/sync_server_module.py | 20 ++++++++++++----- openpype/pype_commands.py | 13 +++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index c69407e295..a98ba8d177 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -298,3 +298,25 @@ def run(script): def runtests(folder, mark, pyargs): """Run all automatic tests after proper initialization via start.py""" PypeCommands().run_tests(folder, mark, pyargs) + + +@main.command() +@click.option("-d", "--debug", + is_flag=True, help=("Run process in debug mode")) +@click.option("-a", "--active_site", required=True, + help="Name of active stie") +@click.option("-r", "--remote_site", required=True, + help="Name of remote site") +def syncsiteserver(debug, active_site, remote_site): + """Run sync site server in background. + + Some Site Sync use cases need to expose site to another one. + For example if majority of artists work in studio, they are not using + SS at all, but if you want to expose published assets to 'studio' site + to SFTP for only a couple of artists, some background process must + mark published assets to live on multiple sites (they might be + physically in same location - mounted shared disk). + """ + if debug: + os.environ['OPENPYPE_DEBUG'] = '3' + PypeCommands().syncsiteserver(active_site, remote_site) \ No newline at end of file diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index f2e9237542..4bec626744 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -699,7 +699,11 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Called when tray is initialized, it checks if module should be enabled. If not, no initialization necessary. """ - # import only in tray, because of Python2 hosts + self.server_init() + + def server_init(self): + """Actual initialization of Sync Server.""" + # import only in tray or Python3, because of Python2 hosts from .sync_server import SyncServerThread if not self.enabled: @@ -722,10 +726,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.enabled = False except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). - format(str(self.sync_project_settings)), exc_info=True) + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). + format(str(self.sync_project_settings)), exc_info=True) self.enabled = False def tray_start(self): @@ -739,6 +743,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Returns: None """ + self.server_start() + + def server_start(self): if self.sync_project_settings and self.enabled: self.sync_server_thread.start() else: @@ -751,6 +758,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): Called from Module Manager """ + self.server_exit() + + def server_exit(self): if not self.sync_server_thread: return diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5288749e8b..0a897e43e4 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -284,3 +284,16 @@ class PypeCommands: cmd = "pytest {} {} {}".format(folder, mark_str, pyargs_str) print("Running {}".format(cmd)) subprocess.run(cmd) + + def syncsiteserver(self, active_site, remote_site): + from openpype.modules import ModulesManager + + manager = ModulesManager() + sync_server_module = manager.modules_by_name["sync_server"] + + sync_server_module.init_server() + sync_server_module.start_server() + + import time + while True: + time.sleep(1.0) From 19b5d47b2494498d19701e1441ed1b0a05aa34a8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:46:45 +0200 Subject: [PATCH 02/57] OP-1920 - skip upload/download for same files --- .../default_modules/sync_server/providers/local_drive.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/providers/local_drive.py b/openpype/modules/default_modules/sync_server/providers/local_drive.py index 8e5f170bc9..2961a07cdd 100644 --- a/openpype/modules/default_modules/sync_server/providers/local_drive.py +++ b/openpype/modules/default_modules/sync_server/providers/local_drive.py @@ -84,6 +84,7 @@ class LocalDriveHandler(AbstractProvider): if not os.path.isfile(source_path): raise FileNotFoundError("Source file {} doesn't exist." .format(source_path)) + if overwrite: thread = threading.Thread(target=self._copy, args=(source_path, target_path)) @@ -176,7 +177,10 @@ class LocalDriveHandler(AbstractProvider): def _copy(self, source_path, target_path): print("copying {}->{}".format(source_path, target_path)) - shutil.copy(source_path, target_path) + try: + shutil.copy(source_path, target_path) + except shutil.SameFileError: + print("same files, skipping") def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From 421ac7716ea004c3738aa1c5ceb00518d05aa94d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:48:04 +0200 Subject: [PATCH 03/57] OP-1920 - override get_local_site_id from env var --- openpype/lib/local_settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openpype/lib/local_settings.py b/openpype/lib/local_settings.py index 66dad279de..af8c3cdbc8 100644 --- a/openpype/lib/local_settings.py +++ b/openpype/lib/local_settings.py @@ -522,6 +522,11 @@ def get_local_site_id(): Identifier is created if does not exists yet. """ + # override local id from environment + # used for background syncing + if os.environ.get("SITE_SYNC_LOCAL_ID"): + return os.environ["SITE_SYNC_LOCAL_ID"] + registry = OpenPypeSettingsRegistry() try: return registry.get_item("localId") From 02b78dc7f8ec7a5c55e270156fef31f0f33ec8ff Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 16:55:11 +0200 Subject: [PATCH 04/57] OP-1920 - added syncserver command --- openpype/cli.py | 11 +++++++---- openpype/pype_commands.py | 7 +++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index a98ba8d177..583fd6daac 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -305,9 +305,7 @@ def runtests(folder, mark, pyargs): is_flag=True, help=("Run process in debug mode")) @click.option("-a", "--active_site", required=True, help="Name of active stie") -@click.option("-r", "--remote_site", required=True, - help="Name of remote site") -def syncsiteserver(debug, active_site, remote_site): +def syncserver(debug, active_site): """Run sync site server in background. Some Site Sync use cases need to expose site to another one. @@ -316,7 +314,12 @@ def syncsiteserver(debug, active_site, remote_site): to SFTP for only a couple of artists, some background process must mark published assets to live on multiple sites (they might be physically in same location - mounted shared disk). + + Process mimics OP Tray with specific 'active_site' name, all + configuration for this "dummy" user comes from Setting or Local + Settings (configured by starting OP Tray with env + var SITE_SYNC_LOCAL_ID set to 'active_site'. """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' - PypeCommands().syncsiteserver(active_site, remote_site) \ No newline at end of file + PypeCommands().syncserver(active_site) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 0a897e43e4..ed3fc5996b 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -285,13 +285,16 @@ class PypeCommands: print("Running {}".format(cmd)) subprocess.run(cmd) - def syncsiteserver(self, active_site, remote_site): + def syncserver(self, active_site): + """Start running sync_server in background.""" + os.environ["SITE_SYNC_LOCAL_ID"] = active_site + from openpype.modules import ModulesManager manager = ModulesManager() sync_server_module = manager.modules_by_name["sync_server"] - sync_server_module.init_server() + sync_server_module.server_init() sync_server_module.start_server() import time From a96b0b8d989d52a9a138859f1960c4f485c9c0af Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Mon, 25 Oct 2021 17:44:39 +0200 Subject: [PATCH 05/57] OP-1920 - fixes of names, tray not triggering --- .../default_modules/sync_server/sync_server_module.py | 7 ++++--- openpype/pype_commands.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 4bec626744..d8a69b3b07 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -694,13 +694,16 @@ class SyncServerModule(OpenPypeModule, ITrayModule): def tray_init(self): """ - Actual initialization of Sync Server. + Actual initialization of Sync Server for Tray. Called when tray is initialized, it checks if module should be enabled. If not, no initialization necessary. """ self.server_init() + from .tray.app import SyncServerWindow + self.widget = SyncServerWindow(self) + def server_init(self): """Actual initialization of Sync Server.""" # import only in tray or Python3, because of Python2 hosts @@ -719,8 +722,6 @@ class SyncServerModule(OpenPypeModule, ITrayModule): try: self.sync_server_thread = SyncServerThread(self) - from .tray.app import SyncServerWindow - self.widget = SyncServerWindow(self) except ValueError: log.info("No system setting for sync. Not syncing.", exc_info=True) self.enabled = False diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index ed3fc5996b..bb7ad152dc 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -295,7 +295,7 @@ class PypeCommands: sync_server_module = manager.modules_by_name["sync_server"] sync_server_module.server_init() - sync_server_module.start_server() + sync_server_module.server_start() import time while True: From be36900ffa037b4c98df70b87f68f4864e34a9aa Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 13:55:47 +0200 Subject: [PATCH 06/57] OP-1920 - added always_accessible_on to Settings --- .../settings/defaults/project_settings/global.json | 1 + .../projects_schema/schema_project_syncserver.json | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 45c1a59d17..46e5574eb3 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -319,6 +319,7 @@ "config": { "retry_cnt": "3", "loop_delay": "60", + "always_accessible_on": [], "active_site": "studio", "remote_site": "studio" }, diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json index 3211babd43..88ef2ed0c3 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json @@ -26,15 +26,24 @@ "key": "loop_delay", "label": "Loop Delay" }, + { + "type": "list", + "key": "always_accessible_on", + "label": "Always accessible on sites", + "object_type": "text" + }, + { + "type": "splitter" + }, { "type": "text", "key": "active_site", - "label": "Active Site" + "label": "User Default Active Site" }, { "type": "text", "key": "remote_site", - "label": "Remote Site" + "label": "User Default Remote Site" } ] }, From c69c15310fe1a55ffa9daf1410ad189ff125adb7 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:02:24 +0200 Subject: [PATCH 07/57] OP-1920 - implemented always_accessible_on Needed when new representation is created to map wherever it needs to be synched in the end --- openpype/plugins/publish/integrate_new.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 451ea1d80d..fe780480c2 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1028,6 +1028,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): """ local_site = 'studio' # default remote_site = None + always_accesible = [] sync_server_presets = None if (instance.context.data["system_settings"] @@ -1042,6 +1043,8 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sync_server_presets["enabled"]: local_site = sync_server_presets["config"].\ get("active_site", "studio").strip() + always_accesible = sync_server_presets["config"].\ + get("always_accessible_on", []) if local_site == 'local': local_site = local_site_id @@ -1072,6 +1075,12 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): meta = {"name": remote_site.strip()} rec["sites"].append(meta) + # add skeleton for site where it should be always synced to + for always_on_site in always_accesible: + if always_on_site not in [local_site, remote_site]: + meta = {"name": always_on_site.strip()} + rec["sites"].append(meta) + return rec def handle_destination_files(self, integrated_file_sizes, mode): From ba87f9bc09a14c30a0d9082c295d0803ba7f6219 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:47:08 +0200 Subject: [PATCH 08/57] OP-1920 - added documentation --- website/docs/assets/site_sync_always_on.png | Bin 0 -> 27817 bytes website/docs/module_site_sync.md | 39 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 website/docs/assets/site_sync_always_on.png diff --git a/website/docs/assets/site_sync_always_on.png b/website/docs/assets/site_sync_always_on.png new file mode 100644 index 0000000000000000000000000000000000000000..712adf173bc6049a2f55b7ff24d4d6cf09928ad3 GIT binary patch literal 27817 zcmb@t2{@bWwmz=&r>go@(NgrQmX@Nesb;OVilT;4GgV{6JjT=solsTOTr)vpiYY?T z)=BJ5z0cX#-e+I?oa^^buNvZgpJBc0S@*r}wesqLz82?E{-bPc zY@FKnZX2<&9l)`%?LYFzVc?xaHAE%wZJ)1^)-5)8@7a0amxInX^=`7URm5@Z+8+Xb zXZN~i?#sq@yp8pLU%O|~V>UK-HSOCsO@eGFM2_@RGa1XwWiP|SUs7D1PTl=PI)8hp z!%>6%!lzrZUtI1UXky$>*g(|p_CP!N8QxQh&fJ(_$@;TX@Z{Wxs(508%Sf`zV>zi; zH#rwuS6cr#&2yla)9dho-b4G_^k!`hu*RxH)<0v_tEy|k&LM>h#S4Y763dL`-At$w zV(>3uiEM0rV9iB?-#_WT0Pp>#0Ng#w|#d8b0E-R$RcsL(&QVk0_<6(UM%tB@RPX^c3nPr<88?Qq zdt;w3sQ5il3hfB?yQg)WJk_SP?x&d{1Rin9VrSj=`E#=A40I-z2%Rsp#_xo`Svyu} zKX+nAk&Dw?sBLv9ICbsxUtuaRW8pZp>^M!a>YuF`msZ!@#^bEJ*`5z$!Z&45AMgrI zE1aky^^a>!OATZbD=T-)M-)P>K{I0mL0r=Z_EjHNC;(S?bMcLSmi8blVDeHJbl`?u zwXjJ~beX)BZaYRL+u=_82C`C{wwf|i6MbR9b^sG@3pUH^E4^r;g6@DjTB2=eqsu)0 zvON}+55od(*mi?Uhn$-$G*qn;r!5r7^B^Qln1tqH z+8f(N-twv1e7P&T4q_1z>A2$FN&Rx6C6%wiA9&*i%t+}a&6ctt$FI&hZBdGG*-N>; zTURLyi{%*oR?dtv{m=BMkbTW7F&AN_*7=1BwRl0=GxnkJm89J=~VdFiE|GR01NxvX?LrN9V2=>?J7n0RKws=oS2fq z+?RLyoyVB+`*y1TT99pP6^u}XTJ6Ls6UMwKD(QBR{Q(Qx-4@9D0@K8-nB%YSq&9uN zFs_hsZgPot$#D;HuWK`AMF+iiVb_~)Vkmfb9ijRu)r*(z17!|yy((^eC-hAnL zShnn{sn;a+@Zsd`g-hEaet8UQL7~8=Z!>8yimn@%V1E<{Lskd7nW*R8BE|V0xBCeD zC5_%pO_h&?-M8IT)J)yjT?1!q&XKB%n+nXg`93!39bwRJP=Y6up$g49JdDndFq$p} zc7s_i?-Y96@b&7@W)?F$FgI(8yg85TLXyMt&yKz;}!v8JT0Fz;cE4eF>^C7;Mqv(fpz>?Cn`aBW^K$ zKh2Gh2%2%N9VE}JzE?S$TT?lIF#5({coX=V-`p2(x$50J(cL&7PC47}Gb&L*GYf<1 zR|7|1RuO0NawHnASqu$0@ny^?xaA#Mdgm5+wpQ>w<)AKg+(CJaR@E!;=NV;+N@0F$ zjT!8s_Qghj-xxQ|L~#l6fpwS;cG#Z+vL&rs%MlOHjHf?Gj0I;Zj1kAwe;G}mc$H~6 zx42^tqG{szuDrA~Fj>{#w9sRE7R^=JFgem+3f1Ep8G2~Lb@OFKCxJviV1MTdKf1^F z9uPBwQmx1@)B;*J*u@?OD(&0(@&{@1@qC=YjZQr0a9e&O9L`YkgZBz4wZbp<59Hj7 ziv82*())%CyZX;jV>$CMSak~usaGcY5MC*qK7msrZle`3DfZkB!I9khmRR1-Mr(}u z#@%=|)W{}ev!v|J!NG=M^HtOwQSr8K>&_#w4yBgH4XV6dA$=_KLU36D^8pU*75zEn zjZwC57&v00om#^>0&^1K*`{?s;Rgp+Cg|m!t~cmi8HC!O0Z=rX|Dsojwxjp?Hpdm^{?7ql{f|ZYq@9J`Kj{d z)MBUR>BBp4Hzk&@oU_?g9bTc@?Fyw>Isz;HU|`kgTkJghBHFT zEP}GbW&Phf@tzuS&VLmBh{?EUDJkEW-JL2YBEDWzG@Rw#%{iV3yDISJXoip*_p!Dm zbWWvC@unWyocv|Ha?VBY$Jqr1t)5*XEDFyXdEdN=t5-d0zIlcd?J5mdJ~(dq_EKuo zL*4Z93H}}29zHxtbYHE!y-!&>W4S)Od_Ceus9e2)~b!q~pD&(H$ z()OE7II-l{$Ry3;t0I3Yw86_@jA~^TE>9D(6LV*lemfNeBEK*aaYCh(e`lW>=hd2V z-iSHmlg0fyl&cyST`PoMRLf1HC5IPGd|#gu&QlQ73pR-lf9TzWBKoaFFlWP%$5l&n??i79*S`Ck-~9Q! z5J9?{4iW7b2KjMqiLITby%3nTG`aJR#V4Mq#SfSN`m>O;zgvVQkX%HkEq z!+MMQP1^)($F1K{r?|RRjx{dZJ*gw@o_kaDHJ^a*q4ggxz4lN_L*wt1;k?z}6(>$i z^0V%F^5hSID*>;AAIz-yXG8vW+mo)}L+<@MFV-cml z9515fHd1b{$X&|m3!hEzsZYb%MLi=`PPX~(?6S6RSAOp_To0*$*|vpWCuQntwj`W+ z-(K#;nz-W-`(WCcqnYHxt-s0CbkvJUrR1n*VlpRe_Dpn6;p)<9)}{Y#BiXH1|1Bu} zhi!PamPni@xQ6&4zJ6kf?rUuiV$QY0juSwv8#H>mu2>~ce43FkbB$!yF%OHKcY;~z zE=6eQc2O2CuM{w-dbZfIUWaXtsdY2;1{?0hq-w~P7HOMVWJVEZ4Pp1I>zow^C-}Vt zN$S7Y5L`?RYIJS+6wL}j~*MO4Y6_BU#t5|~c zd1%_&R=*^zD~=HjKdY>7HcqbHIR9gBU$n@^p95Zn*0HBm7mlG$eKjddj>B>2MwD{a zbUKLltM}Job=QAD39`!`Wkus4`#vWn8yy z6$h!cd0NZEZluedi{MdkGRshGbPz3(Oe5lRQySb{cY~IoWfo}dL8(knT!wq_%6*7A z<)D+;;|ZgM7tXZ<^m2LSn?YFX32dqp_e;p3G0x#$-z56&!HF$6Zqp@!mnft<9a>O8 z7N*FP9@fH*u;k5c;$~0x;)3+tVvnaE=koZP0R0>0^Q5kMXvFgPkW8&4e zH#fyZEelim5img;Xhs3Hae-6Nmib}0$YLUnYigyd9bD?Pz>AuAomlIdN*q#X5S2w? zj}!G>0*!I>5%>zfLXCM3iULEd90)s2EZn)LIwruH<*L_XUU+WGk1;ElcK|P(6H}Vx zWQ~ySeLt*H)eDx@{Ix`oEbxmx(YpCl~3ThoJ%aV;&0tb`4Tfu_LtO6RD8kc+{t zOINI`3-8JiZ>=4MjH_s7Sq^M0I8C^UDOZKjFZ;kokQf`WN}w1=j5&^$37G3K`WN!txix!nbolk<=CF)*7UC5^lHYvmqIo*A{vlmUl;pH)J*+zaWFDwaQ;4lYY{^X{cMblby|4Pv697*oAk#DbDP z?Sd5nK~}Qtp%JM&j3&3=L4G~LoC7zR&o#G0+Qpa1Q@O(|>eaM=$M8O;V}%cq`M1z7 zrn&6Qv1ceYslNflzeaO*FA%iQrB;my342bx*@>x|qup-@S7VzRARb3QU-pg0yu`TGknix@LV;#PUA2CpPz( zC3W}f7(VwrRm$%!r(hLzk?UhhWxHR&crqpa2|F$;akzBoZiLt%=KOr^<*ekf=}t%5 zlyL1R^5JaBz3}xPxdD>(&FCppsxy-zev;d-SyI&x2*iYfQk|g)TVWZbS_HFOy1Q04 zj<}J>4>4AlU*twala7^Q9GL^^2*@TwabXaLOT*CY5*GM3s>tol&TMQqHlTv=T%7v# zn<1w0sI-3CH)Gn&UBwm}v}e&RKz_(&hJW5yzYY#58|Xe-YnT>}p9A zO9_vR`BFmI7-W`jDa)n*M%7ttp#(H7ciWL3EroO5_0J7PE>R|v>*~Lo&ht{!^Hl+h zQcL{|lv=!}Vx2E2&uR`R^hm$kbZ#a8yxwN#IVg+bMXRm^`zI$-F^SVwr#8NO0lkbT!~eSMPH*xu*e+wH9D4A)Kk?|=3SCJ*Ro-grfiV|4V+fhW!QX; z7_ZzOz24K0`y~i*ZPL*3%UrrPI4?rW%6&Mx(|w^{6up&x+g75Mo{f_Wt6*;5V_>w; zj|PRQhI^RB;i%H#AccwAUgA<_vrw#g^TPAxg==BTH`RR*maUlVHE6ro?%0K-brtJQ zwW|5*cAKf&Su-9GMDGe*4;GQMOUAXW7%hAyNu(^&s{lWTN=Q(xmWg#nTx!2wQnx@c zvgE2%<4Oa8h?NiW$kbo%wcAhYi*&LDYbb$`)cQ*(s+|^d|DTfPAqrD4xuuPZOYeF^ zvRFVTr*gHjXSZ^B_m+PMa#Ll260(-7H{L{!C{0^7FH_yT{B6upy?AJVyQ+qu0HW>==#=*^bm2|2ZE>tUq?j1sL6Akf%VT*LWfB%s z^O%aS#%m^<<|1O#HHH=`nd8nf73UC$b_@kh#2YJ(H^F@pljIf2RTN3daoVgIIbnTj z7jTMq65|fre34ahn=~{SEV@m-S^XhvDi2ECUeNA~0K$yM@`jRCmI6xW zjz7|dM;SkEfvv&MJhlL@r!v39S)4-egkZEHf3f|Hi0_!=dozorDOg6jT5W?NXIOw9 zaV7>|03a2-Xk!EMZAsvoEeMph{>(P=)hLFrSyy(_v0bUmCoXZJ4t;`-?~$Gk=tm!) z%85<*5MiTA-q=a*tzTbny3@==G{IG;zJVKS@%aGUY27lW?GKX@S@f4BSg|a;=f3eJs8vV6;wtc-|+PX5l(b-jW%5Sa8|+yIQ=%BnSLQD5u=}jDRWH-vz|Sh z7GBmqehGwCsq)VG76Udi-o30vO<3?f9nVnTfqz`Pzsgwe%Ut*6iJwb9S-r8f{M;{& zPmGX-xMaQ!MENIg9Z$o*`$uN(%vKLE0`JxuC$e8NCf?FUY~6bLK+6`4Jn$vADCkDRNtQh~@I0H3f3baGc)p`R5pC1RPQce&0#DAUGc?AJSKXBi*CdmzfzSP#apSq=z$c zxZ;)Pq6mHnTyKaZ?yat^7LDF=R<~z%j%R(KXsemM>b;z>_I8K*R06wV)VWEL(DziA z0CFAb^<%gVXumK+(ArwfGd`);t%}h7z>6@>b&&D3mHzee&6#gku`fP~N14sIjysx# zElEF4Tdz}I?2E_?Hn72WKk!5>hh+JNUxPQ@oJ{bs(&~cgJX@=@9Ee}pN!E`PBT{=_ zG7g206KM=Ny-jFpc5@Ay?>f^;v?*2~(_FHr!NxZIVp*_gTCQ-;A*MSdh$x_RR;{oa zJO>pHQ>)??y}q5ECz642;=djq<{IkJvm>*azTGggeYPHfLEKf|_ds}YJwT`5;3vjD zin-`a&O&@(9fuDs!J}Csp2kX-|7@TZ(~&4b#DfmWnuB= z6u*PnpdG*(bA`jWYKo_Cn|VB{)8vEP>3s?B`Z@<&+?tiDvK@5Xsmmt>P6mJy2L7}G z0xjdVvA@xvhuG)58y>9;8D*yz+cQ2^$fh~l=ptJU?kqE59dD^WU7J07H)uxw({zQF z2Mp&H!JlAW{n$KQe6XF8QR1V zYv{IO?n$RWsEs<~4jh>OHlvFa7-*N4fg#UprMc zIJEY%K^YDfZ30nG;MD$U_DQ=Lx1Gw!C_B zmj(j4ChGB9Ppe((iMaG=Rnm03-dKca`(mX&ms*4N;4X<6UNJZe+DXVTo6ruIvW5mt z*<^7#p;PzFw6 z>xqwX7w8d9g9*LPGCKpbv4+VJ_S%3P#kMjh<$XBy<{`Qt^Q;pM6TZqYZdC7N5L2do z6X(*QDwqpQwU@&vjCYN?EgRbp6J3DA&|dj9y_T?MZ+?AYmJHhek&o`L`TV{#!rpYe zIcOBBeqb&E$7A1wcdc<7_-Gh%Zh-(Mv=3~#&-Ri9#8xH{>`GuHd0*l$7 zqn=eMpOE^R&(Pcs!Vj9 zQ!3XiRO@n$0>^-yKcX~A$a_`emSFq zogw-#&7SN*K)9tp+!MPryv&c`68vA4oyJjK!Vlx-Yr{Whm`s{}vYbF*Z2}2ul%R>* zW9LEo3u+&B8%J%v;+H)=b<=jLT<+f^>8gwK={LGs82~ zrUF|dH{equ5gUxHdB!1kvqts@pLkL0Ob^HwzlTw?~wU(DTkG#_}=Nw7o z=Barza4?bk1t4vBojfL}*zoH}fa0Uaj7z8wU8nUjA>#eBne!3DgVN2}IGYen0{vC% z8?pKKzHidKtP6295!3Q9slT+tR?|ZTJ)l~i6s75!#QT{uHOQvsPKX5K%N0;+A+{+y z>DqWD`(krw3wa{Uovz`0m%^u5z(dq}OvvV1PYMg)F(2RDjgJPA)@L%p1& zy3W9hf=`|`3uUauXGz1i0|OObcAd#GG`C09z(Z`e9i8VF8xP>Emg#>QWV>24`emzu z4D+s$BJL)KEu8Q97#=Sky*PmCiAL2ZLkd)(mPHG0lC&>>)TXswj|hq1^z`s!NV>pg zZ2KJJNs)5Z-l$Zi^b83SISH(ud*(0F$7IEJS%O4V(MnrILzlW~N4i6%Q@CpduOdm- ziD?OwiQrt%A}=-v6NIQEfH&Vj#tN!FN`90vv|K-qs`!SMR=O^0`8D_YYv9DQas7#wUk2;h;-l3LWx5hIo%9hMWp2ZxMdxZ zvR;Woj5_sJfc%%(l}jAQh0ooJbZ$^52rNDY+@sGkW5|m#&Ib_{dULEtzUp~kkNkK{ zy)^#53a{LUo#PR+zo^4=Efb3SlnbP(=K3$0VbkE2G16H5JqM9C97NdZyzEl_VOpYN z?SxY@RLsObTlvD+i+=q8^?T-o+(CF3K09l12m0Ze`wh!Qw;R8d@0wC7(M){O!H&_>z5BAw4 zF>5p@X=~#k=2}9MHQs-)=DP_Dr_?zzF^dpzs@p;ua||_Y98$G*&@4aGs~Yjl?2zZj z5#-`Q>ImI2F6+l>%L&s-6Ot8W4RDA)_4Eb=a1Mjbv_M(p;gh9LZ(ay04|vh)!-2l8 zSxIr96upPi<2H#8?fvqH`bg>za8%e%+h-Y`GrdHo<&9nqNShSZdE_9NKFg|DB3?uJr8x& zExk_t1M1V2E7HwJ&h}dGBn1lBfRh&>k!OJw(k+|Wacn1US85Q9n!mgHw|cJUIcg7i zgcFDgDibD6>SfmviaGhS>ck(GS2tiq%l&Yih0JzJN?teQjX-K0P->cR2SqHQsg9f5 z`C%H}ekxAKOp;i9Bw4LgGT_FxsswtdX|f5PJ!1)jf46%{ZTQGwNs^E_r%mq~v>JhP zIdH25(7FJaxMa?W^KCXMs}Lo(Oq_n z8oy!Ss#hgTF^zh)x5I|-5>lLj_?eeKAH1^G0m%P-c_aDO+tOT9nGJ8gYyKp6ZS z65|;4*tT#sBqu5+4MtsEfwriKK}S6{zo<$(sr!z3>T;z(R_Iy?F-!TKuWPn%iao># zM^0M-k2)tKhBJU=_9y*Es8(qqi9J^#zRf-j+^LFx1qK5`hFhB--GkD9-x@oJ^VK_z zsOUi<30IOa{p;QyB@TtDX}WSA&#W^PP<$YWdTAQ&R{5UTc>xz)6`8h^XdF3EWIy4t zaf2oV@!a%EL6CfEmE&$FyiS?K!-lCnn}l|-K@~({U&Mx}9tx{QedPLvcoLl9=4Pbl za<^!2to2f^sK^7SJ5KLOyMDOn#NV>z;Uj>;dVL_RYva3_N2TrnRzSK|7ubr%l46iu zv-RscGk+_}?FYe?d3T!wI6vFPGY6P?d-UYyo4-}%!e0Els$Spa<2V826dXZ*9u^=c zi4KmH?(p3!MBID(_mZCDH7f$_S4{+V7$wyoaU-sb95@Pz4YQ^&y>VRFpG!;y0254V z$6PIu{iB^^hjti(5ccyTG&a|A;+s_1uTK*JuL9h4SFKpS!nVK5X9mv>Z%!7SV5X}1 zVkYjDFj0cfCZwXe z>)8uI`Z-M*51J|L*qEIoh@Rm4sqg3N_r^b%JmW<NU!xfgx--)~qO!DX13i7%#@y{*q32n)h&NcqbW?wBE^ z%0Z8*@z^f!XH6Mi>R9v4Ql>|AB*A*$G5y3f|L;8t3rUA&YW|Asla-X;33s)4_NT0B z$Mg+gEvK(ms$B`4K-CyEdOqKjGXZy>a;^``x$A~huu>*K=aeJH)m@!7?kH6!AC?=V>)h0Rvp|9KzSU@%z5ZZZvgRm0^Ug z%C_S0%#;v&V@bg`(1Ib?4S*h^5^0m0J5q9P&sz0smxT0PA{q*iU=(igwYyAPt3xah z6`nku{_&I^aM@IN+*j}eJGthhVDJSRREn*Bp($EOpVnbwn9b);5a4%Us`gjj0qf1h z%#IsV4uZMA5RXzh@OYa~#aJ`2)B5SvXENB$*Ud!p4B9-LBC4{3^>&tXVwIo{XM~@H z_n0}~(KDJ=I~k_R_rwahxJg(vDt$~TKE;DSlw7P6NDM^Z@OT(h6#C54f9dGYG8ouv zplZ;g4S|{x!*Vom)6Bgu66m zSRVBQhQsq1%$XZ|=o{;%DKr*-x+y%cPk&m$ZeaM4?MTMpFUVBG82jO0_s4 zXIUp*N$eK)s6iN)4O*eLA4Dd6l)ZFPpa09Ul|Qws=$(8kX+U`Rh~YL+3RY!`f+1L z6-QqGF%XTsqi5lZIVqr&aRRmb2qI6`&1laZ0O@|u55|EF6S(i|rL6^dh2@Y$jbV4N z0}iwG)Y2sQcEHdHict*xhj50P-VOP7_^^=J15~^BreFv-XL3xtxBoDC-ZQi|nY7pN zc=|pKaJB8A7&-f(ZSk^)^C})0#TBIc9|9=DnJ_~27-vO_&ohdq0d&IAl9^{ybQxR6 zzI0eCLB83|GcOS;S9tE4mP$6?!EzaIvBmKh;15iQyGG6=5ve z@-QR^k3JgG@OKMU%35x9o>2H6h)4-{DiOUdapTGTtCp-_d$>w1Q&J`E{uUB6-7)3S zgS0%5reHa^rw6lIcG=DqwHVjOhGroF+kV1&+WlJ4+XK(jb3T`>4$N`wl|(fH>0S?6 z!?Cf|NB%Qr{sa!x8O2eml|iPJQ*s_xs^#KI;6g9A={_^>J>Ym}`S<3Se=CUmXGP|J z8sZ>NYQa~9i6YA;S-rmqr*eIoJgyP=J`kdW)`x5huB)m6>$rZm?(n;+rI0y$)G3pC z-qc@bs1L0`k87O`iWU~qk2lnZGk~kYfNh;Cbd;!RU{Ib#4^;hCUe}-+12gv!K9iEK0*E^$pOC)v? zJG*&Z(x%}-7VLD+H%Q3PDH_ z=AAXPcDwwP_~6^h{bH!)f!V)|{l6o#`rr2LgTOtFXPyc86CKAHOKsK5XW|>0J?_uY zIU3xd4-sI+WZ%4Y$Hj^E7AU~0|NX(Rh2Z?={-kbO~>plro zi&A7#K>eS{PZ~DUS3XGo5}o9dn8@#u>y-GM$N`TXOAstxRhk?Yv=@9Cu^S*a)GpW@ zjXPN!q;)IY03lcZ7y-}}yK_MMnMT~gMuBn(?x6L;anGg)k*}^ldc0M2#?%A$gAA?n z92JgQQh~8J1pzAGr`pNb-foYuNo%;cA!MUVQBo!MM$I-ST|mf5j)sQ;CxC8d85-_NdN8no51Ql(w+!5ybrQ`?EmV{gb>*DD~8w z%ws1m>hl#D%gP5J;B<6Aj&KEu(0>{NkbC>uO`>U9u0-!afVw~V060*`AA2nRn3Kux zp13&?z%Z=W{(sw3_n*T?@YnoFebiaQv*H!HoqETTUE4PyEY^R`b@dd*-MxHohGYM+ zFXO+$5YS9=PDCuE=GU8~9;B`j#UY}am0)}>_Wt!b9ChmN8N|PKg8cum7hs4v-G3m_ ztQdL#;2j#h4a?cl6r-ygdRuy{49AHw{(LYAOW-8h38SPb;1a#Cl&ZUOn z(z15A_FHun#8iQ=2Y{91Tltu_hIS zIx>(U+MNSzr=hc9fqL|uUqrJW@n6bRy@ENlS~WFguDkc7O<4BP?h7tszQB^No}1=L zj#8%h-1aW|B6U&WnpJ@yp>{9q=XDxa0|E5&Xzk+B4zVG@Ur4hEnZ?qE%U7Q`B zTa!Z)#FBx^>7a)O0;?X!cSVixV>zL2Fi#lvG3*nFHETfQ37Dy%;Q!MQXTNQC|!X z<7bb7r0NWlmBf-Nxwzkx49X=;f#|uJBKaEq$a|M0=g~JMjV0X8ieUWC+jK?^8yeu+ z$=0;C*Z$X~CChO9eVsR0dB(_p8S35O{m;EZ+!wO?rX+0oh69T27C;s(0^$e*rDa|V z%lLA_c|X!u#kadXokiky|n#FsZI!k5n2}RC=?LNZwlWy+g@b+|Av2Rj+3FJ|lOK)SV*fiN3B`l zw3EM4(GAvvJ;AqWIu9H%)6mT!Tf_IbGG}K**br-|3zSgEvo09Kci`moN&J-vO%7vwwxr1pKkn=lxYK23frMAK)*FJ zRjmOyyjleG>K*%Iw4RDLgP*Bs9UVDg{c0>f8FCIRnf1f;3Xl;)cxpiYoT4$jMWu}) zpJC}spc{0C1-`L4i1!pPtg@TKJ;rkPwFi25)9GQYuvJDhdvNx#-fNZ1!?h=)dQ3!0$u%UjpbzXZ-yT0Q(^IBw(>uS4P17JH-{5PPRL_%sAkEas)_$B1_yYuhLe z>r_@)wgG4P_7sYxmZ|7-g=L*$@BOQP191K4+(3NO|L*Y>Kjr{F@M*2X?|kKh$&@9` zW{o`6`Mh;oJY_0HQpu$@<5*{J1(rSbuD$(&eb1|2^^86NSn32Qo2Bn(kQy|LW9FV@ zI-J}y-wojGO_wenCU@`U#O;<3cxzh9@9{V;eRh)@3Pdo5JvL#;N!P)KXL7WeV^8*d(7!wNbq$po|wtB#Wh78t&r|k$JvD`gzbqIaHZ~ z!w*PN4GtVSq%VDUtlkN#8Zsvx{{zPYAKCvV(-Ujw?#Seo&*X;fp|{ihS9n5#nNt;{ z`y+hv`w_-~eR5?FJ9JP?DSqLlTBs)~^(Pm%_~0?_gM>g=Uo;a>Fsa>s@-5V@z{#Nb zo=kzk@W`KrRUqYCJbcC7iuBEIwI;Y~t!4S+hBMV1ay`U};Pk+xyC2oSH7;w6^F3Q% zF6gc7bnR4)6Kb?1dqU)@A62$ac8>cN@VPQ)?|!}Qz}$@u!ptktg?g{i{Lwp^o5$F} zIzc(A(fenRaeXiU+)S;`yTPn^b`U3K8XYE>we=#*W7;ZnI70sG68D;gKz`*xTvJav zd8EoDY0fH*U#2nB(j*JkynAptLhCg-ej31~Cr>9w`b7dR^V)trbWVgm4-`{s;fSe) zfaN`mW4+BLraiM?rZbVAN&d!pkVUO-0-rT_^7a8MGJm$a`!@kpZBNkue~Z+t4>T-Q zfB*Aq_DR%-u`gaJ0qzq{IJRTF38Du5z26R_SXl7+)Bn0C{sn0Jz3x-#%rOyQ3Iyv#LkujI~1cU0IDBO95fJyzg}_3d)4OW zzIcLy=rAT6x-2xMZVDc^9fFoyA7T1vw6Dc3bXSO&DAj1RMXezE9*gsh6C#5+?DJFl^S29?;xhYI zG_~?u?Uf{$$B;|Q<;r5l0Jd6`-pOhJ&dN^QdCJ6CQ6sEmW9AMY`0m^jd0cJ%)Afe$ zQUf39p2cvinyN!*9F8m{GxMU2T)=xTMqqtQcf*LWmzVN;Ph+5-1fdPTuRk598XlAI z(32ZHuS28{xI#>(-jx@;5QF)3NTCU#BOZVO=`9txT8lDdC--9=3~#u8tToaIC#GMnV55@Sqx;|Bt z5`a78LNt$NEDL#^?7fCc8hIyUF!+nR&=nK?Y8mAnP3;YQ=JcBYxK;ExeOw2C5q|-N z_I{oTK|R2wx_Q3cLw=ZC8w^lyAD7tv8JoNsq;Hbu^sIJm-QU0kQZ^`rW=agq%$}y8 z(jO??v=1o<2=v@Mzh&kImHJh<{TD_KYMCc_XF}U1xA+gSMyV2loGLUz>wbjc`_29~ zq#VzJZrf6h7&}E^@-zCR{hI=Yfa%Zc5)Kd1Ksb z(%`<4W6osj<+#9y(OOCvLFHSK!8Rd9KhFoE=?0!f1!a8#4XUk*49~1#4goHg?)6tn zA?{5Ic`8aTnG1IyV~g(U5ie4wqJxveZ)N%PwonGu)gvATlyxn`?{XetZAUM<0()2nOQ=SUZTvojAyPOiN-W@N!klvNV>-EbaOMjCq~mm zap|So`z8bH<(XMk5hGh&-!Y8dz#j@f)Dg+@AvlM{mm;)ZX!sh2BZXR`p`~_A?E|3E zJ?3g_-FNKLcaP>CsjoW29_e$7L+|1!I!Wo}>fA8vLiLXo=%!y*o;k2IAd8Z1rOFbxBxASL0AB0V-@`IC-YkFO|a6Y3_gIRT&lv&fdQ}x?{wzH-iEDEy8kn?>a4PMn6)1Ds{cFK{uxl1ePe8bTym^c3&Emn-C`I z@wb4uNXb9-Y8`IFCHJlQ2eGR#ZiI2p?23r2-B6+d^WcdAQxMj4n*9Tu{6fL&cU zggMWXfBL4!V<<#|&ySNxrOG&9&f6cAqqL?zDqE{B1r z#8AaW#|aIZ7sd=^t47PMKg`rW=3lw$s;x8Rth5hhSGO7)s3V;9E^69!u4vGYma^UJ z8x5p5L%8hRV1%+ER!nU&Vxl^wx~rS9Tg@1}|4c}g-89VdG1Du^2S+vb4Am3perz!) zH6&Zx6a2BWa?v+uBS-~jKa&jQ_DT6`QhGbgZ%1k&G%-_HuG7>wSAJP~;H|;^RZ(4; zFw(`X@r76sepgYF_zi0ok$1Aj!3xZ-^!FAh&1ByFF zNsHI3!fRIJ-g=3h0DIn7*)}V{)!a;w4GHig07+FObqtqUj<{Rg9U)C;?=iB{WrmcZ z$%daA1AH0%+^LBr3)e|X$Yum{&1rUm6KE-*=~uG%Y>7*;o#~uG{Cw@ylm(r8_ZPo6 z?JCt-DR%w^cW)f|$5Rod$FLuYouPc2R*^(6f|l*dH0Fwj(`2vT$?}$tETqZ;`TH({ zOLueQ5zu)#q`e8U0+Q7;XD&@b`(872%1%*bkML~Ay6g|w&k(+mbGunsY+e=MCumJT zN9NhfYmYf~w#sp*vU2MGO)~v2Y~-Jl;|KpqdE2XIKmU8-`QMKB=#~s%n|$`3e<(?^ zm=$z+uWZQ5)mX3p7oF{&m;R3*f9+dN1Wy8mUtX#=D^-^PrlN5It(T8)0xe(y;q>-U z5FT<9B=xbR9FO$h;D&~NU4u!~wp%>#paR!^E(VnPCr?CgpZMu;Gy1>VB|kW_l(pX)wEb5lEVL}v zN7CwCng4oOB7DLy7g}jqH!nIiYTO#)Z$_14gN`hyVZpC|Kp^=G;ZjbhhF0qcq{_(y`h?ir4K%Ni|2;?@?HFfyF;$C959>n8@ zn(-ai`&PX35oAm6Ng-e$F(7~Z{OQ#1>Z_?VyK`QCkARNJgSqkhf-i-a2zU!(7I4`O zIhLv?ss4Z}|4<-Q1dND!&Rl3Y^O)39(+Bb`LmC zmRcjp>qmSOs&kx=NCUl|!Q3B_K#`TGY0aCY(1l_plo}p#Lq`zuTCgAS3=VADJ%3=f z^?iVo95mS(;D!!&pXbd4H&4;T^(05~)$0=Cu;8+kyi%Z3PGB_YYf58=*RqiE05ej& zJ2X!TxOe~0k~+4>FAzT$Nat-S8#S3>PH!@9v&6;GR-pRy`N`uZO@c`&<+&a3o|g`g z9=-c>Pl!9+{WmqNPYc*9>vixyrxE|w`~SIVU@(?KSmb}w@(8F{<3~sA_GUQ!pEDaE zsC4`Re3=)JcYk^G%{Q0n4#OG%&!=USh!>%`OxO_k-vFeLZ9RTBKKGAdAlYVvKp;RTO15F{`?$Lz z|Evhsk@z+40$;A_us-5vm46=PHbh_glRm48dT&xO8@f?7keZ19xK)$?HnH0rmS5G- zo3{~Dxc)t6XVF4FKVqqZS(c7aHXSc;4bXENNDWyUfvoTgrK@{M| zlXq-iglU>wN8N&jSfY-n95$XoYvXK`bd$T)%ruDsg8^tK-kFm1gaTr4=%belso%)D zZ`#RkU)c^J-{^o=UmZ#fo#up4nZ~~3=ZP!Z*I)$N(=oV+D0!8v1KKHZMk-cLia-E_ z$8I1Z43kr*Zuhp-D7S-S$1%3G-^c6MLzfVW80`zQ&54kRA+Tp4y=pMr>#dZzS3!VE z)lHN;(Bv>TVMLd@52WnxJ_A={ zaEUy<%QQ7&C3URu6zFnwTQxl`d<}{g7u5^cxz?Lz0&l=k!uxezo8-wOFPzPxyqLTL za-(Oc>QA2?IH8z4N4ls+9sjtK?>1jwSlt3Y!1iPeh`hjqflDmyCM|gh>9!B~9dgtg zuIJf+wlyzrqb;yJAQvOpV}eC4Zg&US!{A8_cv+9b?Yvr=5#ly71iw+MEV-s54l1rw z3sE{^yIvnUlw*oIZ4xwpz@ow@(W4lj$7kx(_W#xPcWHFlHpXF*8}lGGlnJ>Avshd48Yw z_xU}~`+47Y{`KK|&Go&m^E}SuJkA3IFos)!VZyJ^k?ou&jaMr-EqW|>2P!ndKJa3p zdodrqF1H8PIa=dy`bX78-G|{4+7hjP>)KbduU^=@X3FiliAVf-5u~mb=)X_Xkf?Sq zSb2g#m9XywxWo^O`)lTw2lM1nZ6fs!23XtI$D#f*FI#u6)3K1Qs?vb3Rb2(v71Tb> zf&;kovTM0J%za};=!e6jYvq6MEi(v({0vDHBzVD)k@OsrRg(1(ia~%-nC^0iE?Nn@ht%XRI{*=*2b9 zkx1QZ3msQyn%d2-)pRe>mx42V2RMPmA2%_cxXvBax_5J=zQfgHw$SK`mx^ns=^^7T zo=WK>nXba8r7xBndj}J~>uE&>*5RsiN-a<{9q(=If!*#f?@a$oHEFrA{$A-0YXdT6 z-KYdKp6)+SC#ZzO*((!>3t1*Z!E04-)a)u>O07ksdB|#~ISw~BsKvc&)CAoU<^s0m zIk?D^fAt$amLN*xBAUybdFnOWfKLRcVO4+0tZy%?xQ8cRY#d5P=zT^jfEQ!~j(7_g z$j~SEjvBh~haPR=S=!jRg21S+q;TSt-v%{P6r79>Z|2XL%&a>{N)P%hPW^p%c-xK4 z1wD{oYfRrNoz%56+9^|_NK-j=Q2!#31?DMxfB7~1yz?1%i+2B)AQ!EBY&_y?&j>RN zB6kiZv%s-X{euImcK%br@E`Q-{(Ki({uzcZk)n18J+vpHH$tVL!G@9A+C4$v!{9L& z04V(XN62$hLc%U7z4el%+12+%unc<(8=Jx-$B)Zs`5DSEsUCLDV8y`nBtg&bH=rqg zxG5WlGvgqjVBPbbuK$`>zA020Im=um^U-Ruxfq zuawXNXTnl+vv>E_AlO~9)dM=L;2SZlWIx5K4fW~k)b+xE=69>CJz&EnBVrH2R<_g} zR8b{Q5=2j)IAzgn#QT}nT2gnFSAf3Pp{LhJA)Lfj_h2Rp!BY+|I)&y(R|hPef~oTC z#c+a<%X!;crYucRlMB60QM&x2)^)B?ckvP3UAZrRq3wFe*n=p@Bvxa3&}5XW+MaQY zDG9l%;h@nz;6iNg8q47Xdr&SdYnjK@<}lZzwY}q6D{K4eh>*FwyFW7xnX9}tctkR8 zCXSfjY@t}|_QahvoDFk9TzaXr+O(>@_6!CZ(){z7;qW4m>#iI*-lK%M!C-q@t_sF_ zzmOZXdex1tC!L4Z!WD+E%@Q+ft+Cp>n>Gr*F5pQHNo>8O70p;;j6xk;u!(Mc_aJZX zWT}%J2vNp~OouCaiA|QvT#rDYf_Cw+^)JFliSixt$4#2+;Nzo^K05%vwW5GxMl(nd z9S_tkWde(iio)}Wrti$kTV0uhM5MFXEK$0=Fu(j#j*3gryEa@QMN$%eL2F%eZmADL z>{UufDabSH#vJt6yvg)fU$fv3GP_8NPRy!_WKOgov%;L`8 zv)6MpdWrAP|7~|UX8h>AlDsckSE}$&Oz#t~?WSK*xjb>IoJydye0d6|@DSr^kT?AU z4*5~lws#Y7y|s2V%-pgo5D1Vs?K9{5G~)cma3eFk*bXup`~w zPDkTVMbTXXjQuqz>B>VQj}5o6gqw3$8z-cfMh%W{^5Cuo?G zm!ixjzh1sq42_KCf@nO`PY_cO-F-Q9!=*Afz?k;xO;g6(%$1=8+YJoSwE!PAbPomO zSngdD)@DJ$K(>4j(=SEDH}PE+MH18HpOPVd_Zl`j=_+$)hZ+p{Zcf$Xa2yy5j+ z%aoP+20I4rYlm%LfMtbX%wHR6^dO6d*znF@Rbo)jwBq4G0Z*mvy{&H(;eudm3Q}@* zmNBR1G-&x;{Dq<^dn8B0I9sHQGm>Efx1sIuf5OUf zvHiK%T%TQxIdB&Y>**^EqHmF7rF9d|*hqqiwEyr=;7Idr2PqwV{)3nb;t}}#(*x+g zL)+OJ=eL(l3~s4@kNLNXZ>A{O6qbx$wwzi+u9kIQMXpoV;%QJd)PM${{?GlpdVTQFtzvcST|&UPOo$p<-z}-{LLHgk@v6P0`+L4)Uv-LOSBZ0D8Mfp)RB{I4 zaxUlN_}w%cEBr`(B;%P{dyn4PmF!q)sC(8xspk?*b-r?)^<7=@l*z^lIKfDT;bw|m zh|yF-#?gQ&o6J8FQ~ZZLc;(a*>*b*z~9j^;x_0UjPxKk$v()4G$BgG*bu zX*FjtgjAo5wfnTxNtnO>UDoC0^-Z_g}+$J`5oGa{!w7MOLNPupI<@mJ8yZrVoJXz(t=5>6P$ zQDP)~GTfnuf9&k!VQz2fz3E9H#ioV^`qQC=hx>;R&WVRtgiaYUav~6iib;~f>t1MJ$qu(s}lNf_0JKKGbqV)OF06j-$ z&>3D+bRlKH419xDU{NS)T?c!12~m4LypieS<4EDOU6Pyhyc2B3efaWL%h2`=FH5cX zMj4ca33=W!L-)v%xIAhA89c#=O~RIa)JblwfP>(v4pQNt&bbLCmObt4yrZCM@jJ7IeewL4q_XnJt@5qB9DefvR0 zw=CALb36Gz<|{)UCzKmmwxfDeDyj{7B{CufE9I1Jk&njELlR^Nk|AyM(;-IT{uzU# z^prDsq|B$75wK?6(*>3N8CB52QKL9{;Bi3F0o&RRyn+VvS_y%Y-5}Fk0T4icGwb;e zubfU{ys$w%&1Y6P>GRr`^@23mL>O!%l!2rl#97M`?;AXul(9ycCJlr^gCt%KBqIm& z2r%c!W9{aTd3oj>H+d|$QdmT`DQD%EkLVc!{kX;g>ldmW-R;#szqXQz1s-4|h>785Ro$VQx$xw!1Ml0X8Y^aQ{Ok23MSiK;?=*`|8 zev^|4U;K+aw+M0dKuMv0aT^Vc#{U+{`C=v&Jm(mdAZ#jnapT7Z-&Ajf%Er{S8ZUfJ z^+915SB7=-VZ@UT=bUmR-lKELt7wvwUDqsB>?v0r9Z-0`_>0|3WQCOlyRQy;w&-Fj zB_SVL?w(&Zwhm^%XhoJ@bJaRtv@=>G!GhVYYC~~6(qTg2;B%8e4F!xJVumy@LqCu# z5NB#O+?G?_Qd}=3rCoZc`z5tn;gtN1#vFsd*6tM4@rpK)zWt(Gy&Z69|HtK>mGhKA zRn~2oDF;+p0s?txhd9%- zkn+iJSxrMXUuJ00lIAWwyb+^2v>>UtOxajES!15!6I6A8;jHl4m`4o6RZ|)5OD{dY z1m%*Pdp&w#CPOzruu*v9ZVa%j^7?uS3>H`>rm3EIutmt{oq$=Nid;IQ;kNSjn_9fM zr0x(?!Pap}O0jm4^~LZ`jEp7^!o1?s9#EH-b{V47Q0zG}u-C)06+3u~0)90P#518q z1L*qV@r>&}1xjmU7l^W(%p0y;ze#fC2E^=(Csa#8J)@7~NFmRQ*{(4%g&uo+)aC|J zjM-!xU_F8ZQh$Dn=SYFCJ;lclsakrheXCG5`4-VfXulKD=i8VTe8r8M6{nL6C9gxM z2`?jMHtx|@V+hluae72W7#G7%9rzg1+_ERMW^rHX@8Fp2cLDBYhqgfo2qd~=eQ=Ef6%8gG58oO^uKw*2fc9$_TFpfbGWUMynm zlP}GWfg{R7)a-d9yElsWqH8V^*48gj(`+0U>k#Z;_?&0;P5$c?56;yYX-uEc!_ZH{ zj16wlPPpggbuneZ%*)@wG~b)gvi^Fwe8*KxU)|cfSQ#h=Tq9(G*8;42LZPIkg;pP# zQP90$G<8p0FSYD;#wB&0v{tQovmOnjP#+230bp&rMX}l2b-2NN6XXwU{JzMa`{eIw zD*nk)pP%^}u*XiC_xMgd=fS$=MA>fcH)y=C$M>1!EAzGq-8wDM%l`5cjZVe><;FY!AC4i67Md|T{ZVBTo| zhlCSc)BXE_QyO^Wq5c0Kq^19MaS)-R7teI?U8Lij+dASc>h4h8j(E zl;{BT1q8l8-hBuq4qJ-fDV?}JkstpDOD)-^mrKs{wXs{yDDWoldWS0!40-Cy+}x)t z7$m(Vbbp@K4Viqp8Ma^7)2>=j7lO}E>FTzm`rxqwIk;L}bK=H0q^M&Pu4?I_(p#)% z!#}*}HUrbUNC1Y;rM?X^$6}Zmh3j_2N)*+{Qzp)$AMRF2s=YAoosoZX;|AuWa@<>7 zov;0HRUmIvhWGXP*1`1SNqd(~;m;b3pin-4OxRU56OB{%t6?h&O7MmPoXkBJ~$b0V-g-`?s{sBZ^VZKE@# zaL8Hr(;(tQwp)_lpCszNQiMHMZjK=}F^ibCd6=WS=rXWmp z$5W*(50@u(vn$(;UvuFfzi}TV!2%As4}U^gJFeeD;l8Dk0%Iua%l=D)s9K}3q3_sF zG|cDYi{l=zd4AF=RA<<=KpXc{UCz)rH>aCxGMjSSdl$R$lnTj1~3+p zOBezR0K8btDr@Nl<&15!PcPPB*wmJJb)DO#d-2Eut4}c>`;!n0nyw!#mcvyuYHYiQ z@*vBp>XW|KSCb|IK~`rQB3;fKF0~DrO;+;d=?vyws8F*9v>r*&xlD8k3*{ED#S?}t@MyC#UGhcJaxfsk@$%4TpXUwTH&1V-gOif6AMC!PwUu0NZhqv9UyZ)k+ zYt?NHPzw95AG;#xEbul;qU#LHOk{p#NauK0p-opxar$pUhJ%#}@bio^_q%nlHvQFu zW-=AV5C${&h>HxakG#q&MqAo_h80H_0qfn4Hy|5zq|97oOloJpd2R3tdaBAmuMAH0 za__uLr`L_qOkmWBQ!f3>Th-QbMqKul!CQ`~9jh&VkLJ^_F}`&8EZeI=&MUKKJ9bZK zTLnzL3zQsEt1EEPtSi#3***0Drmh@wn0{`!0*tQ5U^pnKJufzhcveECni+I?c(=@eWIT}+-*Ui z&G>9XS(nXBM@I~xT2H%Dy?*N#A#I+E2s4VXN{=ks*C-~|-ZQh!+jV7Ycq6CD3VKK9 z3j~4|dTOz5{>vIkkm>2yn1T_#vYFou${S-S_iOe!Q&__F??+Qnw=?ywkA8*}1*O3L zTGhGA`Xab&uOGG?xjVDNdR z-E3-@`M^Rzq<5nQ{xxePGf@97{q~EgoGD{NagpVh4`;;li4;~OjpSVuNuuS9Y!7g} zZ_|@^XTN=SRA2u=*P7{8*+Op-tO zH;CKKI*~S*L2Z4haxC@+bVkG7f5FspHE9mC5OAxAI(qc=DUb+IfSUQ`lSM&Bo@Tp@ z{7~MT`lDt2bfdLciT)#MSD;RfWOBc$5>b5!(asjXYv5&@8zDy}yx>~3v{0rw{I>Mh z8P5m5Gc`6UXZTHsW>>`!ETfj-Dv8@xb29B^cFO~?RNk6!X?P9#j?YHwudTusXxGZB zKmp2LI@nUbjoE_=LA@w%B;m^lMG|d{*Q^S9V5<`jg%8=P-KixUG<-ns8m0JK#pO6K zrwSkb(c&Ic*Go(vAThFfoZAshJI*;^g~3?N1`qS%O=y{k$PaLAIoB+*CTx&%yI}xb zmH*FKQ!dI@==)0{hdG=0pA;CLm5yWKoSq7VvDtBJjI4P?{i@L{AfvwA)o}1J#1bc| z<$^W=Xp!OofkL=<(CQ!khIK;LOvmSfF~`GApmnj|$wu=zDZUJha7WCd+S6Z2ZrL!zRU(71(pXmTeN!%ohYJ!Wbh&?qGVO|7%NK zg0Y9o!3ydPEqv4C5nJl%7JQcI^~q{o$COsaYd!5dG^ewEt($rY6_YF-5+&3=v85on z({>TiY@@6c=b5aYvw}KE(0aZi@a+}82J$GUR6#f$^JZAul3!-oDafNopQL3^hV{LR z-f758uVl|OZxj=DAMB>!=LR=I+Z<}*mY=+`%Tx7eATYv658wfgZAV=~b|F@S8-&*` znX4mw=3{qWef#;|gk&YDWIrrugnpg^s6va2bVUYmZ`h?0|A`R?&^FN@8V4S~0s9)Vf&} z;*|C*HB`zJ+4I!gl1kjeb4V@fT&6AMf8DaFh27pV(k`HAt){ zCtUs=)wH3;*t(K(EZFh*CvCl>z2ld7r}<~TuIB5yyht9wAlpXLjg5qlk+-?Y9E}e` zGLpM`tC7>YXtj$@aH~Ws8V81R^8(F()AWT(?3*TkVBO+?I~zC17Z{=jnxs^sq~Tq; z)cr~T@i_|WeXD03SBK9Syc1_cJzWiMfd!`e*!$RIE&LAWx$+5xQEu%KMMQq8{Ek*X zan$kd1VTA8Pz$TQXi)K(n)hah5pWC5Bz#as+fh$L3>_s(y+R{Qh2q@VbLB36@QYJ+ zESZDro(dh)bY*%oBdbFjXM58-MeUldQFkN!Mr&|Q=92*!8OXKRK3{_^Q?S4+<%YSEbsA`4iB|Y1L`hGpsW48 zBEn>AW3_zcK;^dZMoX8yXFJh%E*DPn%9DJgQNLn)y^@ z6haKD2&DFgwde+RmYTP=<{!}bfCoTU_Z&bM4r&H3>K?jmp15l+KD8P7#ka@Mk@eb8 zue?5A&8l-mSo63|+T^`1O|bJqQ^DLKNuJarlvM$a9Nj<=hk~N!aYusFc+xu8BY_5c+{xE`=o>Z z>14W~^i#t0=;n&Ke@148jKyZs88z2k2gsM(-ugJNfS%JpTRtI%Jq532_k?n0`W>cRG+9FcqQ z=hn(K(085;b!SYiscO1h@R${yWpSR0vo^)W>kI72pEexOb4^GK#CD1dnVCJsH08^e z^V;|~dh+gaK{63{xY!Z zyAFDvPh#fIxQ^RAsV=sr(<#;TGKo8+d(_sk+aC46J9qt>`dERidw&VPy3o6!#XZAe zm|d%7OEC4ETS`aUbHF^--LmAmUIg8`EM|%28(SJ2y$rdUYe> z`=6$0xZf@}oJ?6#Va!{~Ajlk|$4(EXKTV%2L6PPWDYD;+h{eUPC<$_4FdQu6;Sg){UO)z^`D zuOO}P4%Cf@D7~oC-a5#spMt2x_C3XM+ literal 0 HcmV?d00001 diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index b0604ed3cf..31854e2729 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -140,3 +140,42 @@ Beware that ssh key expects OpenSSH format (`.pem`) not a Putty format (`.ppk`)! If a studio needs to use other services for cloud storage, or want to implement totally different storage providers, they can do so by writing their own provider plugin. We're working on a developer documentation, however, for now we recommend looking at `abstract_provider.py`and `gdrive.py` inside `openpype/modules/sync_server/providers` and using it as a template. +### Running Site Sync in background + +Site Sync server synchronizes new published files from artist machine into configured remote location by default. + +There might be a use case where you need to synchronize between "non-artist" sites, for example between studio site and cloud. In this case +you need to run Site Sync as a background process from a command line (via service etc) 24/7. + +To configure all sites where all published files should be synced eventually you need to configure `project_settings/global/sync_server/config/always_accessible_on` property in Settins (per project) first. + +![Set another non artist remote site](assets/site_sync_always_on.png) + +This is an example of: +- Site Sync is enabled for a project +- default active and remote sites are set to `studio` - eg. standard process: everyone is working in a studio, publishing to shared location etc. +- (but this also allows any of the artists to work remotely, they would change their active site in their own Local Settings to `local` and configure local root. + This would result in everything artist publishes is saved first onto his local folder AND synchronized to `studio` site eventually.) +- everything exported must also be eventually uploaded to `sftp` site + +This eventual synchronization between `studio` and `sftp` sites must be physically handled by background process. + +As current implementation relies heavily on Settings and Local Settings, background process for a specific site ('studio' for example) must be configured via Tray first to `syncserver` command to work. + +To do this: + +- run OP `Tray` with environment variable SITE_SYNC_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) +- start `Tray` +- check `Local ID` in information dialog after clicking on version number in the Tray +- open `Local Settings` in the `Tray` +- configure for each project necessary active site and remote site +- close `Tray` +- run OP from a command line with `syncserver` and `--active_site` arguments + + +This is an example how to trigger background synching process where active (source) site is `studio`. +(It is expected that OP is installed on a machine, `openpype_console` is on PATH. If not, add full path to executable. +) +```shell +openpype_console syncserver --active_site studio +``` \ No newline at end of file From 6f3944d01aa54d3f15c0d88ce7f0c8d617f96f50 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:48:25 +0200 Subject: [PATCH 09/57] Hound --- .../default_modules/sync_server/sync_server_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index d8a69b3b07..82c1dc178a 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -727,9 +727,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): self.enabled = False except KeyError: log.info(( - "There are not set presets for SyncServer OR " - "Credentials provided are invalid, " - "no syncing possible"). + "There are not set presets for SyncServer OR " + "Credentials provided are invalid, " + "no syncing possible"). format(str(self.sync_project_settings)), exc_info=True) self.enabled = False From de8cfeff7f996ee2d33ca0b235ea3c60ccc66ebb Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 14:52:12 +0200 Subject: [PATCH 10/57] OP-1920 - renamed reset site method --- .../sync_server/sync_server_module.py | 28 +++++++++---------- .../sync_server/tray/widgets.py | 4 +-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 82c1dc178a..1fee0b4676 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -146,9 +146,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if not site_name: site_name = self.DEFAULT_SITE - self.reset_provider_for_file(collection, - representation_id, - site_name=site_name, force=force) + self.reset_site_on_representation(collection, + representation_id, + site_name=site_name, force=force) # public facing API def remove_site(self, collection, representation_id, site_name, @@ -170,10 +170,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if not self.get_sync_project_setting(collection): raise ValueError("Project not configured") - self.reset_provider_for_file(collection, - representation_id, - site_name=site_name, - remove=True) + self.reset_site_on_representation(collection, + representation_id, + site_name=site_name, + remove=True) if remove_local_files: self._remove_local_file(collection, representation_id, site_name) @@ -209,8 +209,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): """ log.info("Pausing SyncServer for {}".format(representation_id)) self._paused_representations.add(representation_id) - self.reset_provider_for_file(collection, representation_id, - site_name=site_name, pause=True) + self.reset_site_on_representation(collection, representation_id, + site_name=site_name, pause=True) def unpause_representation(self, collection, representation_id, site_name): """ @@ -229,8 +229,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): except KeyError: pass # self.paused_representations is not persistent - self.reset_provider_for_file(collection, representation_id, - site_name=site_name, pause=False) + self.reset_site_on_representation(collection, representation_id, + site_name=site_name, pause=False) def is_representation_paused(self, representation_id, check_parents=False, project_name=None): @@ -1240,9 +1240,9 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return -1, None - def reset_provider_for_file(self, collection, representation_id, - side=None, file_id=None, site_name=None, - remove=False, pause=None, force=False): + def reset_site_on_representation(self, collection, representation_id, + side=None, file_id=None, site_name=None, + remove=False, pause=None, force=False): """ Reset information about synchronization for particular 'file_id' and provider. diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py index 45537c1c2e..b401411db5 100644 --- a/openpype/modules/default_modules/sync_server/tray/widgets.py +++ b/openpype/modules/default_modules/sync_server/tray/widgets.py @@ -411,7 +411,7 @@ class _SyncRepresentationWidget(QtWidgets.QWidget): format(check_progress)) continue - self.sync_server.reset_provider_for_file( + self.sync_server.reset_site_on_representation( self.model.project, representation_id, site_name=site_name, @@ -786,7 +786,7 @@ class SyncRepresentationDetailWidget(_SyncRepresentationWidget): format(check_progress)) continue - self.sync_server.reset_provider_for_file( + self.sync_server.reset_site_on_representation( self.model.project, self.representation_id, site_name=site_name, From 1772e7bf8d887c059d28ca923448e5fd925285b8 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 29 Oct 2021 17:15:47 +0200 Subject: [PATCH 11/57] OP-1905 - implemented exit on key interrupt --- .../modules/default_modules/sync_server/sync_server.py | 1 + .../default_modules/sync_server/sync_server_module.py | 1 + openpype/pype_commands.py | 10 +++++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 2227ec9366..48df5aad1b 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -246,6 +246,7 @@ class SyncServerThread(threading.Thread): asyncio.ensure_future(self.check_shutdown(), loop=self.loop) asyncio.ensure_future(self.sync_loop(), loop=self.loop) + log.info("Sync Server Started") self.loop.run_forever() except Exception: log.warning( diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 1fee0b4676..281491eedf 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -771,6 +771,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): log.info("Stopping sync server server") self.sync_server_thread.is_running = False self.sync_server_thread.stop() + log.info("Sync server stopped") except Exception: log.warning( "Error has happened during Killing sync server", diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index cfa16012d0..e160db0f15 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -3,7 +3,6 @@ import os import sys import json -from datetime import datetime import time from openpype.lib import PypeLogger @@ -332,8 +331,17 @@ class PypeCommands: def syncserver(self, active_site): """Start running sync_server in background.""" + import signal os.environ["SITE_SYNC_LOCAL_ID"] = active_site + def signal_handler(sig, frame): + print("You pressed Ctrl+C. Process ended.") + sync_server_module.server_exit() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + from openpype.modules import ModulesManager manager = ModulesManager() From 8a37b9065e340be28a1a7c85a4be68ac8308e9a1 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 3 Nov 2021 16:22:20 +0100 Subject: [PATCH 12/57] OP-1937 - added alternative_sites to System Setting for a site --- openpype/settings/entities/dict_conditional.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/openpype/settings/entities/dict_conditional.py b/openpype/settings/entities/dict_conditional.py index 6f27760570..3621c1319a 100644 --- a/openpype/settings/entities/dict_conditional.py +++ b/openpype/settings/entities/dict_conditional.py @@ -762,6 +762,17 @@ class SyncServerProviders(DictConditionalEntity): enum_children = [] for provider_code, configurables in system_settings_schema.items(): + # any site could be exposed or vendorized by different site + # eg studio site content could be mapped on sftp site, single file + # accessible via 2 different protocols (sites) + configurables.append( + { + "type": "list", + "key": "alternative_sites", + "label": "Alternative sites", + "object_type": "text" + } + ) label = provider_code_to_label.get(provider_code) or provider_code enum_children.append({ From 26b9bbaedae7e21b6f4346b403160a495eb53d83 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:03:57 +0100 Subject: [PATCH 13/57] OP-1937 - added alternative_sites integrate_new --- openpype/plugins/publish/integrate_new.py | 68 +++++++++++++++-------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index fe780480c2..2f90cd7d66 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1029,31 +1029,25 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] - sync_server_presets = None + alternate_sites = set() + system_sync_server_presets = instance.context.data["system_settings"]\ + ["modules"]\ + ["sync_server"] + log.debug("system_sett:: {}".format(system_sync_server_presets)) + if system_sync_server_presets["enabled"]: + sync_project_presets = (instance.context.data["project_settings"] + ["global"] + ["sync_server"]) - if (instance.context.data["system_settings"] - ["modules"] - ["sync_server"] - ["enabled"]): - sync_server_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) + if sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) - local_site_id = openpype.api.get_local_site_id() - if sync_server_presets["enabled"]: - local_site = sync_server_presets["config"].\ - get("active_site", "studio").strip() - always_accesible = sync_server_presets["config"].\ - get("always_accessible_on", []) - if local_site == 'local': - local_site = local_site_id - - remote_site = sync_server_presets["config"].get("remote_site") - if remote_site == local_site: - remote_site = None - - if remote_site == 'local': - remote_site = local_site_id + sites = system_sync_server_presets.get("sites", {}) + log.debug("sites:: {}".format(sites)) + for site_name, site_info in sites.items(): + for added_site in [local_site, remote_site]: + if added_site in site_info.get("alternative_sites",[]): + alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1068,21 +1062,49 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sites: rec["sites"] = sites else: + already_attached_sites = set() meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] + already_attached_sites.add(meta["name"]) if remote_site: meta = {"name": remote_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) # add skeleton for site where it should be always synced to for always_on_site in always_accesible: if always_on_site not in [local_site, remote_site]: meta = {"name": always_on_site.strip()} rec["sites"].append(meta) + already_attached_sites.add(meta["name"]) + + log.debug("alternate_sites:: {}".format(alternate_sites)) + for alt_site in alternate_sites: + if alt_site not in already_attached_sites: + meta = {"name": local_site, "created_dt": datetime.now()} + rec["sites"].append(meta) return rec + def _get_sites(self, sync_project_presets): + local_site_id = openpype.api.get_local_site_id() + local_site = sync_project_presets["config"]. \ + get("active_site", "studio").strip() + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + if local_site == 'local': + local_site = local_site_id + + remote_site = sync_project_presets["config"].get("remote_site") + if remote_site == local_site: + remote_site = None + + if remote_site == 'local': + remote_site = local_site_id + + return local_site, remote_site + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 691faaf70d7abbd191d5ce7c22d188256c577066 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:20 +0100 Subject: [PATCH 14/57] OP-1937 - added alternative_sites to upload/download --- .../sync_server/sync_server.py | 8 +++ .../sync_server/sync_server_module.py | 53 ++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 66d4e46db7..6eaede048c 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -80,6 +80,10 @@ async def upload(module, collection, file, representation, provider_name, remote_site_name, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id @@ -131,6 +135,10 @@ async def download(module, collection, file, representation, provider_name, local_site, True ) + + module.handle_alternate_site(collection, representation, remote_site_name, + file["_id"]) + return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index a2cfd6f6b9..af672e7a6f 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -109,6 +109,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): # some parts of code need to run sequentially, not in async self.lock = None + self._sync_system_settings = None # settings for all enabled projects for sync self._sync_project_settings = None self.sync_server_thread = None # asyncio requires new thread @@ -769,6 +770,38 @@ class SyncServerModule(OpenPypeModule, ITrayModule): enabled_projects.append(project_name) return enabled_projects + + def handle_alternate_site(self, collection, representation, processed_site, + file_id): + """ + For special use cases where one site vendors another. + + Current use case is sftp site vendoring (exposing) same data as + regular site (studio). Each site is accessible for different + audience. 'studio' for artists in a studio, 'sftp' for externals. + + Change of file status on one site actually means same change on + 'alternate' site. (eg. artists publish to 'studio', 'sftp' is using + same location >> file is accesible on 'sftp' site right away. + + Args: + collection (str): name of project + representation (dict) + processed_site (str): real site_name of published/uploaded file + file_id (ObjectId): DB id of file handled + """ + sites = self.sync_system_settings.get("sites", {}) + for site_name, site_info in sites.items(): + if processed_site in site_info.get("alternative_sites", []): + query = { + "_id": representation["_id"] + } + elem = {"name": "sftp", "created_dt": datetime.now()} + self.log.debug("Adding alternate {} to {}".format( + site_name, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): @@ -919,6 +952,14 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return self._connection + @property + def sync_system_settings(self): + if self._sync_system_settings is None: + self._sync_system_settings = get_system_settings()["modules"].\ + get("sync_server") + + return self._sync_system_settings + @property def sync_project_settings(self): if self._sync_project_settings is None: @@ -1004,9 +1045,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): (dict): {'studio': {'provider':'local_drive'...}, 'MY_LOCAL': {'provider':....}} """ - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") - + sync_sett = self.sync_system_settings project_enabled = True if project_name: project_enabled = project_name in self.get_enabled_projects() @@ -1064,8 +1103,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): if provider: return provider - sys_sett = get_system_settings() - sync_sett = sys_sett["modules"].get("sync_server") + sync_sett = self.sync_system_settings for site, detail in sync_sett.get("sites", {}).items(): sites[site] = detail.get("provider") @@ -1434,9 +1472,12 @@ class SyncServerModule(OpenPypeModule, ITrayModule): update = { "$set": {"files.$[f].sites.$[s]": elem} } + if not isinstance(file_id, ObjectId): + file_id = ObjectId(file_id) + arr_filter = [ {'s.name': site_name}, - {'f._id': ObjectId(file_id)} + {'f._id': file_id} ] self._update_site(collection, query, update, arr_filter) From f19b39185d47b4666a8065bb155823dbcaefd4dd Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 13:04:54 +0100 Subject: [PATCH 15/57] OP-1937 - fixes to stabilize sftp provider if wrong settings --- .../sync_server/providers/sftp.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 07450265e2..8549c1c981 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,8 +1,7 @@ import os import os.path import time -import sys -import six +import paramiko import threading import platform @@ -37,7 +36,6 @@ class SFTPHandler(AbstractProvider): def __init__(self, project_name, site_name, tree=None, presets=None): self.presets = None - self.active = False self.project_name = project_name self.site_name = site_name self.root = None @@ -64,7 +62,6 @@ class SFTPHandler(AbstractProvider): self.sftp_key_pass = provider_presets["sftp_key_pass"] self._tree = None - self.active = True @property def conn(self): @@ -80,7 +77,9 @@ class SFTPHandler(AbstractProvider): Returns: (boolean) """ - return self.conn is not None + return self.presets.get(self.CODE) and \ + self.presets[self.CODE].get("sftp_host") and \ + self.conn is not None @classmethod def get_system_settings_schema(cls): @@ -108,7 +107,7 @@ class SFTPHandler(AbstractProvider): editable = [ # credentials could be overriden on Project or User level { - 'key': "sftp_server", + 'key': "sftp_host", 'label': "SFTP host name", 'type': 'text' }, @@ -421,7 +420,10 @@ class SFTPHandler(AbstractProvider): if self.sftp_key_pass: conn_params['private_key_pass'] = self.sftp_key_pass - return pysftp.Connection(**conn_params) + try: + return pysftp.Connection(**conn_params) + except paramiko.ssh_exception.SSHException: + log.warning("Couldn't connect", exc_info=True) def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From ab36fdfe711400059e103e6475c43f60a6de754c Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:00:03 +0100 Subject: [PATCH 16/57] OP-1937 - added documentation --- .../docs/assets/site_sync_system_sites.png | Bin 0 -> 13118 bytes website/docs/module_site_sync.md | 47 ++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 website/docs/assets/site_sync_system_sites.png diff --git a/website/docs/assets/site_sync_system_sites.png b/website/docs/assets/site_sync_system_sites.png new file mode 100644 index 0000000000000000000000000000000000000000..e9f895c74347ac8f311e81ad3f1dd6a388723695 GIT binary patch literal 13118 zcma*OcU)6V_b&>fB2p9)QK}#yf`HO{5vBJk9Yl~08fxgGpi~i2>Ai*o1VSKmP>|jU z20{d+210Lvz}@kA?>+Z-@AG-j`9p`xcZa}e7Uvo@&$qU$moZ1 zAH#c&Q%KO#8GqhnP)N@ntgm&?xNS5IC=~DRn4dLGpk>!vBPcus1qEF_Jxv`P z@_9|P4V3U7ZhirYU=4Bv@9&rrYIh3PBr*f z&{FE?=t!^{hvM~4Ds#cl~NyTJB;1#?u(1ypd z&XW4!PozkV9X%bXdHRNe)RNegBLa$?SLEgVzMST}d^JVxki%sJ&kb z_evkRSiAbUmwFdvmaylyMPm(2yq)r0ynOk&QEXb%!X{qDRR*jHy{lKcK1K!4l;g}) zSP?8P5G@{mMTiR}3`bjR4F}=1NkW!#!Mumi=(5qf%;V^h#q6XNqGH2y^;*C_ z`RR7GN&~!jnELO+3{`_?bD-lw+vw9C&7F3oG@*XBoEARe;qM>qL)9y_iqKUe<8v3H zC3QJA_3%H@()1PEFe~Kfx_T5mjbY#1#U({^wd=y;;4bMI6Q7QO!ZJBq6fH@Ox>mC} zg-{(q7Y~~bEcHe6nR_#aQa(BOlAHNw@_QEX>}ej6%D1_{hF2D^f0C8GIO^Q&lAbj= zJB}O2PSmQ1n%QRk3D>7TX=Kw=r*1|XYk0g!?>x~N5!gQt>`j2cn>u{#&S?kCw_x_a z9$8>(ZFaiqJVcLv*;N)25t{8OHQTpYlP4;n&Dm{T$m4bijlqm@to-Q z!mK6zQIQhDtdn-X1vkRL#yerPPe5fe6S+mEOmtNz(wDOG;v)9FjAbEkaEEo;OmnWB zQ<{^h`>5ZPoxguSDlJ#+v@+wJ%>~)-MMQ5l|ACPq{j7l?uEre0)FSEG_uqG2jni!2 zg6g9l7!?PRKaXf;US}SVK?P6ffGt>Re!r@4AIR{VD%#)7dZXLy_LP{S^(88EDGb{x zO^;;(gk*vrwkumeEJV=-%{C8~-OfEq)2)@jt`jbx>b)(l}_KCKMFoK*Xr2 zy|3*XC2Es7{{D3)qYq9KcFWm5I&_i0W$ye^G11rhp}?$Q<2*~}jP!=C|AN1ZN?g{v zOs$suVgV;t^jAQ4t=3>9zt0%D3{WS<&Vs*dMz8pJ?eWW!R{T|*?CaCLO%Arh))BdS z0;V-n2@?xfaSdq%Cg*8^P~xB~oJk}Dp6J?EXLbT=8HK~B{u%(JtqjY5vgput))qAvWEzC5R=Q#o{F*WXt?YYeu)t&^OdW6JXtIVT<@%P{R?q@Jw&U|Qs1wTt~+VKKk zukFuR1k<8%o#Fv)bHDELw}bfx76=Tjx~>vURJI9M6{77c6i3$S;zEU_d-C8qkrD8|B|*&ICx2i@97*lwr`^WN{~G&$MpmG$MDuVDu1;btIOyY(5le%N|Th_nlKcwBY>3nHQhG>pb~H#Lw; zXUrel+=H)ZW~(3QxXXUkV0FC#YIg)qUAUm@r2^|DDl2{pY`(ng95J9 zXy&&%y1KV)CwlzGuAfLCg3dw_eysN$qN3 zLBF7o1CzL+_@h8VXUQ{@=S%e?#=oF8k@!6FPKIx1EjF+fYU z8+uRb6v!*cltixnzvJ4C{}S@hL+#IIfXo80udKg8P@$a^*2rSLN&mNUv67m~XS(VG z>UyYf^2kpqsf9jX@{BdLwPmIC8h7>B&0-NGX17BPEQ!1~|NQw=>XqGmkAa2;1YJz) zho^6lC;)GR=q_IAsf}MV_AA{`Rgd;0pc-`m-J#fhQZU{h9#z##AD$F!8CVxRav?}~-JGe|qg zHhvD19S`yX-hHqD@3jVte&S0hBmq;oj zD=$eDVM~+jgTjsEjIIban|EM}m~AFL9-Fx(PQ~ecD^fg>9N9iLHo?Vw{L`0eV&HlaDZKoX5x%IX?UtZv-#oO0)Zt);oy354Dymiw{A=M!7j*z8L zm`UEPMB(wcYX~Y}s+HZ=0qdkqD=`u^1?{+4!o$7@rm*vX(a<+ z4uj}i4_sP!SG7cAtd*6ZWj2JS&M(pUTw5~*gRp>Zx6L5VS;z&OdF;Me`H-o2$?9nf zN|;4Ea%NAd&Rzr6e@2<&c^cHRM!83rn@(k3Fk4D3F$}@3?{{x@J^bAeo3j; zIY&g`)+IpBI&b8Ia_6SciicHV>l`&FU5ej&r{h%pf%2mlJPjSx6MUdb9w~(Xz_XFYy z4aE@q`d|B>7ktye2-LeUVh|E!kiz8qh@VLzkIDfp=ZEvDVPlv}b$Kc@B)|5C z_Qap+MJjCr^IyCs6|d$axe|5`puz5$7!v1QooAB_Yi_VSBloj50in_td-cRB-)Dx>sfJ%hXy`Ii8NEwK=7(iUcnP6u%yYL?Q>Zb|9nHGd-Ws_Fs>iRc=IuTLR!GLAjB=RF#M|v-1fiEUbSk^_{8b{gH^OB%%E5gM@B0|utwq%w-NW2sgB~j6vn&*KY zciYhw$_@oc#q^f>-pLXtkAAm@#rB~GzhC;pA3>y;^x^zgG{tC1_L*cFlU32GCPJV# zx2(?LGrv$7H<^#hgWI2-JLk2;^S`G6+%qV*6nnWGkG4yol z3{g9AmUMkxMwSqAXLYh5rMkIzZ}EfQ$DJuK*304T!O<^tuHwXA1h)Iqsj2)DNgK-{ z*F=_n&(lI`+*b~W3$~J4$)}GQI~0bavdbrmREoAX_AfA>)I0V}&Gv)_K3yAj@9)D(8-xn-1j1)|? zU1(Pmn}r4q!lvAQ4+!lt#}oxm7@?mf;-4Zy6RQ;}blu7tvCp@!`-l5OlY8Ot9`S8z z!2g$HdtNk;jv^oe2$4=AEvyY&aDl^-t zH*>@3%wyMfZX8(deGOv*YDhDgC2#KqU&kJz_ZMvJRs6g)+V>~4>y>kLD3jFIXyj$Z z6XKWlM~@|Iyng$Xwi($LGDbAOg7(YWrcMKY!0|zm|79#nt^X ztJFxRI7Yh1^z=xS%6QYx`l>xG7JrDRL!hA8J3m9R>a(hF#xA`lZnU8&=a!TO4#f61 zU(W`}IV?OG%X_jjh#Zjd_%-=MX70lfwpJz6%75tS)E{t6;8>xk$ZN#X^bhXQr;Rn3 zlw;7IVmF$*HHS(`mu6+*36K6A$Ypv8#k$?;l3 zXnOS_%G_z7VOAz(qNP+)C%sJi1|2Xmcq5ltRRo8;4Pcg>&`o#qK8Q1S@JA7ONv3Z>tUY!jMoIHM1ae@`Y zX+bAdKZGYreZZ66U!{DA2k%Y;akeENdHct*bv{lsd&8Q@Gv9CjVmp<1VR=@hSW^=4 z*xq^Bhza-3EH8BGD|qa$Ff2UJ2ihtdISuXJ4OjO1m{%_SW?+;11ZmWoc4F;&^Cz%$ zll=`T%52~@S!Vri;HSr!(1VQ4-1ph9RO ziJR@r-#i~FgJNJD1Dyh3Quxe0WrmNExQ@Z#HV#uUalPnvFx?Ec&b;1sH=3*PalWFb zxW&%%lN3Y3ddRQRSiC}zWmWWnA8y*Rdk3Z6J^%F2&w=)c?h^({1gSvwM` zf8?9gdWS&3_eFvv)=oG`_n9-w1C}-POSTQm03RNQ2t{ysgeQbzv+{f%BJIxE=#(z zJ>C+g_`iaXn79P4^)!-Sh`n^0Q+hS|SJyfz6&_-{Ct1!nUHE0uWhjSKe$-4wUZkrG zoA*8WuJQj1dOm;g#1B%y98!L&^y=TDbIKT1w?kKgpW8??A<}bw+R>>9!l!5kRZA4m z@sPowpN;lc#?izN_83MSNPdoVagkXQ3~Ca^Sy-3^{a*W0@+8@8-;sC(m-g(P%*QHJ z>oy3b=lwEL)qvyR+mego%{+wm%oP6pE8DEYQzpVLM!(%C6Rt>A=_pR6+$=wIBYUck_ z#r_u>|Mp6g#p3Do;zVTBA}z=bBN}R7XqoH>4d{aUVmI4a9rt?mP2JK)(yZ<8rj3<# z7CEsgf>rP8iX#DZA3*c(G13wC`$yi{|B`t)GNrC(1X3&&?8P#TCpp2_WqH`VfO=JM zv{7leILx*%Q*$&zFQK%)ew>YDJ+}+n+g_;JkXeyv4=8?NBcl|_ za=hBnl#OhE`pP5}CuW^KF9DS&tc&7&O4fP_pz3_TCeza*)h{8Zewe3g9~_A6U>3vmU&9x*pLMq@WED~O(7I~$sC9PBjOw>Nu2_xI^b>kPxpKjS!9b;br| z3$7?`5A0n}-0$6qdOp>nrcg$E;eCcXTWEjqK3JGLARU$PW!>k}m23Ox6CxmoQXTa* zx*Ib@9mI`$ruD<4TJ67l=8E#=#CW)tswc@O0;!a0uQ zFHl}NZn`e55;fUEFLTrwnT|N3CCqVEv}HpE<0Qm6hSkJjFIuIl)Hu)u-cO@FKL0d@ z`Qsh)iR_VRXaxIvW@4HRY0uPv+7?-BuAF1LKrug-%IlaR=XT!MCsJfu)BP5)%BSkv ze6CtU=scKwZSpLf)mpXEKcc|&%!dNjm+bdJ`Xf`9Dh{(%JEB= zxX?h_SmF5k9EOh+5joX%b{SH`B{dXVzH9~Y%U;=<*OL*#kTBa!DL*u0hG2!eW=Zt# z-=;*d2ls!z8?H;_5gYp{I19R*5uo6OQVUvh+JYy^Un6OY%WAWj@dtnx9nV!ILp3Yb z4?SB_l1ppm$cHIjn@!wBw2bH#uly-ZSnZjBk9wh)Uh7@HOtV16k;U-pCr4L4r^=%p zDnE~{AhH-OEfxFrK9x&-ci&y@T3RPWo(pW#4uHMiIVx(?r*BNKN(pNogOR)z`7Mfz zR|0=RDQTCn!*>+sB^&d8LIdNM`YET;^|F}B0w4xBmDq?ozVOn6KJunBn@HW#mdh0qOWDJx2M7oZW@?$R=sL&v1y`no zaiOgFA$tQ@-6i$XFF6;!9xyyFD2CW#wT~s)bzJw_pH%Cz%H}5*F>&#v`g0nWJ>?Z^ z{2oj)8E#|EsK{8;a(_;sLq?_aj|aEjr0OR1!59Cniu^a1`5(COpI7At&3{9H(rV7% z{rRtFGJC+(UD-gS$u z1WT#xAxDi zwE6)BY;!`xv-mngo(mfgRnIfobhyhIW2o?>^DWW#E&eBeyyC~n%$S#59-M2URkl9{ zNQ7r`FSNHg=qW5r4AgGoy^wIJ3+>8E$5BCcVQL*Q=h|A?M1RsgkvYO)SFKYhQP|+4 zlCZTe2`f5(zZf{@6N~9}0Wu%yn>W*EMuKKDuc1Dzbl&yc`mAW$eF<)(92j}Y=(zF$*$yedUVz4&cDW=k5rb_yS#VxYXiYX zeR9`La;Md88j)>3L@zq_Gb*n2+2VsbnSQ^V==NPMU>=#gfuN*QIoiiS}v$ z86`bXQp?h9v@BhpgO069d@A#h=cDakU0nedNwb+dSv_3Y#$%_Y>ZGAp+O>vAte-L+ z{qx89e@o&1?cDgkn4u|vug^&!*u})WzFLwr;{ENvY=#>QA0y5;?2c%O1q_inK6k1E z9Ax+yup6%x=w&M~i}Tyk9xD#7jJ_BfnV#DRs1;m{VkD;$DdV#}z7kx{(4}{c23UzE zunno~qOOUH1?SCo>DK?Hq5@E9?R)q@znWUFv!5P&LKGKih6*&2 zO^oHxDFDxz7;iFeB`-Qk$(ciGaS4f4I4X2q6CmyNqmqk^joHLl`vDo*Z?6BOdh~SR zGesrFM$MLG?j%(Es&G5`hx2u0Qvs#FLv?X1EFi$@E`b#oY6IRzz%Q4+8_?GRF~+Nz|6HP@pj&u zxSPpd6BK)3$T}<+-4Ny6{HhDJ+re?)-_l;wbMzz6s3Syt6;gAUwPZ5YJpJG}>(gS5 z@D+4HO6u9gHI*0Mb@MhID|BqsM?w=CQnkCA+axnF=h?Tl2hq$_+wCylk49w>48Z*);IZ#q82~inlg6pB?oXo zz$ylk>BA&+BWv;dLG*D^9%%MuT0VZ$=Js02wTaUc08Fjl8{v|)&3$~KP;Gp3g0sOf z6|{3uqTua$9(2OK9^A&pVN*dLN^5N(GSV=kgFZn{?U#q$|K@+>CfQsX^?I`vaK|6x z0aF;~OF>d~7amRRb|#C%8Z(omb2*hK?AhFYPS{O}zzm2J2K*(PrUEfyjrNGu{>mIO zkY@)y8z`MV=9SokHRd8{`It8XV(GeHro50QAuyH?P&VY4B~vdOdh=&Bf%nRWeBS(# zH1nE7At!T2m~q!W!SYz*SS7ZK`c+K}(prAe3IYC%QS;$CVP2bCWcBHl5wtYs!EC?F zGKY5x?Ra`Ol)vitTktcN`jOIAu|2WUgKbQb26EI4wKo(wcPFhOQ5R9Nb~PKEl`65Y z;WzKq2G|vzpHrID&t21aV&8N%w4&pyB?wcS8u=kdtt@Jg^J%k2A?S;%YUQ0=Oh#na zT;}N2%Z%03kQyoc4zLrruF{3G&wfyN`b~%8MZ>IObs>~qp0+3{8>(z$cWJ=n_IwC5E0J-|nGY zW)rLO-ik)eV!h0fS_F0ipXh3AYg?Sj5&Sy=pZ0treylYE@L@wNR@Y~9<_IjiU#d?v zaW=&Gw6wIE;@SxbbTufdr0#i5jOrIY0RgSZs_MN+$KJ8YvE_j^D=yqXpEl~lw3e7% ztCHF4ti|Qwd{k}6znUduBV5R4@0OQ zvoP-p3xq9eI_s+f)_S|boBmYMn5X$(L2&kvj`pK!#D z`My+9!ftY+waW$_rGrp|_WWJ#u@?C=Vb`5gU@71V8#cNdp~vJ+2iT1=9F0QlgK zg9GUHLB0FhWF3B0O*9B^<#F60OTZA$Uh!~tm!*4ukP#{(ArV6CP1`O{oL*O^QhLMU z{T&vz7t;1pWXBM;ie__KPMEFKyub*=h)~8{4=_3$k4u?tm6QIdeH9P$t!OHvO6IK> zv=zZ6s4cX6@t^$&@?9#wh1~guN!vO!?B{za3v8@#&t=ji!!bGOl7=)EmXt64@29qT zs__FUC6$%BmJ$>HT%`f7M@T)`V1AHzeO-S`TYh&_JN{Mr@K~sE{@id9Vqv~amTq{} zv*J}Qi^3m4EvJ-#sYGX2<=i5nhmL3EPkP%CO$&fKvU-*NIUwxC`N6^$0B^Fb6@Yi+ zo4!W3oCa4nSF!yv$nhju zOz$+hyWjf?do4)>Fgs17jr~WF_kcZ2CNm_^5fLc+Es)CGpSHVyyfpbPhD5ZhR@cqc zm-EAz*~$`d1pp2?Rk5VzbrWeKO=;K)D38n^}sF&Zve_E)dc$Ue_+>v&VRC(+C$qlDFZ)(w zR;8PE0W~3{VW5!~)3`g-IQ3nweNwKpPwqo&pR7m+H4cZX*@?U|J#3xaha@LkPOxF2 zi%T!kL#3G6K)X0><}~N3|8SgXoE^o&!Rv})3z`F;SV_n@W1%h`py74S$>0z7QDIJH^GC2ON$|)TS9bqM>v*af=oic3%`$g2e-2vP9 zfKsJ*ZeFf^M&~E|@Wc-_G2S}1!1ALtoY!SqE1A=nqu{sWxUYj&oI2j9Q3OHS1O0p7 zM0>c#MNOu;h);Doq(O6iuWm2|ayZ>wu~d#c*JA&=u(%kP)nZvdq@^W%Nm2AE{FXZV z7W-9FucbWHyu#R|eq{Nh!9l!asBV0GJa`>$+W=&s$YNA9#e8+jnM!r@tLTX)LRnfm z^ZL~L{^Zwv4Hx_FJn2u4tP(jc@j43y6G|NoqObk<_!BwW*r{SO!^14)og<*E2il!9 z5YJLzpr?tOoV~p!RDVwXg6E{}U3tnPWmX!7Sy~^^Oys=JvqHC41UA9BFZx>-WBp7l zpZ^90$|y4**Xsu%Xq?fT&7$ac($)yIXXx$y>$Tr^J6o9_NXF)-rCcQs1ja zx!8IiGqZ`(9)IP$)9I&wxa5D8t~unfLu5AFetm;%T~VcKGozqX2JV3Em+~cz+oID; zRo@)ukr0mlwR#YI{mRC&KklqIL57GU_e zu$jUq2lt1{UWAp?R#f0j=jBs<@4y}P_(yV1ULr#m*3)W7Xi~dPPYbLN2#P1t%q0f= z@TJs3!ww7#CZZC1B7>&oi78dxvs5e1X}5&(jci6?$`JNN)To--E9lbJgJ$c6B3Oct zvA-EkhVuUjaUC?mVomQBos{dIPh|Qy(HZ@|=M3CtYk$H(7rrBGc7dTVGdA03&mD1p zhF9l4{$#7LA#wT5zO&nJxrWE}D%jMRd&twGJFqO*b$9QD%t=IGV$_!uzJ}Xs#}EwL zv5@sBXUL|<*;h#$>9W~TnZW06(Z1Q%W_hW1DO0K1F>pNGOx%8i8?gXH*Gz%oxUMf3 zv8f}1=Pv+v5rHJ``8^hN3_Zc=67Wj6@1d-jS6cJFxN9UJQA+j9Q9ab`pXVKT6h{e#3UQ!5h z9&XO|8g2dMsU9En5Z!ziGXCIY8jBOCL9;1mHnA%aJIpk*#e9?!$_9Gn!&MGcw=bd& zQPL%x+oJPLMfrOpUn#9I)lI>3ypohY3&(z@W?WzWX}_w)_>}|{v4aDOWCA$rLvBZ; z`d=4|#Ad4VfwYdq-&8y$RHdEe4*Fxq~c}Dvg0zziH-f~p) zE6Xzgh`?-K>IWKbqb^}(6}JkzrX?ZK(%d0vo^kR{M3KH*!Qtb|f6Kg*DDW{twcm6q!MockT zVRO1?yB-G%k=-6229f>FdoiQyZu4ludXc_!QNwZeqq)C3uF=k$PX0a*W12p2OsbQw zcmotyw!72%u)=F>;&peL_69lG?q%SPH&c^`SBnQ$H~7Y=<>q|0bG`WuZ29bwBv5A^ z&!*yYu8OX8$+b&a^&vA2M4OjTORI6>?)NM(^ftCocp8inu^(eku-WQ9%6yu_{`Z{P{G+ALFH zsE3u|WoXjn%d5s5IAEvWdrRVl(zh)x@4Jui#fwx|xm%MIQa5LQaN+Jn6n-ZGG6k^^ zelRxvqyb`E^2pzP0cxL|;HKT7DE7GI5ZS#Brjiq9sVQu>a*W7&&jK@YA*O>xw|2gh zlQEIalC?hZ>Z0Uu*4M4=v=*I7d2A1+m#M2(3~fKZheO$N;NJzW*jxGfbD3ghT@UW< z8gq%hBz*I&4JJ~X^kn0D`bVQ&vm}L!>DVo6EBMP1cA1|AE2#0LBS%fWNAVMKw@izV z?(^QEP^xGfjR`;h+vQ1Vg`O#gJb}33b}HG7*F-LsrYMe_G&I9Lf4*UM?PEFe4uB7k zDV-Yvt}JGC7O9d-01W7i%j{YU&)K0Kcou8=g!ZCp)#^+$F~r(;eB6^7$f1E97QUlt zk^-dl;y`9@Hu4Z~qNOiF3v7wcpf<;Yw*N$|Vvex#k@pLlq|!i%>p}S?b+RbNJ)vyq z2?r@fufdO-jZcp`V|CrLu0Nj@xkb*o?z@RhpRfIGUuPlAl5;V zpOGvab{jxrl%$qJ2>ZC=_Y29Ng1UKXhZTUlcungk5c;$onsZ^g9QZhS<+?lFAE%%pdH&*rXLJM_vL>qHgc~ z2Lq!KAe|ij&!0>DPuPoR&*jNHlPOZ%1r_7}_h%U4S?~V2Qu;p!{*Rwr{Et_^lTH;a Y)~G@38KWNnZe$uybRWYW*}eY10Psf-VgLXD literal 0 HcmV?d00001 diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index 31854e2729..4e3c8e4ed9 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -27,6 +27,38 @@ To use synchronization, *Site Sync* needs to be enabled globally in **OpenPype S ![Configure module](assets/site_sync_system.png) +### Sites + +By default there are two sites created for each OpenPype installation: +- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. +- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. + +Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. + +Many different sites can be created and configured on the system level, and some or all can be assigned to each project. + +Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). + +Sites could be configured differently per project basis. + +Each new site needs to be created first in `System Settings`. Most important feature of site is its Provider, select one from already prepared Providers. + +#### Alternative sites + +This attribute is meant for special use cases only. + +One of the use cases is sftp site vendoring (exposing) same data as regular site (studio). Each site is accessible for different audience. 'studio' for artists in a studio via shared disk, 'sftp' for externals via sftp server with mounted 'studio' drive. + +Change of file status on one site actually means same change on 'alternate' site occured too. (eg. artists publish to 'studio', 'sftp' is using +same location >> file is accessible on 'sftp' site right away, no need to sync it anyhow.) + +##### Example +![Configure module](assets/site_sync_system_sites.png) +Admin created new `sftp` site which is handled by `SFTP` provider. Somewhere in the studio SFTP server is deployed on a machine that has access to `studio` drive. + +Alternative sites work both way: +- everything published to `studio` is accessible on a `sftp` site too +- everything published to `sftp` (most probably via artist's local disk - artists publishes locally, representation is marked to be synced to `sftp`. Immediately after it is synced, it is marked to be available on `studio` too for artists in the studio to use.) ## Project Settings @@ -45,21 +77,6 @@ Artists can also override which site they use as active and remote if need be. ![Local overrides](assets/site_sync_local_setting.png) -## Sites - -By default there are two sites created for each OpenPype installation: -- **studio** - default site - usually a centralized mounted disk accessible to all artists. Studio site is used if Site Sync is disabled. -- **local** - each workstation or server running OpenPype Tray receives its own with unique site name. Workstation refers to itself as "local"however all other sites will see it under it's unique ID. - -Artists can explore their site ID by opening OpenPype Info tool by clicking on a version number in the tray app. - -Many different sites can be created and configured on the system level, and some or all can be assigned to each project. - -Each OpenPype Tray app works with two sites at one time. (Sites can be the same, and no synching is done in this setup). - -Sites could be configured differently per project basis. - - ## Providers Each site implements a so called `provider` which handles most common operations (list files, copy files etc.) and provides interface with a particular type of storage. (disk, gdrive, aws, etc.) From db02c03394f1d5d159152e0873059c6c7146b387 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 14:01:51 +0100 Subject: [PATCH 17/57] OP-1937 - fix broken import in Python2 Not imported exception in that case shouldnt happen, as sync process is not running in Python2. --- openpype/modules/default_modules/sync_server/providers/sftp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index 8549c1c981..267e23f8fb 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,7 +1,6 @@ import os import os.path import time -import paramiko import threading import platform @@ -13,6 +12,7 @@ log = Logger().get_logger("SyncServer") pysftp = None try: import pysftp + import paramiko except (ImportError, SyntaxError): pass From 1e1b9b0416b804e815785fdfa83e7acf38e8128b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:04 +0100 Subject: [PATCH 18/57] OP-1937 - added file id to DB --- .../modules/default_modules/sync_server/sync_server.py | 4 ++-- .../default_modules/sync_server/sync_server_module.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 6eaede048c..8518c4a301 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -82,7 +82,7 @@ async def upload(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id @@ -137,7 +137,7 @@ async def download(module, collection, file, representation, provider_name, ) module.handle_alternate_site(collection, representation, remote_site_name, - file["_id"]) + file["_id"], file_id) return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index af672e7a6f..9ff706d6dd 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -772,7 +772,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return enabled_projects def handle_alternate_site(self, collection, representation, processed_site, - file_id): + file_id, synced_file_id): """ For special use cases where one site vendors another. @@ -789,6 +789,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): representation (dict) processed_site (str): real site_name of published/uploaded file file_id (ObjectId): DB id of file handled + synced_file_id (str): id of the created file returned + by provider """ sites = self.sync_system_settings.get("sites", {}) for site_name, site_info in sites.items(): @@ -796,7 +798,10 @@ class SyncServerModule(OpenPypeModule, ITrayModule): query = { "_id": representation["_id"] } - elem = {"name": "sftp", "created_dt": datetime.now()} + elem = {"name": "sftp", + "created_dt": datetime.now(), + "id": synced_file_id} + self.log.debug("Adding alternate {} to {}".format( site_name, representation["_id"])) self._add_site(collection, query, From 8ba0e606711c0bc440dbaf43750bbfeb3b685e9f Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:19:23 +0100 Subject: [PATCH 19/57] OP-1937 - fix - integrate_new --- openpype/plugins/publish/integrate_new.py | 98 ++++++++++++++--------- 1 file changed, 62 insertions(+), 36 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index ab79d95fb7..c0a90ee78e 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,25 +1030,6 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] - alternate_sites = set() - system_sync_server_presets = instance.context.data["system_settings"]\ - ["modules"]\ - ["sync_server"] - log.debug("system_sett:: {}".format(system_sync_server_presets)) - if system_sync_server_presets["enabled"]: - sync_project_presets = (instance.context.data["project_settings"] - ["global"] - ["sync_server"]) - - if sync_project_presets["enabled"]: - local_site, remote_site = self._get_sites(sync_project_presets) - - sites = system_sync_server_presets.get("sites", {}) - log.debug("sites:: {}".format(sites)) - for site_name, site_info in sites.items(): - for added_site in [local_site, remote_site]: - if added_site in site_info.get("alternative_sites",[]): - alternate_sites.add(site_name) rec = { "_id": io.ObjectId(), @@ -1063,28 +1044,48 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): if sites: rec["sites"] = sites else: - already_attached_sites = set() + system_sync_server_presets = ( + instance.context.data["system_settings"] + ["modules"] + ["sync_server"]) + log.debug("system_sett:: {}".format(system_sync_server_presets)) + + if system_sync_server_presets["enabled"]: + sync_project_presets = ( + instance.context.data["project_settings"] + ["global"] + ["sync_server"]) + + if sync_project_presets and sync_project_presets["enabled"]: + local_site, remote_site = self._get_sites(sync_project_presets) + + always_accesible = sync_project_presets["config"]. \ + get("always_accessible_on", []) + + already_attached_sites = {} meta = {"name": local_site, "created_dt": datetime.now()} rec["sites"] = [meta] - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = meta["created_dt"] - if remote_site: + if sync_project_presets and sync_project_presets["enabled"]: + # add remote meta = {"name": remote_site.strip()} rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + already_attached_sites[meta["name"]] = None - # add skeleton for site where it should be always synced to - for always_on_site in always_accesible: - if always_on_site not in [local_site, remote_site]: - meta = {"name": always_on_site.strip()} - rec["sites"].append(meta) - already_attached_sites.add(meta["name"]) + # add skeleton for site where it should be always synced to + for always_on_site in always_accesible: + if always_on_site not in already_attached_sites.keys(): + meta = {"name": always_on_site.strip()} + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = None - log.debug("alternate_sites:: {}".format(alternate_sites)) - for alt_site in alternate_sites: - if alt_site not in already_attached_sites: - meta = {"name": local_site, "created_dt": datetime.now()} - rec["sites"].append(meta) + # add alternative sites + rec = self._add_alternative_sites(system_sync_server_presets, + already_attached_sites, + rec) + + log.debug("final sites:: {}".format(rec["sites"])) return rec @@ -1092,8 +1093,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() - always_accesible = sync_project_presets["config"]. \ - get("always_accessible_on", []) + if local_site == 'local': local_site = local_site_id @@ -1106,6 +1106,32 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return local_site, remote_site + def _add_alternative_sites(self, + system_sync_server_presets, + already_attached_sites, + rec): + """Loop through all configured sites and add alternatives. + + See SyncServerModule.handle_alternate_site + """ + conf_sites = system_sync_server_presets.get("sites", {}) + + for site_name, site_info in conf_sites.items(): + alt_sites = set(site_info.get("alternative_sites", [])) + for added_site in already_attached_sites.keys(): + if added_site in alt_sites: + if site_name in already_attached_sites.keys(): + continue + meta = {"name": site_name} + real_created = already_attached_sites[added_site] + # alt site inherits state of 'created_dt' + if real_created: + meta["created_dt"] = real_created + rec["sites"].append(meta) + already_attached_sites[meta["name"]] = real_created + + return rec + def handle_destination_files(self, integrated_file_sizes, mode): """ Clean destination files Called when error happened during integrating to DB or to disk From 828c601d144dc0d24cfacdbe1a4ba29157349e99 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 16:38:29 +0100 Subject: [PATCH 20/57] OP-1937 - fix - failure when Site Sync is not enabled --- openpype/plugins/publish/integrate_new.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index c0a90ee78e..6c51f640fb 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1030,6 +1030,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): local_site = 'studio' # default remote_site = None always_accesible = [] + sync_project_presets = None rec = { "_id": io.ObjectId(), @@ -1090,6 +1091,7 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): return rec def _get_sites(self, sync_project_presets): + """Returns tuple (local_site, remote_site)""" local_site_id = openpype.api.get_local_site_id() local_site = sync_project_presets["config"]. \ get("active_site", "studio").strip() From 199717a0df976d21e23c5f2566a843ed3562771e Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:10:19 +0100 Subject: [PATCH 21/57] OP-1937 - fix - wrong icon used Icon of provider from last configured site was used --- .../modules/default_modules/sync_server/sync_server_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index 9ff706d6dd..bfc7c75c83 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -1109,8 +1109,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): return provider sync_sett = self.sync_system_settings - for site, detail in sync_sett.get("sites", {}).items(): - sites[site] = detail.get("provider") + for conf_site, detail in sync_sett.get("sites", {}).items(): + sites[conf_site] = detail.get("provider") return sites.get(site, 'N/A') From 8d00883a04d22bb19454e6e18d57e8afe3e9bd91 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:26:34 +0100 Subject: [PATCH 22/57] Hound --- .../sync_server/tray/widgets.py | 2 +- openpype/tests/mongo_performance.py | 22 +++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/tray/widgets.py b/openpype/modules/default_modules/sync_server/tray/widgets.py index 01cc0d46d2..15da1b5d19 100644 --- a/openpype/modules/default_modules/sync_server/tray/widgets.py +++ b/openpype/modules/default_modules/sync_server/tray/widgets.py @@ -140,7 +140,7 @@ class SyncProjectListWidget(QtWidgets.QWidget): selected_index.isValid() and \ not self._selection_changed: mode = QtCore.QItemSelectionModel.Select | \ - QtCore.QItemSelectionModel.Rows + QtCore.QItemSelectionModel.Rows self.project_list.selectionModel().select(selected_index, mode) if self.current_project: diff --git a/openpype/tests/mongo_performance.py b/openpype/tests/mongo_performance.py index 9220c6c730..2df3363f4b 100644 --- a/openpype/tests/mongo_performance.py +++ b/openpype/tests/mongo_performance.py @@ -104,8 +104,8 @@ class TestPerformance(): "name": "mb", "parent": {"oid": '{}'.format(id)}, "data": { - "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa - "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa + "path": "C:\\projects\\test_performance\\Assets\\Cylinder\\publish\\workfile\\workfileLookdev\\{}\\{}".format(version_str, file_name), # noqa: E501 + "template": "{root[work]}\\{project[name]}\\{hierarchy}\\{asset}\\publish\\{family}\\{subset}\\v{version:0>3}\\{project[code]}_{asset}_{subset}_v{version:0>3}<_{output}><.{frame:0>4}>.{representation}" # noqa: E501 }, "type": "representation", "schema": "openpype:representation-2.0" @@ -188,21 +188,21 @@ class TestPerformance(): create_files=False): ret = [ { - "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "{root[work]}/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_A_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_B_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id2), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), "size": random.randint(0, self.MAX_FILE_SIZE_B) }, { - "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), #noqa + "path": "{root[work]}" + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/v{:03d}/test_Cylinder_C_workfileLookdev_v{:03d}.dat".format(i, i), # noqa: E501 "_id": '{}'.format(file_id3), "hash": "temphash", "sites": self.get_sites(self.MAX_NUMBER_OF_SITES), @@ -223,8 +223,8 @@ class TestPerformance(): ret = {} ret['{}'.format(file_id)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderA_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 @@ -232,16 +232,16 @@ class TestPerformance(): ret['{}'.format(file_id2)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderB_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 } ret['{}'.format(file_id3)] = { "path": "{root[work]}" + - "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" #noqa - "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa + "/test_performance/Assets/Cylinder/publish/workfile/workfileLookdev/" # noqa: E501 + "v{:03d}/test_CylinderC_workfileLookdev_v{:03d}.mb".format(i, i), # noqa: E501 "hash": "temphash", "sites": ["studio"], "size": 87236 From 24c1aed73939a8641c5224e664a88f6b039abb90 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 4 Nov 2021 17:42:19 +0100 Subject: [PATCH 23/57] OP-1920 - renamed env var SITE_SYNC_LOCAL_ID to OPENPYPE_LOCAL_ID --- openpype/cli.py | 2 +- openpype/lib/local_settings.py | 4 ++-- openpype/pype_commands.py | 2 +- website/docs/module_site_sync.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/cli.py b/openpype/cli.py index 2a7e0c173b..f937d5818e 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -337,7 +337,7 @@ def syncserver(debug, active_site): Process mimics OP Tray with specific 'active_site' name, all configuration for this "dummy" user comes from Setting or Local Settings (configured by starting OP Tray with env - var SITE_SYNC_LOCAL_ID set to 'active_site'. + var OPENPYPE_LOCAL_ID set to 'active_site'. """ if debug: os.environ['OPENPYPE_DEBUG'] = '3' diff --git a/openpype/lib/local_settings.py b/openpype/lib/local_settings.py index af8c3cdbc8..97e99b4b5a 100644 --- a/openpype/lib/local_settings.py +++ b/openpype/lib/local_settings.py @@ -524,8 +524,8 @@ def get_local_site_id(): """ # override local id from environment # used for background syncing - if os.environ.get("SITE_SYNC_LOCAL_ID"): - return os.environ["SITE_SYNC_LOCAL_ID"] + if os.environ.get("OPENPYPE_LOCAL_ID"): + return os.environ["OPENPYPE_LOCAL_ID"] registry = OpenPypeSettingsRegistry() try: diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index e160db0f15..2ccb4c8a0b 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -332,7 +332,7 @@ class PypeCommands: def syncserver(self, active_site): """Start running sync_server in background.""" import signal - os.environ["SITE_SYNC_LOCAL_ID"] = active_site + os.environ["OPENPYPE_LOCAL_ID"] = active_site def signal_handler(sig, frame): print("You pressed Ctrl+C. Process ended.") diff --git a/website/docs/module_site_sync.md b/website/docs/module_site_sync.md index 31854e2729..d9b53e32fb 100644 --- a/website/docs/module_site_sync.md +++ b/website/docs/module_site_sync.md @@ -164,7 +164,7 @@ As current implementation relies heavily on Settings and Local Settings, backgro To do this: -- run OP `Tray` with environment variable SITE_SYNC_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) +- run OP `Tray` with environment variable OPENPYPE_LOCAL_ID set to name of active (source) site. In most use cases it would be studio (for cases of backups of everything published to studio site to different cloud site etc.) - start `Tray` - check `Local ID` in information dialog after clicking on version number in the Tray - open `Local Settings` in the `Tray` From cc59f9bd3efc20c0cc946972164f4b88d2c1a5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LORRAIN?= Date: Fri, 5 Nov 2021 18:00:26 +0100 Subject: [PATCH 24/57] Add following workfile versioning option on publish --- .../publish/collect_anatomy_instance_data.py | 7 ++++++- openpype/plugins/publish/validate_version.py | 8 ++++---- .../settings/defaults/project_settings/global.json | 3 +++ .../schemas/schema_global_publish.json | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/openpype/plugins/publish/collect_anatomy_instance_data.py b/openpype/plugins/publish/collect_anatomy_instance_data.py index 4fd657167c..e0eb1618b5 100644 --- a/openpype/plugins/publish/collect_anatomy_instance_data.py +++ b/openpype/plugins/publish/collect_anatomy_instance_data.py @@ -38,6 +38,8 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): order = pyblish.api.CollectorOrder + 0.49 label = "Collect Anatomy Instance data" + follow_workfile_version = False + def process(self, context): self.log.info("Collecting anatomy data for all instances.") @@ -213,7 +215,10 @@ class CollectAnatomyInstanceData(pyblish.api.ContextPlugin): context_asset_doc = context.data["assetEntity"] for instance in context: - version_number = instance.data.get("version") + if self.follow_workfile_version: + version_number = context.data('version') + else: + version_number = instance.data.get("version") # If version is not specified for instance or context if version_number is None: # TODO we should be able to change default version by studio diff --git a/openpype/plugins/publish/validate_version.py b/openpype/plugins/publish/validate_version.py index 927e024476..b86d72a658 100644 --- a/openpype/plugins/publish/validate_version.py +++ b/openpype/plugins/publish/validate_version.py @@ -21,8 +21,8 @@ class ValidateVersion(pyblish.api.InstancePlugin): if latest_version is not None: msg = ( - "Version `{0}` that you are trying to publish, already exists" - " in the database. Version in database: `{1}`. Please version " - "up your workfile to a higher version number than: `{1}`." - ).format(version, latest_version) + "Version `{0}` from instance `{1}` that you are trying to publish, already exists" + " in the database. Version in database: `{2}`. Please version " + "up your workfile to a higher version number than: `{2}`." + ).format(version, instance.data["name"], latest_version) assert (int(version) > int(latest_version)), msg diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 45c1a59d17..134435d909 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -1,5 +1,8 @@ { "publish": { + "CollectAnatomyInstanceData": { + "follow_workfile_version": false + }, "ValidateEditorialAssetName": { "enabled": true, "optional": false diff --git a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json index c50f383f02..375f0c26da 100644 --- a/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json +++ b/openpype/settings/entities/schemas/projects_schema/schemas/schema_global_publish.json @@ -4,6 +4,20 @@ "key": "publish", "label": "Publish plugins", "children": [ + { + "type": "dict", + "collapsible": true, + "key": "CollectAnatomyInstanceData", + "label": "Collect Anatomy Instance Data", + "is_group": true, + "children": [ + { + "type": "boolean", + "key": "follow_workfile_version", + "label": "Follow workfile version" + } + ] + }, { "type": "dict", "collapsible": true, From a8da0df9de55a22b9f497a3d201af9e4e0a81289 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Mon, 8 Nov 2021 13:31:56 +0100 Subject: [PATCH 25/57] fix cases when parent of asset version is project --- .../default_modules/ftrack/lib/ftrack_base_handler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py b/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py index a457b886ac..2130abc20c 100644 --- a/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py +++ b/openpype/modules/default_modules/ftrack/lib/ftrack_base_handler.py @@ -570,9 +570,15 @@ class BaseHandler(object): if low_entity_type == "assetversion": asset = entity["asset"] + parent = None if asset: parent = asset["parent"] - if parent: + + if parent: + if parent.entity_type.lower() == "project": + return parent + + if "project" in parent: return parent["project"] project_data = entity["link"][0] From f8e594be800010b4e535de9e051a93982365f369 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:55:11 +0100 Subject: [PATCH 26/57] added module command --- openpype/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index 512bd4663b..e1f9265c1f 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -57,6 +57,12 @@ def tray(debug=False): PypeCommands().launch_tray(debug) +@main.group(help="Run command line arguments of OpenPype modules") +@click.pass_context +def module(ctx): + pass + + @main.command() @click.option("-d", "--debug", is_flag=True, help="Print debug messages") @click.option("--ftrack-url", envvar="FTRACK_SERVER", From dee8156a0c2263ac4d75fba9a957ebfd582dff14 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:56:17 +0100 Subject: [PATCH 27/57] implemented 'add_modules' which will call cli method on module to give ability to register custom cli commands --- openpype/cli.py | 1 + openpype/pype_commands.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index e1f9265c1f..d68cba45c6 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -57,6 +57,7 @@ def tray(debug=False): PypeCommands().launch_tray(debug) +@PypeCommands.add_modules @main.group(help="Run command line arguments of OpenPype modules") @click.pass_context def module(ctx): diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index 5fac5cacc7..f4a29091d0 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -41,6 +41,17 @@ class PypeCommands: user_role = "manager" settings.main(user_role) + @staticmethod + def add_modules(click_func): + """Modules/Addons can add their cli commands dynamically.""" + from openpype.modules import ModulesManager + + manager = ModulesManager() + for module in manager.modules: + if hasattr(module, "cli"): + module.cli(click_func) + return click_func + @staticmethod def launch_eventservercli(*args): from openpype_modules.ftrack.ftrack_server.event_server_cli import ( From afd93a403567f9d79bf4569b8b98419993186c6d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 16:57:02 +0100 Subject: [PATCH 28/57] ftrack uses new cli option to register it's cli commands --- .../default_modules/ftrack/ftrack_module.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index 73a4dfee82..cc9210c216 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -1,8 +1,10 @@ import os import json import collections -from openpype.modules import OpenPypeModule +import click + +from openpype.modules import OpenPypeModule from openpype_interfaces import ( ITrayModule, IPluginPaths, @@ -409,3 +411,57 @@ class FtrackModule( return 0 hours_logged = (task_entity["time_logged"] / 60) / 60 return hours_logged + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group( + FtrackModule.name, + help="Application job server. Can be used as render farm." +) +def cli_main(): + pass + + +@cli_main.command() +@click.option("-d", "--debug", is_flag=True, help="Print debug messages") +@click.option("--ftrack-url", envvar="FTRACK_SERVER", + help="Ftrack server url") +@click.option("--ftrack-user", envvar="FTRACK_API_USER", + help="Ftrack api user") +@click.option("--ftrack-api-key", envvar="FTRACK_API_KEY", + help="Ftrack api key") +@click.option("--legacy", is_flag=True, + help="run event server without mongo storing") +@click.option("--clockify-api-key", envvar="CLOCKIFY_API_KEY", + help="Clockify API key.") +@click.option("--clockify-workspace", envvar="CLOCKIFY_WORKSPACE", + help="Clockify workspace") +def eventserver( + debug, + ftrack_url, + ftrack_user, + ftrack_api_key, + legacy, + clockify_api_key, + clockify_workspace +): + """Launch ftrack event server. + + This should be ideally used by system service (such us systemd or upstart + on linux and window service). + """ + if debug: + os.environ["OPENPYPE_DEBUG"] = "3" + + from .ftrack_server.event_server_cli import run_event_server + + return run_event_server( + ftrack_url, + ftrack_user, + ftrack_api_key, + legacy, + clockify_api_key, + clockify_workspace + ) From 88ac91c82c79f036108ccd37cb95103243588ce6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:50:44 +0100 Subject: [PATCH 29/57] fix help --- openpype/modules/default_modules/ftrack/ftrack_module.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/ftrack/ftrack_module.py b/openpype/modules/default_modules/ftrack/ftrack_module.py index cc9210c216..6db80e6c4a 100644 --- a/openpype/modules/default_modules/ftrack/ftrack_module.py +++ b/openpype/modules/default_modules/ftrack/ftrack_module.py @@ -416,10 +416,7 @@ class FtrackModule( click_group.add_command(cli_main) -@click.group( - FtrackModule.name, - help="Application job server. Can be used as render farm." -) +@click.group(FtrackModule.name, help="Ftrack module related commands.") def cli_main(): pass From 2f4ee4447e0d24b1213dc6290ea16af3e27e9531 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:51:45 +0100 Subject: [PATCH 30/57] example addon has cli commands --- .../example_addons/example_addon/addon.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/openpype/modules/example_addons/example_addon/addon.py b/openpype/modules/example_addons/example_addon/addon.py index 5573e33cc1..f4a1aa4f4e 100644 --- a/openpype/modules/example_addons/example_addon/addon.py +++ b/openpype/modules/example_addons/example_addon/addon.py @@ -8,10 +8,12 @@ in global space here until are required or used. """ import os +import click from openpype.modules import ( JsonFilesSettingsDef, - OpenPypeAddOn + OpenPypeAddOn, + ModulesManager ) # Import interface defined by this addon to be able find other addons using it from openpype_interfaces import ( @@ -130,3 +132,32 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): return { "publish": [os.path.join(current_dir, "plugins", "publish")] } + + def cli(self, click_group): + click_group.add_command(cli_main) + + +@click.group(ExampleAddon.name, help="Example addon dynamic cli commands.") +def cli_main(): + pass + + +@cli_main.command() +def nothing(): + """Does nothing but print a message.""" + print("You've triggered \"nothing\" command.") + + +@cli_main.command() +def show_dialog(): + """Show ExampleAddon dialog. + + We don't have access to addon directly through cli so we have to create + it again. + """ + from openpype.tools.utils.lib import qt_app_context + + manager = ModulesManager() + example_addon = manager.modules_by_name[ExampleAddon.name] + with qt_app_context(): + example_addon.show_dialog() From bf9d4442ea904004ce080a550a5ea1b5aa476d93 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:55:31 +0100 Subject: [PATCH 31/57] added cli info to modules readme --- openpype/modules/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openpype/modules/README.md b/openpype/modules/README.md index 5716324365..7c17025487 100644 --- a/openpype/modules/README.md +++ b/openpype/modules/README.md @@ -22,6 +22,10 @@ OpenPype modules should contain separated logic of specific kind of implementati - `__init__` should not be overridden and `initialize` should not do time consuming part but only prepare base data about module - also keep in mind that they may be initialized in headless mode - connection with other modules is made with help of interfaces +- `cli` method - can add cli commands specific for the module + - command line arguments are handled using `click` python module + - `cli` method should expect single argument which is click group on which can be called any group specific methods (e.g. `add_command` to add another click group as children see `ExampleAddon`) + - it is possible to add trigger cli commands using `./openpype_console module *args` ## Addon class `OpenPypeAddOn` - inherits from `OpenPypeModule` but is enabled by default and doesn't have to implement `initialize` and `connect_with_modules` methods @@ -140,4 +144,4 @@ class ClockifyModule( ### TrayModulesManager - inherits from `ModulesManager` -- has specific implementation for Pype Tray tool and handle `ITrayModule` methods \ No newline at end of file +- has specific implementation for Pype Tray tool and handle `ITrayModule` methods From 887e4ba004446392f89b605727d6f7447b1b5452 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 17:55:51 +0100 Subject: [PATCH 32/57] do not crash of failed calling of 'cli' method --- openpype/pype_commands.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index f4a29091d0..d4cba6f93e 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -47,9 +47,18 @@ class PypeCommands: from openpype.modules import ModulesManager manager = ModulesManager() + log = PypeLogger.get_logger("AddModulesCLI") for module in manager.modules: if hasattr(module, "cli"): - module.cli(click_func) + try: + module.cli(click_func) + + except Exception: + log.warning( + "Failed to add cli command for module \"{}\"".format( + module.name + ) + ) return click_func @staticmethod From 901eb5bc9dbd850a32c43d3e4f4ea0acfabb5aa3 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 18:02:19 +0100 Subject: [PATCH 33/57] added cli method to default methods of openpypemodule --- openpype/modules/README.md | 2 +- openpype/modules/base.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/openpype/modules/README.md b/openpype/modules/README.md index 7c17025487..86afdb9d91 100644 --- a/openpype/modules/README.md +++ b/openpype/modules/README.md @@ -22,7 +22,7 @@ OpenPype modules should contain separated logic of specific kind of implementati - `__init__` should not be overridden and `initialize` should not do time consuming part but only prepare base data about module - also keep in mind that they may be initialized in headless mode - connection with other modules is made with help of interfaces -- `cli` method - can add cli commands specific for the module +- `cli` method - add cli commands specific for the module - command line arguments are handled using `click` python module - `cli` method should expect single argument which is click group on which can be called any group specific methods (e.g. `add_command` to add another click group as children see `ExampleAddon`) - it is possible to add trigger cli commands using `./openpype_console module *args` diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..5773c684b6 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -431,6 +431,28 @@ class OpenPypeModule: """ return {} + def cli(self, module_click_group): + """Add commands to click group. + + The best practise is to create click group for whole module which is + used to separate commands. + + class MyPlugin(OpenPypeModule): + ... + def cli(self, module_click_group): + module_click_group.add_command(cli_main) + + + @click.group(, help="") + def cli_main(): + pass + + @cli_main.command() + def mycommand(): + print("my_command") + """ + pass + class OpenPypeAddOn(OpenPypeModule): # Enable Addon by default From 2709ece76ad9823921b668427b647958131a2110 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Tue, 9 Nov 2021 18:08:30 +0100 Subject: [PATCH 34/57] check of cli existence is not needed anymore --- openpype/pype_commands.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/openpype/pype_commands.py b/openpype/pype_commands.py index d4cba6f93e..5ff57ab6ad 100644 --- a/openpype/pype_commands.py +++ b/openpype/pype_commands.py @@ -49,16 +49,15 @@ class PypeCommands: manager = ModulesManager() log = PypeLogger.get_logger("AddModulesCLI") for module in manager.modules: - if hasattr(module, "cli"): - try: - module.cli(click_func) + try: + module.cli(click_func) - except Exception: - log.warning( - "Failed to add cli command for module \"{}\"".format( - module.name - ) + except Exception: + log.warning( + "Failed to add cli command for module \"{}\"".format( + module.name ) + ) return click_func @staticmethod From d4f04f72da7a952cca2709df0fd6e42a6b374637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LORRAIN?= Date: Wed, 10 Nov 2021 12:07:07 +0100 Subject: [PATCH 35/57] fix line sizes --- openpype/plugins/publish/validate_version.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openpype/plugins/publish/validate_version.py b/openpype/plugins/publish/validate_version.py index b86d72a658..e48ce6e3c3 100644 --- a/openpype/plugins/publish/validate_version.py +++ b/openpype/plugins/publish/validate_version.py @@ -21,8 +21,9 @@ class ValidateVersion(pyblish.api.InstancePlugin): if latest_version is not None: msg = ( - "Version `{0}` from instance `{1}` that you are trying to publish, already exists" - " in the database. Version in database: `{2}`. Please version " - "up your workfile to a higher version number than: `{2}`." + "Version `{0}` from instance `{1}` that you are trying to" + " publish, already exists in the database. Version in" + " database: `{2}`. Please version up your workfile to a higher" + " version number than: `{2}`." ).format(version, instance.data["name"], latest_version) assert (int(version) > int(latest_version)), msg From ab85ba115bf605d109cca608fa059bb26b0e762d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 15:06:17 +0100 Subject: [PATCH 36/57] OP-1978 - fix - wrong site --- .../modules/default_modules/sync_server/sync_server_module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index d60147a989..b4c1729013 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -1039,8 +1039,8 @@ class SyncServerModule(OpenPypeModule, ITrayModule): sys_sett = get_system_settings() sync_sett = sys_sett["modules"].get("sync_server") - for site, detail in sync_sett.get("sites", {}).items(): - sites[site] = detail.get("provider") + for conf_site, detail in sync_sett.get("sites", {}).items(): + sites[conf_site] = detail.get("provider") return sites.get(site, 'N/A') From d7fed6f259dcf79b6e6253e6735a28bd8dfba352 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 15:07:02 +0100 Subject: [PATCH 37/57] OP-1978 - updated Settings to use sync-server-sites --- .../defaults/project_settings/global.json | 2 +- .../schema_project_syncserver.json | 93 +------------------ 2 files changed, 3 insertions(+), 92 deletions(-) diff --git a/openpype/settings/defaults/project_settings/global.json b/openpype/settings/defaults/project_settings/global.json index 45c1a59d17..8cb31e8f2e 100644 --- a/openpype/settings/defaults/project_settings/global.json +++ b/openpype/settings/defaults/project_settings/global.json @@ -315,7 +315,7 @@ }, "project_folder_structure": "{\"__project_root__\": {\"prod\": {}, \"resources\": {\"footage\": {\"plates\": {}, \"offline\": {}}, \"audio\": {}, \"art_dept\": {}}, \"editorial\": {}, \"assets[ftrack.Library]\": {\"characters[ftrack]\": {}, \"locations[ftrack]\": {}}, \"shots[ftrack.Sequence]\": {\"scripts\": {}, \"editorial[ftrack.Folder]\": {}}}}", "sync_server": { - "enabled": true, + "enabled": false, "config": { "retry_cnt": "3", "loop_delay": "60", diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json index 3211babd43..85121c471a 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_syncserver.json @@ -39,100 +39,11 @@ ] }, { - "type": "dict-modifiable", + "type": "sync-server-sites", "collapsible": true, "key": "sites", "label": "Sites", - "collapsible_key": false, - "object_type": { - "type": "dict", - "children": [ - { - "type": "dict", - "key": "gdrive", - "label": "Google Drive", - "collapsible": true, - "children": [ - { - "type": "path", - "key": "credentials_url", - "label": "Credentials url", - "multiplatform": true - } - ] - }, - { - "type": "dict", - "key": "dropbox", - "label": "Dropbox", - "collapsible": true, - "children": [ - { - "type": "text", - "key": "token", - "label": "Access Token" - }, - { - "type": "text", - "key": "team_folder_name", - "label": "Team Folder Name" - }, - { - "type": "text", - "key": "acting_as_member", - "label": "Acting As Member" - } - ] - }, - { - "type": "dict", - "key": "sftp", - "label": "SFTP", - "collapsible": true, - "children": [ - { - "type": "text", - "key": "sftp_host", - "label": "SFTP host" - }, - { - "type": "number", - "key": "sftp_port", - "label": "SFTP port" - }, - { - "type": "text", - "key": "sftp_user", - "label": "SFTP user" - }, - { - "type": "text", - "key": "sftp_pass", - "label": "SFTP pass" - }, - { - "type": "path", - "key": "sftp_key", - "label": "SFTP user ssh key", - "multiplatform": true - }, - { - "type": "text", - "key": "sftp_key_pass", - "label": "SFTP user ssh key password" - } - ] - }, - { - "type": "dict-modifiable", - "key": "root", - "label": "Roots", - "collapsable": false, - "collapsable_key": false, - "object_type": "text" - } - ] - } + "collapsible_key": false } ] } From 3594c8019af35af4a188ddbd055110742cc9c83d Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 15:08:17 +0100 Subject: [PATCH 38/57] OP-1978 - updated sftp to use sync-server-sites Contains changes from #2206 --- .../sync_server/providers/sftp.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/sftp.py b/openpype/modules/default_modules/sync_server/providers/sftp.py index d737849cdc..094b14a859 100644 --- a/openpype/modules/default_modules/sync_server/providers/sftp.py +++ b/openpype/modules/default_modules/sync_server/providers/sftp.py @@ -1,8 +1,6 @@ import os import os.path import time -import sys -import six import threading import platform @@ -14,6 +12,7 @@ log = Logger().get_logger("SyncServer") pysftp = None try: import pysftp + import paramiko except (ImportError, SyntaxError): pass @@ -49,22 +48,15 @@ class SFTPHandler(AbstractProvider): format(site_name)) return - provider_presets = self.presets.get(self.CODE) - if not provider_presets: - msg = "Sync Server: No provider presets for {}".format(self.CODE) - log.warning(msg) - return - # store to instance for reconnect - self.sftp_host = provider_presets["sftp_host"] - self.sftp_port = provider_presets["sftp_port"] - self.sftp_user = provider_presets["sftp_user"] - self.sftp_pass = provider_presets["sftp_pass"] - self.sftp_key = provider_presets["sftp_key"] - self.sftp_key_pass = provider_presets["sftp_key_pass"] + self.sftp_host = presets["sftp_host"] + self.sftp_port = presets["sftp_port"] + self.sftp_user = presets["sftp_user"] + self.sftp_pass = presets["sftp_pass"] + self.sftp_key = presets["sftp_key"] + self.sftp_key_pass = presets["sftp_key_pass"] self._tree = None - self.active = True @property def conn(self): @@ -80,7 +72,7 @@ class SFTPHandler(AbstractProvider): Returns: (boolean) """ - return self.conn is not None + return self.presets["enabled"] and self.conn is not None @classmethod def get_system_settings_schema(cls): @@ -108,7 +100,7 @@ class SFTPHandler(AbstractProvider): editable = [ # credentials could be overriden on Project or User level { - 'key': "sftp_server", + 'key': "sftp_host", 'label': "SFTP host name", 'type': 'text' }, @@ -130,7 +122,8 @@ class SFTPHandler(AbstractProvider): { 'key': "sftp_key", 'label': "SFTP user ssh key", - 'type': 'path' + 'type': 'path', + "multiplatform": True }, { 'key': "sftp_key_pass", @@ -144,7 +137,7 @@ class SFTPHandler(AbstractProvider): "type": "dict-roots", "object_type": { "type": "path", - "multiplatform": True, + "multiplatform": False, "multipath": False } } @@ -176,7 +169,8 @@ class SFTPHandler(AbstractProvider): { 'key': "sftp_key", 'label': "SFTP user ssh key", - 'type': 'path' + 'type': 'path', + "multiplatform": True }, { 'key': "sftp_key_pass", @@ -199,7 +193,7 @@ class SFTPHandler(AbstractProvider): Format is importing for usage of python's format ** approach """ # roots cannot be locally overridden - return self.presets['root'] + return self.presets['roots'] def get_tree(self): """ @@ -426,7 +420,10 @@ class SFTPHandler(AbstractProvider): if self.sftp_key_pass: conn_params['private_key_pass'] = self.sftp_key_pass - return pysftp.Connection(**conn_params) + try: + return pysftp.Connection(**conn_params) + except paramiko.ssh_exception.SSHException: + log.warning("Couldn't connect", exc_info=True) def _mark_progress(self, collection, file, representation, server, site, source_path, target_path, direction): From 763adf5a53c80c5ec1eba556323085842344d974 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 15:08:43 +0100 Subject: [PATCH 39/57] OP-1978 - updated gdrive to use sync-server-sites Contains changes from #2206 --- .../sync_server/providers/gdrive.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/gdrive.py b/openpype/modules/default_modules/sync_server/providers/gdrive.py index 8c8447f8f0..a5b68a12c3 100644 --- a/openpype/modules/default_modules/sync_server/providers/gdrive.py +++ b/openpype/modules/default_modules/sync_server/providers/gdrive.py @@ -73,13 +73,7 @@ class GDriveHandler(AbstractProvider): format(site_name)) return - provider_presets = self.presets.get(self.CODE) - if not provider_presets: - msg = "Sync Server: No provider presets for {}".format(self.CODE) - log.info(msg) - return - - cred_path = self.presets[self.CODE].get("credentials_url", {}).\ + cred_path = self.presets.get("credentials_url", {}).\ get(platform.system().lower()) or '' if not os.path.exists(cred_path): msg = "Sync Server: No credentials for gdrive provider " + \ @@ -87,10 +81,12 @@ class GDriveHandler(AbstractProvider): log.info(msg) return - self.service = self._get_gd_service(cred_path) + self.service = None + if self.presets["enabled"]: + self.service = self._get_gd_service(cred_path) - self._tree = tree - self.active = True + self._tree = tree + self.active = True def is_active(self): """ @@ -98,7 +94,7 @@ class GDriveHandler(AbstractProvider): Returns: (boolean) """ - return self.service is not None + return self.presets["enabled"] and self.service is not None @classmethod def get_system_settings_schema(cls): @@ -125,9 +121,11 @@ class GDriveHandler(AbstractProvider): editable = [ # credentials could be overriden on Project or User level { - 'key': "credentials_url", - 'label': "Credentials url", - 'type': 'text' + "type": "path", + "key": "credentials_url", + "label": "Credentials url", + "multiplatform": True, + "placeholder": "Credentials url" }, # roots could be overriden only on Project leve, User cannot { @@ -136,7 +134,7 @@ class GDriveHandler(AbstractProvider): "type": "dict-roots", "object_type": { "type": "path", - "multiplatform": True, + "multiplatform": False, "multipath": False } } @@ -176,7 +174,7 @@ class GDriveHandler(AbstractProvider): Format is importing for usage of python's format ** approach """ # GDrive roots cannot be locally overridden - return self.presets['root'] + return self.presets['roots'] def get_tree(self): """ From 965d798656f722ef7d08a7ee1f5f154167d5f060 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 15:09:25 +0100 Subject: [PATCH 40/57] OP-1978 - updated dropbox to use sync-server-sites Contains changes from #2206 --- .../sync_server/providers/dropbox.py | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/providers/dropbox.py b/openpype/modules/default_modules/sync_server/providers/dropbox.py index 2bc7a83a5b..84a540be5d 100644 --- a/openpype/modules/default_modules/sync_server/providers/dropbox.py +++ b/openpype/modules/default_modules/sync_server/providers/dropbox.py @@ -24,25 +24,19 @@ class DropboxHandler(AbstractProvider): ) return - provider_presets = self.presets.get(self.CODE) - if not provider_presets: - msg = "Sync Server: No provider presets for {}".format(self.CODE) - log.info(msg) - return - - token = self.presets[self.CODE].get("token", "") + token = self.presets.get("token", "") if not token: msg = "Sync Server: No access token for dropbox provider" log.info(msg) return - team_folder_name = self.presets[self.CODE].get("team_folder_name", "") + team_folder_name = self.presets.get("team_folder_name", "") if not team_folder_name: msg = "Sync Server: No team folder name for dropbox provider" log.info(msg) return - acting_as_member = self.presets[self.CODE].get("acting_as_member", "") + acting_as_member = self.presets.get("acting_as_member", "") if not acting_as_member: msg = ( "Sync Server: No acting member for dropbox provider" @@ -51,13 +45,15 @@ class DropboxHandler(AbstractProvider): return self.dbx = None - try: - self.dbx = self._get_service( - token, acting_as_member, team_folder_name - ) - except Exception as e: - log.info("Could not establish dropbox object: {}".format(e)) - return + + if self.presets["enabled"]: + try: + self.dbx = self._get_service( + token, acting_as_member, team_folder_name + ) + except Exception as e: + log.info("Could not establish dropbox object: {}".format(e)) + return super(AbstractProvider, self).__init__() @@ -106,7 +102,7 @@ class DropboxHandler(AbstractProvider): "type": "dict-roots", "object_type": { "type": "path", - "multiplatform": True, + "multiplatform": False, "multipath": False } } @@ -169,7 +165,7 @@ class DropboxHandler(AbstractProvider): Returns: (boolean) """ - return self.dbx is not None + return self.presets["enabled"] and self.dbx is not None @classmethod def get_configurable_items(cls): @@ -393,7 +389,7 @@ class DropboxHandler(AbstractProvider): {"root": {"root_ONE": "value", "root_TWO":"value}} Format is importing for usage of python's format ** approach """ - return self.presets['root'] + return self.presets['roots'] def resolve_path(self, path, root_config=None, anatomy=None): """ From cc0672d42974a3381108d62fb175e7447dbca987 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Wed, 10 Nov 2021 17:38:17 +0100 Subject: [PATCH 41/57] OP-2017 - fix - enum for color coding in PS Some keys are really weird in PS --- .../projects_schema/schema_project_photoshop.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json index f00bf78fe4..ca388de60c 100644 --- a/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json +++ b/openpype/settings/entities/schemas/projects_schema/schema_project_photoshop.json @@ -67,7 +67,17 @@ "type": "list", "key": "color_code", "label": "Color codes for layers", - "object_type": "text" + "type": "enum", + "multiselection": true, + "enum_items": [ + { "red": "red" }, + { "orange": "orange" }, + { "yellowColor": "yellow" }, + { "grain": "green" }, + { "blue": "blue" }, + { "violet": "violet" }, + { "gray": "gray" } + ] }, { "type": "list", From e6d76c5b580252a4eea6be22cf02de0380e2786d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 14:17:09 +0100 Subject: [PATCH 42/57] fix python 2 unicode conversion --- openpype/lib/python_module_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openpype/lib/python_module_tools.py b/openpype/lib/python_module_tools.py index cb5f285ddd..69da4cc661 100644 --- a/openpype/lib/python_module_tools.py +++ b/openpype/lib/python_module_tools.py @@ -22,6 +22,9 @@ def import_filepath(filepath, module_name=None): if module_name is None: module_name = os.path.splitext(os.path.basename(filepath))[0] + # Make sure it is not 'unicode' in Python 2 + module_name = str(module_name) + # Prepare module object where content of file will be parsed module = types.ModuleType(module_name) From 5acdb6e182efede7d01d136c431c41329e77e203 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 14:53:57 +0100 Subject: [PATCH 43/57] added docstring --- openpype/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openpype/cli.py b/openpype/cli.py index d68cba45c6..6a4d8f1120 100644 --- a/openpype/cli.py +++ b/openpype/cli.py @@ -61,6 +61,10 @@ def tray(debug=False): @main.group(help="Run command line arguments of OpenPype modules") @click.pass_context def module(ctx): + """Module specific commands created dynamically. + + These commands are generated dynamically by currently loaded addon/modules. + """ pass From adbd0b1c5e8b5d2d2a7a9a862798a373b982b431 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Thu, 11 Nov 2021 18:01:14 +0100 Subject: [PATCH 44/57] OP-2015 - added queue for studio processing in PS --- .../webserver_service/webpublish_routes.py | 20 ++++++++++++------- .../webserver_service/webserver_cli.py | 11 +++++++++- .../defaults/project_settings/photoshop.json | 3 +-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 73e5113f38..a56b7c48c3 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -20,11 +20,16 @@ log = PypeLogger.get_logger("WebServer") class RestApiResource: """Resource carrying needed info and Avalon DB connection for publish.""" - def __init__(self, server_manager, executable, upload_dir): + def __init__(self, server_manager, executable, upload_dir, + studio_task_queue=None): self.server_manager = server_manager self.upload_dir = upload_dir self.executable = executable + if studio_task_queue is None: + studio_task_queue = collections.deque().dequeu + self.studio_task_queue = studio_task_queue + self.dbcon = AvalonMongoDB() self.dbcon.install() @@ -182,8 +187,6 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): msg = "Non existent OpenPype executable {}".format(openpype_app) raise RuntimeError(msg) - # for postprocessing in host, currently only PS - output = {} log.info("WebpublisherBatchPublishEndpoint called") content = await request.json() @@ -225,13 +228,13 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): batch_data = parse_json(os.path.join(batch_path, "manifest.json")) if not batch_data: raise ValueError( - "Cannot parse batch meta in {} folder".format(batch_path)) + "Cannot parse batch manifest in {}".format(batch_path)) task_dir_name = batch_data["tasks"][0] task_data = parse_json(os.path.join(batch_path, task_dir_name, "manifest.json")) if not task_data: raise ValueError( - "Cannot parse batch meta in {} folder".format(task_data)) + "Cannot parse task manifest in {}".format(task_data)) for process_filter in studio_processing_filters: filter_extensions = process_filter.get("extensions") or [] @@ -263,11 +266,14 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): args.append(value) log.info("args:: {}".format(args)) + if content.get("studio_processing"): + log.debug("Adding to queue") + self.resource.studio_task_queue.append(args) + else: + subprocess.call(args) - subprocess.call(args) return Response( status=200, - body=self.resource.encode(output), content_type="application/json" ) diff --git a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py index d00d269059..b784105461 100644 --- a/openpype/hosts/webpublisher/webserver_service/webserver_cli.py +++ b/openpype/hosts/webpublisher/webserver_service/webserver_cli.py @@ -1,8 +1,10 @@ +import collections import time import os from datetime import datetime import requests import json +import subprocess from openpype.lib import PypeLogger @@ -31,10 +33,13 @@ def run_webserver(*args, **kwargs): port = kwargs.get("port") or 8079 server_manager = webserver_module.create_new_server_manager(port, host) webserver_url = server_manager.url + # queue for remotepublishfromapp tasks + studio_task_queue = collections.deque() resource = RestApiResource(server_manager, upload_dir=kwargs["upload_dir"], - executable=kwargs["executable"]) + executable=kwargs["executable"], + studio_task_queue=studio_task_queue) projects_endpoint = WebpublisherProjectsEndpoint(resource) server_manager.add_route( "GET", @@ -88,6 +93,10 @@ def run_webserver(*args, **kwargs): if time.time() - last_reprocessed > 20: reprocess_failed(kwargs["upload_dir"], webserver_url) last_reprocessed = time.time() + if studio_task_queue: + args = studio_task_queue.popleft() + subprocess.call(args) # blocking call + time.sleep(1.0) diff --git a/openpype/settings/defaults/project_settings/photoshop.json b/openpype/settings/defaults/project_settings/photoshop.json index eb9f96e348..0c24c943ec 100644 --- a/openpype/settings/defaults/project_settings/photoshop.json +++ b/openpype/settings/defaults/project_settings/photoshop.json @@ -30,8 +30,7 @@ }, "ExtractReview": { "jpg_options": { - "tags": [ - ] + "tags": [] }, "mov_options": { "tags": [ From 79fce6a5b453ed88fe53374578fec797dded539a Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:05:20 +0100 Subject: [PATCH 45/57] example addon is not using inteface --- .../example_addons/example_addon/addon.py | 16 ----------- .../example_addon/interfaces.py | 28 ------------------- .../example_addons/example_addon/widgets.py | 12 ++------ 3 files changed, 2 insertions(+), 54 deletions(-) delete mode 100644 openpype/modules/example_addons/example_addon/interfaces.py diff --git a/openpype/modules/example_addons/example_addon/addon.py b/openpype/modules/example_addons/example_addon/addon.py index 5573e33cc1..162868e4d4 100644 --- a/openpype/modules/example_addons/example_addon/addon.py +++ b/openpype/modules/example_addons/example_addon/addon.py @@ -15,7 +15,6 @@ from openpype.modules import ( ) # Import interface defined by this addon to be able find other addons using it from openpype_interfaces import ( - IExampleInterface, IPluginPaths, ITrayAction ) @@ -75,19 +74,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): self._create_dialog() - def connect_with_modules(self, enabled_modules): - """Method where you should find connected modules. - - It is triggered by OpenPype modules manager at the best possible time. - Some addons and modules may required to connect with other modules - before their main logic is executed so changes would require to restart - whole process. - """ - self._connected_modules = [] - for module in enabled_modules: - if isinstance(module, IExampleInterface): - self._connected_modules.append(module) - def _create_dialog(self): # Don't recreate dialog if already exists if self._dialog is not None: @@ -106,8 +92,6 @@ class ExampleAddon(OpenPypeAddOn, IPluginPaths, ITrayAction): """ # Make sure dialog is created self._create_dialog() - # Change value of dialog by current state - self._dialog.set_connected_modules(self.get_connected_modules()) # Show dialog self._dialog.open() diff --git a/openpype/modules/example_addons/example_addon/interfaces.py b/openpype/modules/example_addons/example_addon/interfaces.py deleted file mode 100644 index 371536efc7..0000000000 --- a/openpype/modules/example_addons/example_addon/interfaces.py +++ /dev/null @@ -1,28 +0,0 @@ -""" Using interfaces is one way of connecting multiple OpenPype Addons/Modules. - -Interfaces must be in `interfaces.py` file (or folder). Interfaces should not -import module logic or other module in global namespace. That is because -all of them must be imported before all OpenPype AddOns and Modules. - -Ideally they should just define abstract and helper methods. If interface -require any logic or connection it should be defined in module. - -Keep in mind that attributes and methods will be added to other addon -attributes and methods so they should be unique and ideally contain -addon name in it's name. -""" - -from abc import abstractmethod -from openpype.modules import OpenPypeInterface - - -class IExampleInterface(OpenPypeInterface): - """Example interface of addon.""" - _example_module = None - - def get_example_module(self): - return self._example_module - - @abstractmethod - def example_method_of_example_interface(self): - pass diff --git a/openpype/modules/example_addons/example_addon/widgets.py b/openpype/modules/example_addons/example_addon/widgets.py index 0acf238409..c0a0a7e510 100644 --- a/openpype/modules/example_addons/example_addon/widgets.py +++ b/openpype/modules/example_addons/example_addon/widgets.py @@ -9,7 +9,8 @@ class MyExampleDialog(QtWidgets.QDialog): self.setWindowTitle("Connected modules") - label_widget = QtWidgets.QLabel(self) + msg = "This is example dialog of example addon." + label_widget = QtWidgets.QLabel(msg, self) ok_btn = QtWidgets.QPushButton("OK", self) btns_layout = QtWidgets.QHBoxLayout() @@ -28,12 +29,3 @@ class MyExampleDialog(QtWidgets.QDialog): def _on_ok_clicked(self): self.done(1) - - def set_connected_modules(self, connected_modules): - if connected_modules: - message = "\n".join(connected_modules) - else: - message = ( - "Other enabled modules/addons are not using my interface." - ) - self._label_widget.setText(message) From f0a913e02493ba0c2a8885c69d77ec740d5d936d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:05:33 +0100 Subject: [PATCH 46/57] moved interfaces to modules directory --- openpype/modules/{default_modules => }/interfaces.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openpype/modules/{default_modules => }/interfaces.py (100%) diff --git a/openpype/modules/default_modules/interfaces.py b/openpype/modules/interfaces.py similarity index 100% rename from openpype/modules/default_modules/interfaces.py rename to openpype/modules/interfaces.py From b2b532de36327c845da37cb4c8c30c88debbfdc1 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:06:09 +0100 Subject: [PATCH 47/57] modifid current interfaces import to keep backwards compatibility --- openpype/modules/base.py | 55 +++++++--------------------------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..01d54d9051 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -212,54 +212,17 @@ def _load_interfaces(): _InterfacesClass(modules_key) ) - log = PypeLogger.get_logger("InterfacesLoader") + from . import interfaces - dirpaths = get_module_dirs() - - interface_paths = [] - interface_paths.append( - os.path.join(get_default_modules_dir(), "interfaces.py") - ) - for dirpath in dirpaths: - if not os.path.exists(dirpath): + for attr_name in dir(interfaces): + attr = getattr(interfaces, attr_name) + if ( + not inspect.isclass(attr) + or attr is OpenPypeInterface + or not issubclass(attr, OpenPypeInterface) + ): continue - - for filename in os.listdir(dirpath): - if filename in ("__pycache__", ): - continue - - full_path = os.path.join(dirpath, filename) - if not os.path.isdir(full_path): - continue - - interfaces_path = os.path.join(full_path, "interfaces.py") - if os.path.exists(interfaces_path): - interface_paths.append(interfaces_path) - - for full_path in interface_paths: - if not os.path.exists(full_path): - continue - - try: - # Prepare module object where content of file will be parsed - module = import_filepath(full_path) - - except Exception: - log.warning( - "Failed to load path: \"{0}\"".format(full_path), - exc_info=True - ) - continue - - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - not inspect.isclass(attr) - or attr is OpenPypeInterface - or not issubclass(attr, OpenPypeInterface) - ): - continue - setattr(openpype_interfaces, attr_name, attr) + setattr(openpype_interfaces, attr_name, attr) def load_modules(force=False): From a051f4e8b6cdc5f069bde46187d3583f13d8dc35 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:18:04 +0100 Subject: [PATCH 48/57] moved 'ISettingsChangeListener' to global interfaces --- .../settings_module/interfaces.py | 30 ------------------- openpype/modules/interfaces.py | 28 +++++++++++++++++ 2 files changed, 28 insertions(+), 30 deletions(-) delete mode 100644 openpype/modules/default_modules/settings_module/interfaces.py diff --git a/openpype/modules/default_modules/settings_module/interfaces.py b/openpype/modules/default_modules/settings_module/interfaces.py deleted file mode 100644 index 42db395649..0000000000 --- a/openpype/modules/default_modules/settings_module/interfaces.py +++ /dev/null @@ -1,30 +0,0 @@ -from abc import abstractmethod -from openpype.modules import OpenPypeInterface - - -class ISettingsChangeListener(OpenPypeInterface): - """Module has plugin paths to return. - - Expected result is dictionary with keys "publish", "create", "load" or - "actions" and values as list or string. - { - "publish": ["path/to/publish_plugins"] - } - """ - @abstractmethod - def on_system_settings_save( - self, old_value, new_value, changes, new_value_metadata - ): - pass - - @abstractmethod - def on_project_settings_save( - self, old_value, new_value, changes, project_name, new_value_metadata - ): - pass - - @abstractmethod - def on_project_anatomy_save( - self, old_value, new_value, changes, project_name, new_value_metadata - ): - pass diff --git a/openpype/modules/interfaces.py b/openpype/modules/interfaces.py index a60c5fa606..e6e84a0d42 100644 --- a/openpype/modules/interfaces.py +++ b/openpype/modules/interfaces.py @@ -263,3 +263,31 @@ class ITrayService(ITrayModule): """Change icon of an QAction to orange circle.""" if self.menu_action: self.menu_action.setIcon(self.get_icon_idle()) + + +class ISettingsChangeListener(OpenPypeInterface): + """Module has plugin paths to return. + + Expected result is dictionary with keys "publish", "create", "load" or + "actions" and values as list or string. + { + "publish": ["path/to/publish_plugins"] + } + """ + @abstractmethod + def on_system_settings_save( + self, old_value, new_value, changes, new_value_metadata + ): + pass + + @abstractmethod + def on_project_settings_save( + self, old_value, new_value, changes, project_name, new_value_metadata + ): + pass + + @abstractmethod + def on_project_anatomy_save( + self, old_value, new_value, changes, project_name, new_value_metadata + ): + pass From 9a74a6a1e8602cb8000b088630c585927141851c Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:18:19 +0100 Subject: [PATCH 49/57] removed unused import of ITimersManager --- .../modules/default_modules/timers_manager/timers_manager.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openpype/modules/default_modules/timers_manager/timers_manager.py b/openpype/modules/default_modules/timers_manager/timers_manager.py index 7687d056f8..1aeccbb958 100644 --- a/openpype/modules/default_modules/timers_manager/timers_manager.py +++ b/openpype/modules/default_modules/timers_manager/timers_manager.py @@ -1,10 +1,7 @@ import os import platform from openpype.modules import OpenPypeModule -from openpype_interfaces import ( - ITimersManager, - ITrayService -) +from openpype_interfaces import ITrayService from avalon.api import AvalonMongoDB From b09382bf8dbd3c34666306bc5db8e2c5cf046d6d Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:20:09 +0100 Subject: [PATCH 50/57] raise ImportError if interface does not exists --- openpype/modules/base.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 01d54d9051..cb39666626 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -107,12 +107,9 @@ class _InterfacesClass(_ModuleClass): if attr_name in ("__path__", "__file__"): return None - # Fake Interface if is not missing - self.__attributes__[attr_name] = type( - attr_name, - (MissingInteface, ), - {} - ) + raise ImportError(( + "cannot import name '{}' from 'openpype_interfaces'" + ).format(attr_name)) return self.__attributes__[attr_name] From d3fd330f944d212e90b3a5548bad38b78d6955e6 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:21:38 +0100 Subject: [PATCH 51/57] removed unused 'MissingInterface' --- openpype/modules/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index cb39666626..3f59d5900b 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -329,14 +329,6 @@ class OpenPypeInterface: pass -class MissingInteface(OpenPypeInterface): - """Class representing missing interface class. - - Used when interface is not available from currently registered paths. - """ - pass - - @six.add_metaclass(ABCMeta) class OpenPypeModule: """Base class of pype module. From 992986fd1be27612b95e4a3007aac926c53dbd77 Mon Sep 17 00:00:00 2001 From: iLLiCiTiT Date: Thu, 11 Nov 2021 18:31:42 +0100 Subject: [PATCH 52/57] skip module directories without init file --- openpype/modules/base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openpype/modules/base.py b/openpype/modules/base.py index 7779fff6ec..692495600c 100644 --- a/openpype/modules/base.py +++ b/openpype/modules/base.py @@ -333,6 +333,15 @@ def _load_modules(): # - check manifest and content of manifest try: if os.path.isdir(fullpath): + # Module without init file can't be used as OpenPype module + # because the module class could not be imported + init_file = os.path.join(fullpath, "__init__.py") + if not os.path.exists(init_file): + log.info(( + "Skipping module directory because of" + " missing \"__init__.py\" file. \"{}\"" + ).format(fullpath)) + continue import_module_from_dirpath(dirpath, filename, modules_key) elif ext in (".py", ): From 87191d8d300338defa81c3894af849d98562e5d3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 12:08:04 +0100 Subject: [PATCH 53/57] OP-1937 - fix alternate sites for default studio site --- .../sync_server/sync_server_module.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index ca322b9f64..c5649afec4 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -793,20 +793,34 @@ class SyncServerModule(OpenPypeModule, ITrayModule): by provider """ sites = self.sync_system_settings.get("sites", {}) - for site_name, site_info in sites.items(): - if processed_site in site_info.get("alternative_sites", []): - query = { - "_id": representation["_id"] - } - elem = {"name": "sftp", - "created_dt": datetime.now(), - "id": synced_file_id} + sites[self.DEFAULT_SITE] = {"provider": "local_drive", + "alternative_sites": []} - self.log.debug("Adding alternate {} to {}".format( - site_name, representation["_id"])) - self._add_site(collection, query, - [representation], elem, - site_name, file_id=file_id, force=True) + alternate_sites = [] + for site_name, site_info in sites.items(): + conf_alternative_sites = site_info.get("alternative_sites", []) + if processed_site in conf_alternative_sites: + alternate_sites.append(site_name) + continue + if processed_site == site_name and conf_alternative_sites: + alternate_sites.extend(conf_alternative_sites) + continue + + alternate_sites = set(alternate_sites) + + for alt_site in alternate_sites: + query = { + "_id": representation["_id"] + } + elem = {"name": alt_site, + "created_dt": datetime.now(), + "id": synced_file_id} + + self.log.debug("Adding alternate {} to {}".format( + alt_site, representation["_id"])) + self._add_site(collection, query, + [representation], elem, + site_name, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): From 1431bc5ac9887c6097cdc91887e4e1954cefecc3 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 13:30:37 +0100 Subject: [PATCH 54/57] OP-2015 - fix - adding to queue decided by configuration --- .../webpublisher/webserver_service/webpublish_routes.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index a56b7c48c3..445fa071c5 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -205,7 +205,10 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): # Make sure targets are set to None for cases that default # would change # - targets argument is not used in 'remotepublishfromapp' - "targets": None + "targets": None, + # does publish need to be handled by a queue, eg. only + # single process running concurrently? + "add_to_queue": True } } ] @@ -222,6 +225,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): "targets": ["filespublish"] } + add_to_queue = False if content.get("studio_processing"): log.info("Post processing called") @@ -247,6 +251,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): add_args.update( process_filter.get("arguments") or {} ) + add_to_queue = process_filter["add_to_queue"] break args = [ @@ -266,7 +271,7 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): args.append(value) log.info("args:: {}".format(args)) - if content.get("studio_processing"): + if add_to_queue: log.debug("Adding to queue") self.resource.studio_task_queue.append(args) else: From 27b6197ee8def8259080d0a90f0c12e3771c8324 Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 13:52:28 +0100 Subject: [PATCH 55/57] OP-2015 - fix - adding to queue decided by configuration --- .../webserver_service/webpublish_routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py index 445fa071c5..e34a899c4b 100644 --- a/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py +++ b/openpype/hosts/webpublisher/webserver_service/webpublish_routes.py @@ -205,11 +205,11 @@ class WebpublisherBatchPublishEndpoint(_RestApiEndpoint): # Make sure targets are set to None for cases that default # would change # - targets argument is not used in 'remotepublishfromapp' - "targets": None, - # does publish need to be handled by a queue, eg. only - # single process running concurrently? - "add_to_queue": True - } + "targets": None + }, + # does publish need to be handled by a queue, eg. only + # single process running concurrently? + "add_to_queue": True } ] From 4d2bd5a8813e62e668be8abae05cd315bfd1e12b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 18:27:08 +0100 Subject: [PATCH 56/57] OP-1978 - fix wrong site name used if resynchronizing --- openpype/modules/default_modules/sync_server/sync_server.py | 2 +- .../modules/default_modules/sync_server/sync_server_module.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openpype/modules/default_modules/sync_server/sync_server.py b/openpype/modules/default_modules/sync_server/sync_server.py index 8518c4a301..22eed01ef3 100644 --- a/openpype/modules/default_modules/sync_server/sync_server.py +++ b/openpype/modules/default_modules/sync_server/sync_server.py @@ -136,7 +136,7 @@ async def download(module, collection, file, representation, provider_name, True ) - module.handle_alternate_site(collection, representation, remote_site_name, + module.handle_alternate_site(collection, representation, local_site, file["_id"], file_id) return file_id diff --git a/openpype/modules/default_modules/sync_server/sync_server_module.py b/openpype/modules/default_modules/sync_server/sync_server_module.py index c5649afec4..f6b599d92c 100644 --- a/openpype/modules/default_modules/sync_server/sync_server_module.py +++ b/openpype/modules/default_modules/sync_server/sync_server_module.py @@ -820,7 +820,7 @@ class SyncServerModule(OpenPypeModule, ITrayModule): alt_site, representation["_id"])) self._add_site(collection, query, [representation], elem, - site_name, file_id=file_id, force=True) + alt_site, file_id=file_id, force=True) """ End of Public API """ def get_local_file_path(self, collection, site_name, file_path): From 22f428ffd42b76a43b0b49985bc1f91dc1fc3a4b Mon Sep 17 00:00:00 2001 From: Petr Kalis Date: Fri, 12 Nov 2021 18:27:38 +0100 Subject: [PATCH 57/57] OP-1978 - fix distinary change during loop --- openpype/plugins/publish/integrate_new.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openpype/plugins/publish/integrate_new.py b/openpype/plugins/publish/integrate_new.py index 6c51f640fb..7ff7466a2a 100644 --- a/openpype/plugins/publish/integrate_new.py +++ b/openpype/plugins/publish/integrate_new.py @@ -1120,9 +1120,10 @@ class IntegrateAssetNew(pyblish.api.InstancePlugin): for site_name, site_info in conf_sites.items(): alt_sites = set(site_info.get("alternative_sites", [])) - for added_site in already_attached_sites.keys(): + already_attached_keys = list(already_attached_sites.keys()) + for added_site in already_attached_keys: if added_site in alt_sites: - if site_name in already_attached_sites.keys(): + if site_name in already_attached_keys: continue meta = {"name": site_name} real_created = already_attached_sites[added_site]