From 9ea57c2a9398ad52c5a91b2220ae61955ecf7754 Mon Sep 17 00:00:00 2001 From: Eric Liang Date: Wed, 1 Aug 2018 20:53:53 -0700 Subject: [PATCH] [rllib] Basic IMPALA implementation (using deepmind's reference vtrace.py) (#2504) Rename AsyncSamplesOptimizer -> AsyncReplayOptimizer Add AsyncSamplesOptimizer that implements the IMPALA architecture integrate V-trace with a3c policy graph audit V-trace integration benchmark compare vs A3C and with V-trace on/off PongNoFrameskip-v4 on IMPALA scaling from 16 to 128 workers, solving Pong in <10 min. For reference, solving this env takes ~40 minutes for Ape-X and several hours for A3C. --- LICENSE | 19 ++ doc/source/impala.png | Bin 0 -> 99635 bytes doc/source/rllib-algorithms.rst | 18 +- doc/source/rllib-training.rst | 8 +- doc/source/rllib.rst | 1 + python/ray/rllib/__init__.py | 2 +- .../rllib/agents/a3c/a3c_tf_policy_graph.py | 36 +-- python/ray/rllib/agents/agent.py | 3 + python/ray/rllib/agents/ddpg/apex.py | 2 +- python/ray/rllib/agents/dqn/apex.py | 2 +- python/ray/rllib/agents/impala/__init__.py | 3 + python/ray/rllib/agents/impala/impala.py | 123 +++++++ python/ray/rllib/agents/impala/vtrace.py | 300 ++++++++++++++++++ .../agents/impala/vtrace_policy_graph.py | 217 +++++++++++++ python/ray/rllib/optimizers/__init__.py | 6 +- .../optimizers/async_gradients_optimizer.py | 4 + .../optimizers/async_replay_optimizer.py | 295 +++++++++++++++++ .../optimizers/async_samples_optimizer.py | 227 ++++--------- .../pong-impala-vectorized.yaml | 11 + .../ray/rllib/tuned_examples/pong-impala.yaml | 13 + python/ray/rllib/utils/actors.py | 20 ++ test/jenkins_tests/run_multi_node_tests.sh | 21 ++ 22 files changed, 1131 insertions(+), 200 deletions(-) create mode 100644 doc/source/impala.png create mode 100644 python/ray/rllib/agents/impala/__init__.py create mode 100644 python/ray/rllib/agents/impala/impala.py create mode 100644 python/ray/rllib/agents/impala/vtrace.py create mode 100644 python/ray/rllib/agents/impala/vtrace_policy_graph.py create mode 100644 python/ray/rllib/optimizers/async_replay_optimizer.py create mode 100644 python/ray/rllib/tuned_examples/pong-impala-vectorized.yaml create mode 100644 python/ray/rllib/tuned_examples/pong-impala.yaml diff --git a/LICENSE b/LICENSE index cfe48cb8c429..cd24136c3d3f 100644 --- a/LICENSE +++ b/LICENSE @@ -224,3 +224,22 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +Code in python/ray/rllib/impala/vtrace.py from +https://github.com/deepmind/scalable_agent + +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/doc/source/impala.png b/doc/source/impala.png new file mode 100644 index 0000000000000000000000000000000000000000..a7d12e4b5a0f9135439cd659533b3043eb9a2e70 GIT binary patch literal 99635 zcmce-byOVR(l!c7a7h9Lf+rAM!{9Q65L^a#39f^?Cc)j^-5myp;10oE2lv69Z}L0e zdCz(OyldTi*R1ZD?w+-!cI~ICo~jO#lNG~4CqhR+K){j^|E7R|@G1iV0dWZx8Ggsb z6$20cg{&_n_6_0r?aDAL_>bonthyi5MeG8 z_nJqom-*(VZWB{((`*(iqe1sb^=|J8(kWXL*`&D+L2HnBDZBgw`YCe?yT1xc!4j~G zjG9YmWuI~W_-JeFPiXvQ_ZPyRYrD=|U=q){z^c9bsD0ZGPqKpKN8dMJQU7)7re~EP z_`kp5cLS#>25A2CxXS#~?Wh0zStaaK#(%E5i=#GU{-+^dHI8TxoPRg#`=&%R8uXtl z{b|e6%}E#HxMctB(t=g*8kS@|Yyt~Yjn4rAvm5u?wbR_=wO`E_`)0u`jmaV7>{w+t ztX2`itxWJg+cuPukCov)hdNI#O?{hO=k&M!`P*uKs|CA&!S@Xp-w-l-!mcF`HlDn$ zZZLTL&F}9+P>(%syc>(65(nxQi2N;~rI`sHtwm{`4O8-3;pXZ1Bexq^iJY+=?M?|U z=7wnrMkLmi)!~FT2OLTCqB4d)+mLQSFIB#P_6scP!pxSo@QoDy22CfPimpBMkKH3y z&`%lunx;G*3t_LCvc@|;j2z{BK?<85!y zj>$k}W^3fRRW?amoGVgNo|l~uX4vRM7N(ZOTH8;~g5}4?`}83>p5=P>@a~)wtrEQF zf4DZZMkV_?5vb0LFC8>lGWnP2VkaaWqu0m8)pvlb(rwiU5u^oJ;9hZH2QwU)?Ok{CCLVXy#S&1Lht#pNXx49{ zL$X|Q{P(QNL33X0mVS9t?2yliO~1=s$tis2ew&d6<<00aZk-gYm|d(tMr5$BIBh5{ z<(Zi#$26(M?c{A_+`y`0@e=|%u0+ixF`#`f#cVmkj;e^{z5Ewba5nI<#BB3yL$&g! zp{-0)stLUjv*Dd~<|(`hqIqS5{1y?p3CJzp(#E%J=;nu8(zt26s z*T9}O*yH9@S57d7=G0ds`ld(L7wph;Y<))5B^?(G4D~%`zkyx#_~R>ia#pvR2G#WQ zhFm`;X0;YkfAp^ufzFEz?MoK8i+uKZ^L2@0ilQxDU=STWSVV`8URC~B z9Oye+I^%ntb*Nls{t4Lg*|C)3PD7-GNb0r|(ecBb020>A!2*R_w<~%+S_|a>yS^vx ziQ?j5bR{97pVfVfQ5ERu@ZSSo(*$Xe%{iSeBW9)xJe#ZpzUydypKyhzY@Gm$)8&bV7qub{9HeuY=~k*z zRkY1QUcdcnX^A6M)}=Pcm5Hs4SM=-Gi#TiTxr7}8yux4E=#C#aTj8qZ3pEfC>0pB? zW$e%C)^2{zFU(>YIkPQEe`z$(7uTH;ja+wQQ??{foMpJD7MlH7Y{3X2kqk9T&ZzWy z2XE*Xg|ef=CefXM9wC8p5zYY{SVFzmpx8jSgUwbii1&~4JCy!7W|O>F>)pbIn05#9 z1%~iK#ML;^^`L=&H`r2F5|b7V(ji9Cpx)7H!_|+|3*%Y5ylzv0W860?>IM@2l*J~) z1cEl$wZ~z|a6KiT>CkukDbEKY~B3t7yBIq92-r7HVogPisIN zemjmJ;?kBHnR{Dg5ZT_xjtOMDg&(VnzamkOMSahUgC!e%-zPAYMWjzNfP^JQrq-)5 zlr7w;U#zO1fuJC+z6p zP?%nIBiR(`!MIQmeicevKUT0r>A9~a+|!9Rd?^n6*{PF-?SNVMIisrlHOXRyWb}(~ z&BeS^dHzelyzB!<{zPdBNu3_fT+ykhXun%9sknIOC8@mfm8tI0kvex)bG&EZn+EOex7KffdPUDs_B)) znx@x+?>)!anCD8u9kA%&)A2BPSq%OCRXTFXD{qO^J-cG7L_q9b$rAVdfra+&5IX_1 zyt{g6emUog;8QK%OhDV`>DeZh^dB`R60INV4jn4!jU(ai&_+B6sIJ^IKBZmUY*9g9 zxv1Ynbzs*JjT>^YI51uJ5%Qpj>xB{=dc81-R9dn_(gN|PD0)?vjGjy03vkySQx%(RnH{!i!Sk{v?DeZ2ePta zfoGQ-jG?H^Jke}K?&~jb2ac{T3q5Z!J}XGFTpos8gFTcbO$`R3*vH4mt8Z+1ZC#l% zJercdK+?bR!nZ$%mx#a;zw0TU7NxgdxwP!+P_Ha|W93XGXpX|by&72>lfDuLq*dJ6 zUygL~0vMa0w3LTGz#ZT2GQGWD&>6{Oa}33&!$PaREf_0sNz_PEhyum0r`O~KwA{}< zk2G#dRTFU+)D+;Bx^jh!fHyUh@40#4;*2qd{GlC^epcu5$x@v%EDsWU%UB)?r0$Jj z%}BRO&Ok#@xgwE0N7Xdp2%A><@WuQ7yLoH&x>GU;LF95O&!Zq?-84%sn21NMwc#8b zJHw^$%BDI+XLciKM^Q&T^{)TB2^Pu$Izr;Hc7FNq&e`q*x={BAx<`fO8IZ0nx#tMR zWikIFlY{I_C7VjR5lS;~-W2&t2Y9N=_NusjtNr=##N-4ZT}7 zRDU6&9LrBhg#eLtE=kV4e_G~RBJ+@S#Qo^E?E=m|Vc7LG(Se8=(-v%$DZj{@B%)8m z+B5m;UA*YMJ&EeV0^#ix`*fs6&a{a8P9ZG3$!CgNQ8jX7U-GPC_R54QFQ479i-j@5 zvu~r%Pi(e(Vt@C+^?i6nfodZb&%v&f>uKnea0$eLbru9zO{b#DHg{(%l<`XsE^E6QD4oOsQDRU z_#_zvzF|$z>W;0e!d+%%`Ep>7+v}PQ=7t$Bmhyal8ELr%he&#h+Q&zXst_jNODJ96 z7foB-G*8cqOUDVd2BIOG#H)DRakXvMy-seE5k2g9gwTn3JEW?Ox?aq$sJ6I(GRE=9 zaB@{^z7d(Qsp%O7MMYs58LZ1An#q<~AtO%?l@EUwzgi#C0#>9JZGXHlKaiaCNDCR= z-U*vMkg6)n39sSFE`vqjmQ);)9~-~h zLKtMo_c=6=adCSY3Q)!{WFWJfs#f8+sI5$+?FSOl7LT(5bs0~j-|QgO1jWTE(Ua4! zYXRmWPHW6f}g`DY?dOToaP5x>zDJf~K6X(`GkY5nQ^xd;1`c^W^xh z=~B$u3M@YkQ3a|$?UASsj`j90Xp73S^2>27CV~P2I%J_KlI9E&i>>KGO^eu*YPw=z zu#AJIfcSIV7`}`DXBA%ongtG0_Bi2lC7z41imGLS3ih}^Bn7XMRIW? zc`=t+78Lf{LbEu`dNaXahV# z%XbmCxFe3ITpZ>pC$eekZc)t4h>-?OrJd1w1xIZz`nh1;eEn|vjX@yY@oBu2nl~t{ zVJHzO@exYsMJI(>#)k7kYvyA47^dSSCjnr2LKS2Qke*$FxXu>=v5t6n6u*^s6;0X5;0 z2LS%XMbY*GfAdAwv`56uAN#SxE8NR6byJ>ux4VA59cL;-m5D*=j7;JG1hV zttu1ItwMOb3T*xqlxO7ux60e`Ftt#JGuz`?GF@QOTk-*moEsY!(kG(*Hn&vhs!M8w z2h{erth@*o$K@saOgywj{;};rgE(THABtg+Y8f|j0l6zGdq$>7_x{!Q_xSM;O>b^~ zP*hQoHB}|Kd`uWY9g~X3o3KD_$sLmRn>O?{^z+xrce1+r$RB;h`y6^Q-k;<`mQdqM zjqZ6CIDQ8%iq(4I2?qqn2PYO*RJ}xO(!7JMWwnGzzLHvb1}51*hF z&bvNv?IrRLihc75U*RVqjRmfWX@*K3l3p8^12Q-MG^O$eYyJy}1KZCyfw1Yl3Z;3H zZYy^d9=6W!P{+dw2aC_ctL)evS(_8+P2fgZ?e{vEUs@w6d03LxK=xN)?>1d*lk(pH5ze%VGTS3@V z?b~831_qW?$Kd{GZQ*Pz_bEv58KFUBcvtr5Fr!^SyR;pOYz_^zuKg+k zw>r1?ukBB75#&0fR&MrO_+X>WB+<3;-s7{rn5DZs++{{He&&#D0P7v zI>$e^BCQBbaJkx?^vKPRo#@DaM@6d2;&k?g!FEgBnJ>F6aHjmVGJag8IhENSZW89Y&)w8j@@u@F5|lQP0*o+$FWjT=yVP~Wq@3Mr#c$7R zFDp9Zn8iwO?%7+VdfA{CSE2{s;>#cKMS}?89G{u0$70%u9{(Wa8j-Ys>$$_^l9yd$ zV9?#di|@%MWZubWnT6%lVB4c7W9Yu}^gFFqNn+fdeM}^Lbak5Jr-0_>3SEKJPNMy6 zP_L<(d0$<(Py^E*9S7^mvL(M?etvo$l!`d-a(bMmlf%{}$fv5L*(fMe?^j0$thQsJ zBVO(8_QDW#2LEHg?UX?Ttmw|?qTrTDELlo8d{DIb=*yZqHrX*qVD~P-jPIs43ZbM> z7d0|1Zu}`V|NMK?lf!tLm1pzdmW{Ty6me>t5&;4(X!2s1U5z0oV@^>GEbV@Q26Y@P zWN>B#Dh5*L^zJ#kD$L$Px-N8_%`qE~Y7_C?T}XB!$Yn);__7aA0D_fU|) zg1CKh;@Gwg^Jldsd|l(G15&<5w@DJpN6dQC&3S|`gmqQH`(6a#un5>(0a;#^zzElT zDmQrcvtmqjYDiH@1iq!URW?A|Wyv$Liw5DWNWECjJNZ)VA2{;Y=$@?+Qy0=JQ_u#+ z=^v`a!FUU!AN!P612r4_{FTP4UKSVszTwNH<<0e15G?n2iBpYkjXcev-?Bk`pxB44MC;zdWK`^E zsh8||e;(GJP*+^VzREkJVceqgCleXxZeY1a5a@lY9H2BC$p;+ySG-<(vH<|*YYfTB zWfCEKBqYtgb*!b}t(d{7Yp(-^^#C`>vs}1&(iCC>p_B8Rgu=FA7ZwXs_i0 z%xObp;p~ekt(j1f|Lz&1h0N~Hs2tedmMW=&KvfW32GJgGF8yJz#Z{{Sz3q4yNzfGm zJi5&k?rG}bn#c)19EhlRje>vq6CPp?MtJFVtmKTJoEKGPF*_F0dkvSgr344I@=lo|)sU)2+J-qsvej=U}Xp(FQ*s%L}w^T~Si)wHplFHUxud z_0sB9NaRRg8VnwhDGCdenS1+r>hWk5z@z%V^t!n~pMqoQ=bX=43lE%c98>=})gwVj$& z-vMa1jss;U+d!8R(jR;qB(!bMB#Pk6JeXX0F8wf|`tIGhPmrheLX2phKR$LMJXr6? zLBU|4y>BC{i+-$Vwy=hK;fH@BsJ5jvN^oQ$1dn)l-*lUGXu^RUQ%yFNGS>zb+|{Q^ zQIx%i>{Hgd`6qDs3a?L>SNLJ$QbU2jD@?ichiYZO7(v{4PT2~ncAbH z$DqH&pK)Y!Dg7xxb^A>-w6*Y8Ye3WceXEvHkav~5)@+bPq0dWqh7Irw1|%wXQ`4ia z>BaXIFxI7&U$9r3yeb#348^Dd?zJySmO;v$58qj`^;bVOi6Wa7NpLj)Gxi}ucj3nc z#k5yE3`j=qJX2fQ&226{V=VSF5CH#FN0!KWg051>iZCfAR(;gME0pPx$501SJzbHv zw==#@Cj#I4{;(dIg=F#43NDSwIoyf};(@POTV=K|JZ-Rs^y;k6n-(~^B`xT~8WwJ^!brRhbj7|IJ4 zIeu5Ybf;qobbd4QfKz-$nq`rju^oH=I>hXdzxOG4j;&TvYgb%Nu>7MFy;ZjJU%7`B z%=CnpDBo)m?pjN-3OG{B8hX9~d%uITQD2Y+QNVqg)~3$`H0|UOt3D;yte1{s zxW6T*I~=;Y{nSDl>XK}!^hz!&;OW2}T$`Z0?y{mv+xOG7j~{&B5HDj%0I|3d?4F?* z5m*qFI!5^farj*RF7Q8D!IYpqQIDXJb@pH!O^m`h@6e7>-bMwQe^S3yv42y(R)}0} zYYqQN{x%pT=3@Nw{GU^|`TvvDu=tt8KfV8RDkb{6>HpnSpz>eT`S%s-|BLzP|F1g5 z-}`mBH=G2x@n|zzHDHrCmV-~Lf$X^TT4(RoM_AHUQtZEL0rI<3;G$Hai)SO}cR9^; zn7ji30K&9*8fT6bvqKTy4+iKM7_-XF0%Wf}Jv~E1LlM?&xM!5_Ma9Gj$;nL)X37q% zl0dSucH6^*mYl?dgoG22luYpX`FYFjcFJy(tdkR@_K4b)`1e2zO;SoqmsP#gqE$2` zEIgbI0PvaIi$(Y*-jcL7RixN*HppOsZ%y4BhaK=TWdve;j~dZelYd`v`KC*fI2UUI z*{rLsTY>G&EUgI|&c{sLeKJ?&bAQ}ytzCEjaC?r4iHT5YG!*YR+cp<;)a5_c&HS%1O!q3Vg{%Dkpe?-=+VGw7e+w9F?YU283D5Ts4o3ko zCgU?LGXGubFt_GLBTVAqnBG_^d3P_H5;P`|*hQH#%Z&7?e)&JDY$>N>v;m;mz8MZ` zjP1t#ov2&zl&9WiH`r_pj{UDfif-h~;g+w}Hc@%8|6pte0|i*Eb+(f@91io-zjCg4RNK7pzR|JyiI=F*$1TVlaG ziT|$7AdX}(#e1t7>{}(P!H*I*DwWX5)sptV=J+GOb97kOD`q60MGBsXZCvzn{nc^C*u;=h1KpA=)yrkD6*Ch@b^ z1{OJsv|n}wQ4o!s6h0wJ1Iz62lY8|{?&E)tT&+O=GR*-gMo8pbf+c4K)fAnHx3`F1 zFlHo)=*Z-SSph7l8GP8zXECz9o-YfOkM(@k?-Wu~|i+y;1$c{a5wAfq~KpUTJO z-=n|%bRTka)289ImoR8l^0(^aUuYZb@-_H##^NsWif8Lw4cSwM({k$oD!tXi5s^Fn3--_svRh(@VNE?N0KT^4x_2JoOqj21wGi-T1NOm=TqMy}MekSKfJhTO2%Z zJ>To^SBnmKD_YKoM$>K&7q+BbAEU&`g1u%LsyY4#%P1sD zD_ztAP8R;Fa7bWa3WE6;cc4LIp$gP@D zE*sa+F~xZ7@45aQdx~iPes0X?ZGXXFxd@?pO3r6`o>wa?*jg#jHm1Os{G&3p0x%g} zJHL(hSU$M4Tkk~D@qWB?oF22Pch`H_qOOTDLLON4WU0}9&JyzOWGoLXRey9Low#T>k|<-bxH4G) zs&PC}3G(oR&&9Tj3OIfo#0B5#L5)7aM}#CEtO5KXkgT2ad=QW>&m}uAuEO)^gqF?~ zL-%QrV6NK8e!|_(&OV!le}zicH?*B4p)zo}lYkg8?$~fJv3F25yj6z7yUE0>{X zfrHilrXn}Vdd)L5#N)(Wz~iRN{RaB{6rjdfR6|k%b$D8O|IjH+SCPTviT$IqlP}?+ zoJo^2CME_fO&JC-$)0RD7=2gQT3Bd0KMP9dA0TyJ5;8N}YuZ0Q-x&k&^JD&)+|zPC z8X6eT^YZdquEY3lX&D+6gm&DxLw0s{X4MGU+gZMec#P~X`b|3sBo{`?1S2G4G4}2SM`t5vo;9jXM>D zyNLGw^yF(Qlmj3+OZx+T{W1i%z`ief@(+7heO!xQ8~O9AgaZf3?0uR(-4|<70p!t4 zGJzs5o==56K)`@x&G)c3JRG8%Q}44@W>G!vsqBE)DTz+t_)(2M{JWUtqwD(kwJhIzIXy@Vy$trd7VymS6G=v#bP z!K_`k%SPe1jIny-qf^lnOSK{Syejz)xc})>s|T&u^Iv>}8rO;=PSxoLHqhv(U-D9qbjJ_Zfl{>+ zG3<;NGNz78L#Iki-)h5psmN=7vKq@!3bNRXQ1(-dH*1fK*N@&&YF+r#E~R>DK3=Sm z6l_!MzRfAj|N8b6`O9RbF;ndU72F?KIfW%~WfCWySmIQ2oKn`{ASLzvONJ(WBSb(;)d!==&5f*TA;sntO-!MpX`u_u_xQz+M}1MET& zVt3N&jCgwN{LI#_bIIUwGJjHY0?pV+J9z*NkBm%CUy{_+*n8X#>HapZk;tX|c9?s) zc{@a%u0NHUaXa>m+o=0=Ggwf8FB{mie-+S2K=FQ@h3{^Fn4S4RdISYJPykp#{fM+! zHgI$vP{G_E5mw%`i&e$JXW)3JpxRgqwOtB_Q@$Y%n z#O5V?Pmv1SZ{a1)vJ6QYBuz@>NW}6_!Vv8rW$jC_l|Ctr%W&)5eVg`!<#+cUS_?qu zLfm)m!}-#P>tT~SjcdxesUU8qofvh5C8Qsjv73Gl0$!}HF9k_USDH;ypPir2=6VKg zu|X{7tBqeH2WpS&mUso$b?lr={0-#%u@?LTe^q7rE6-NG=%Z#LxlaDWE;m>RQ!<_A z@o)|#ncT*jf1R*guo{XofkBXu`YLqY3?y)Q5q-kI*aArRa9jIH(FSw&?!sjPdpzH> zuyS$1W4haiV;bOSPJ|abxUA5j?WRX?s{l5Er2YsUb#c<c;Pk2UkGMisLPJI9r!W2+_-%J)UEj_ zqSL!?RQJGpLy6w!_!PbuNs*1WST_6Y#1$b(^8CE98AFp>nZ#j9ed-tX&7Z|`$&HKD z-?ZF+FETg+5B=Ns3F3`%qdRfcU`!b)$>VPhBCNdG+o63CUB->ucOXSXKZ%zqgr(zV z+NYxZ1Ii8gFC&vAbo_gd)Tx<~9D5&v$z}Z;0E@oUuUwPcEO9K^G6C;cA&4btoJNWL z;beBz6(5K>j%SmmWx6F5bP_RoaY6P!QSaL?iO5||X`Z*9j--+BytLOtXh%o=cs_sb zvzXNLH=&e9b|9O#%zDxZOD)CJ$(ZF4AYZRlHE{rjs?z9S0=833^a4i4+|QV3tq}^8 z4NQ&G!6}`F#)b-zo;hsY3!`j2>GX_^hl8DL37Se{leectIW^l$HK}f9@ube1_G05l z7dohtXA*(Rc8!LkB^7PK%T~HIO@2X?VgG2zQD7Lh=FnXj-0lyLjY<7WMdtkFYTFdc zvGY+MZY|D5JqzaMBaHFzpB?0{l;eoiK#w*@rT1AEV1|mI&F-29Juk zzVvm|6$uFkOv{h23ZdW;87qjjy>i#3I$#?p2>f;KMXKGAm#J|!Q&2*PwmBPSwqww= zvH8bR{OwO8K>~SJUh<%D=Ek;q@QAO2$J>5ZIvhp2b3Ku8qgzhbuHf51iqhZsGQShq zH;&G4LJhkuKSZ;Q830LRkQW&ThV1p3yHkzz*i0Xm^(W?|&1x;FGkvI{75F`XrHSbD z`7eUUZ&4M6C~ePI(g^uCE4!@^UcP_5+XCbr^VGDAog=_@UJ2~XB^+S=p{Up!m;6PI zX3Si+Nt@9LL)(K7joxd2MD+FE&E*@xv?~D4z#3w+pR?7-+LUD?xwQWVjZ6UY1Kv&& z?R?ZobE>+M)tIyBAUlXZ_yPbDsPHp=QS%gH@^CVze zP_GcO6a#d^*We&w)gLzhoS~c@!&BtZ&K}K0~D(dcdN%|ff5H4a*tgeYEb*FYCHJKe_ zdkNX~L4(ggP+C?JY7+J7e(>wV>S#R*oqn6)f*68FPqy53*nVQ1Y}A7~Y}#H%HfyaYaPz4pUTH}d2rrYU|{r(fEG%9CX1yXTX>PaLQ> z&TpC|HIC%JRT9;=pQz}2{jf6cSHj*%XiPTz8aDom(aW*fPYN-OApfgG&aa0ReuM5& zk(@W>Ont)^lk2hLUxd8hXwSNWHFoOu2-{q#f0L%L6}8iZz#{~;Sbdv`zSRu0vmB|n z>@ofWu}zOyBi~#kU&dw+m)kZgL&1F2^JV~fQ{~j8D}!{52|*Kmith#z|8n4>OR(nOn<2gg>vq5zaU3Y9xyY1k`^E>eEL(j_US}V(AXoTRku$?DH zPxN%D_xzX+I=yHsude%KpNBZC{3uE3P!9HL)XbVai4`~h!(wO7{GP?xjbj7epCl{> zh;j0q@w5w=T6(pI6;2!#&KvcZaZp`OD}7KgC4y+md?FL=eSjCl?lCz5)=hC3ve`?i zdwg|wP+~9!5))G~!*u6Th2+zd0&?ofkN%+)L)`>~dZdn7S#;2Qa6bhdkAAlI*0IaB zq(CV6$~mgd20O>lIYZT(bPPTp#@kGEH8f%FI_DkJMLmzEiH%kZas~5BAJ`<;Z_yo( z)z~~!V^~?EN~Oo0gd@2)G4_X8L2D3oe=Gvt@r(oQN5teHr8ix0-Iy7}Q2Yo{chG69 zefcbcHQA2`NsZG+b2DK6wMrbiI`|Is0_m65pHeOHntevE!AULJ&Y^Tx7SPyPeCQk8 zURHDbh2{9XuAtoIBQBAEObYc0;9;8m{yq&9K~ zu)Oc!gZ3EC*c{GvLedsl)ulu1xxaPt3Ydf?RfhYqYBHMYkXt(^LWh>4lVo~5ki}^D zrFAcC+;-XBKKK4RaGm8|Wuz`IX>vGnE)9~&F{5THC4BYx^M)Y9 z{gDK+#8`b^Jut>*n=<`voEp$vJq%}W4HQlvl;E(7Oqe8d{S9B__d&r}tj_>ulDf*T zCj|H$Y7&gAwhW_c=rBS!Vr)Ldp6yLmWS}?=glqie42VS!Uoii!fqqOUx+Fx@zKFi} zY&6COCJM`^-^TCZ!ZR1PAsBnz&Y|CzienS>!U<&4vFZFhs7G(gTKL(Au%D@hCFN61 zwx48rNA1`OUGltR6HGc?e(}Z=fNXZm3|{$-B@Za+Fwcl59vvsec3|0aGT5tg_Cn@t zceW!*QBUv6Jv?Tl+b%}~_Pln5j!D~pOt4o!dO7jgSG62NnVc}1&xcblk_gXsCU~+9 zhECazXJewdKw)<-=InQ`nVQCQ#Ze-SR32tu5TEP$vApoo{>+ZMKD~!34ERX>QndOT z9c9s07=DKrn0I&xpaQJn!)V8T29m9r0N&~};TZ$7X@@`OT%gvAKu$*q0RqsK*xU)p zkpR3_f?pRp;^`63Edbn+Tw4$N*k~I`2N%=yF6*JKdDt5k*RQmBHe)F7lU96rgrm%` zpeg4$Q=RObJIH9J41Dj?GcfGDWXA=JKasJ=1eKByhXho^ePDTOJ3c_yTSh{RfNYE> z)E!~HU*np64%%X~m$7e!dIO2$w)qsu34wwFeGeD>h!q)@?$}pMSGl>P`c36IVh&FR zoW28i9m!w-j-lk+Gn7;J$Iy0LvUZl=Gfb{6=Vi_~mC1?0%s%(kYs-9qADC_SXB00@ z@@>aZV56f31H-@L4Y~x$KafbvObxqkbGz?j@;T3qQ}XI*N7Z=(=2NPZ_yIxI#@I1E zo#A}Wm1ZSr1GcYebXvY}H&j}I7pj9sM$1xpTzOZ>scX=rj71;#gr7bXJV8Ib+q!=H zp4h}~ut7v+k2e|q(p~<|-5YfO_rye5*W);n&UxfN9h}pX%uSHjBjfA=gwl6mV>+1h zPup7|kuiSd4D2$LIGhU(Phsa0M>E6ue!(e0L)N1-eCB_77E9DBK1So59aT`vx)e9c zpFiv_&0U?UlJ*tFZ83k};XD}3%as4%TD0)hE6t@fMQuWVs-$MwoPrq2Sq?Yjx(@7J zOD;dkYV~x-sBn z2fUTK+%bQY0@DEC9IvaJ)^iOF4mqz9f^bMs-b;{lMk8AKsgt5Iu6{Pd!5S&>z;p{; z_*&5FHh82pbUWIJu*yR&&bK@#p7rs}<(U2VJL+_HBuPyt;FBYN!*28R*IBxaM{c%? za}>d}tzWt{c$jao{h_!)zF&;!V<*PDcx%$t4;K%IUv;X1iw0<9e2xe!DTjtr?A>a6 z73@%GwRK~KS<|!P6PJQ3DA!Tzx734Z-+@*b$-?l^{VNwrdry9Ts0UY{mE0yN9ru44 z%eL*PI3KA&e1hsnT9yR;xgcrVQ>PXH_U6an5)cv^7q46*Slez_FBh+0`kf+`!e(h6 zc^?689#hff2d`SZN_jX}(sK*|3~fNxU_m^Ya;@aN`jb!Ik3Ugtk5`7xDn==1KA`vP zWVrjBV9MyRr9dSzZCJatPVEkU=^y?wS+*Q^%i4TM%ZJULGzXkDLr(X+2{#%Q6@Tcv&5Y$dU(gF=c)Su* zG2dP5|5KSXB0cJlW^8Ub#tt9|JOv37ZtVywE@?ZU%rFugha|~;=)UC|B?>C=9nluK zsI6boPQ~h%)&2(nDPZipHXoBPS9o5Hvw72J7argYzdz;6u}oR1zI7ea32Za<1zc59Y%>MWwi*uuPbnhRWaN(5y9 zZ4sR)s;;0U8}rjy0zGbDU&|s_?DNG-nrc729MavQwN<-u$qJLM>6Ic7Zuv~GPUZaG z`kVH4GY6TInu|+_pEf7Jo98gh+nDY9)QWm4f`Vip4rla76K^k0ooCbYa6hq@Z8df; zBQfMSJ~mgE8b2=P;r0G%R)fK{jMdbw8{edibRxJS>a8As;A2S6{C#KVlu!V;XJ&|B(0-y)qE)`lJrtqLdP+9 z9%g0bGfvPjA&zRGE@!t`qt~T>m#tWB6hHu%f1)7Ga?l{=5f8_aV+3m za}Hr#*$K!I%-C_LwYp-l`h31Z8thH>euq4OIwz>W&+v~W>u}By&CPzmN@vAVeO$;f ztFCKDPkliCm)Sq#5`e=|aZDq%N;Sr%6cghqs0s6Xby#qGqKgLoTO_#_rjRl3T_K$` zG}xMt^*X3c0FSAj(5o>vrblgp6EgofHat1#^J?(jI;@$+`lHDwk1-cA}m|Ks_E^;NL&v(py+ zOSvMPOwPyo%IYo2;hDQYEb82vL)^UwUSC`??bmMnh+DH^`b*ziOr}cCH;GqVWXLEb zqEb`mcmXLFCzbQQv*uf~IuC@_f?JZjnp5%(_BHeTJ6kj&&M>YM@7E^v))MyEX&(Mb z(AQ@jitc6@kR zmi5-}34TeOCz$=xT}mHdNJoW3t&BhgNIZN?gWYK-8ErQ>=m*7Osl0mumW-muql~vl zuB?9N!HO2Z8osbbyu;G$J=(Ybt_2XcrOs--wq`ru)mArDXZLcW8$Oz+h8UWvjSzj* zUY~u}Y-vn3SIfGwo2JPKqbVQp4ds;}c*!f6i2j+~V0f0ZBuLlU_;jGTW~0mbk@uD3 zO-RJX2CAy>eWT`u*j|FDP4%-9*I)q0_ju#<8&fu{%pL!OO zPo5vs0u|t-Vt@7S;k@3umkiXsRk4?si(@2h@tn5Fg0J1>L>A}~5Y}8RotlBo%2~i2 zw`athD2GA!rWZN{g)m}}w~R=RjAAm&+}*AWDXP_K(PJELYK(AO2A{OsMkS1noLw!R z%_SvBq))pvr41c*C7Bq>&fb=gr_0LnGwDEnhYAiiXShU#N5BhI!Y$Lez2XVRWY5RF zZwNEo$Ci~=YLc2g$l9)BY7j7A>Po3E<5NkohxuE#4ra9OSf!JZKn*X@+!90mm6o>F z%J|OTr(bM@<3D@-hQE6ezdFjYthY4iq{rMp4mH{H>>~9xU%gF2FhHa9cCsGXo)WP> zyNWT>J7HNGFs|8X7Ub)a>%41o*W?t)aN*k?#w4>bTuV#a8F&r+O#`QP3aC9LQZ}Nl zPx6oTav&2noiXGdKd$Ss-b;K&YPD{1y~xnXyW}Dd^|oSuqc?Q~JuaJn)yca` zn9i17Twf`{Mvi&DU;#%p(nd^rPdzm)#o0i_J=TOc8~kO zge?<)SwBB|?9b0o@%JVlers~=;q3xi46bF*P6kajU$=K`=J{!^+P(DRT&+qX%_!-} zVb`hMIxG8*@0_$IxCod_Gua56M-@!L2JBDskUp;P)r>sMekDF}(|a%VELCW#2g4NT zEDuSziR+mCdsHrY6Z3hW2+Ub88g4Q;+;k`fsupz3{nFt?VTP?N^Ud$|OG*4`^{nx0 zwjZ$^Vb>e`vjNIt7-mD9hrLpQsONm>lb%-r_|1z76m`G&k?<=xv)pDudN|!>(9?1zYUck#)Ky1SwSC=7O1E(7?rxNBkq!}%?k?%>Zj=ydkdSWazH~@;cT4wo z@V)o@#<>3F;NE@qUVE-N=URIoi7IeLxKia(XUngC#f$3HRU0%;(&nXs`VsiBAWG}% zZ*JDEJaBU~LJQ6jnR?$gj?is(@;{qqMA2x09U_% z;KkCdhdQ)|I@*ceOE;f1e4DE8MF7%!?srG~xH2f*D$sGdDG<9@8UDsM2-pFT71hsF?(XzS+1vwM)CY6~@LN^7-h z)$@>YaPQ&b#mUGKrH_q2A0H1L9uu_*o4T6P@GuI4OvZ__)$ zAtY#g#c*k8?^A}#*v1YwSWgtws>hu8(9=usXs4>CYD{Jq%UA$fP%FLUWmfzlMe4wc`M;gv_i*F4vLpo1&pGv3+&0^wZYl#ZNpKs zSJZi9oDuIDmMg80wMvCcEU7y^PqWvC=4s;xPA6<5vCr=WW4NFg7!sJ%KLqiQkd`8A znZhd&BFh-pI>V4Vue3F^V6xg;jrFwbI$Ylc(1W+?^l!Jj1Qn@4eu3!*P9aSjK?OI? zcZ%dpq?w|$z5!$i0b~gYNnwabRdAYQMh;k+2BEG=OwE+-8qDxQvuVH zIBq7G%fZQ=b*oq)5nR7czo7dZf)q$6*@{7Dz9w!>sELFz855&9Vn5wg%7;qRWyv^J zewfZXMY0b(A`k0i982oGX)+< zFI1H3NU;3W_sKw2z;$yo;$Hr_51X|r{cH&u8#`!ust8NdyMsu zD>x{vj1tflxo~iH`nC9`TY54IEI+9Xo|-{wKWO@7zq0V!?Q|gA{37^t0v*kw_ZW0> z1UbuWWOCAL1A*hQ>=T+FHZ=8z}T`a00?P_9D=4m(_U#|Q7anfG?d#&0M zayk^1Bc>7j^qg>h=|-_xwF&=7$PY0UoRZt-vU)en=#6kit86+HG#6Oq*Mr58*oAk- ze(cZ7J-6^{h#-!1T&7>v)?7kxN^JTei3J8oarkBVl|SjNqmmve0Yz_|U&^mLlI0q?nv4`ePy>Nguo=xu@Yc zr@^%12>WP4Kjn-Vg3=h^M5x=gWi4=ZMqZ0jKJ!F4`^+?D+Z)OI^~%Ct;UykmlLw3H4$Zh+S+HGuQ|iGAu| z)o{|}Bz4KbKx*Qj$h~&(YueltJJ}k0vEnD?vD3|7FgV6N@;n433Q7#q1;j~{9u5YkdxJz9tyz~HE3KS zW9VL`D%QYU?vjw?G_*-!Xb4N|gWv0cE4yjGE7f0U|g{qsl#LR8Fly@ZT>did8HBV=Pi#PXUnh*4OjoxkH3+?pd}Sm zX8%GCRczoUujcSdjWjRi^kVfG#HS3Ed)*j{#T-6RSAC-#j=iNT+ZKRM7$IUg!@tOm zO8m~#s@|aGDX0Jptf8NYX}`#~9#}oDM6-&ayjJju;VUj*VS-MF^uq^oOwY8yKG{lx zFAFzaTgdw+iRoT;XM0Mo>}J(9)by)TDqC7VjjBWleZ75}V>!I8zFF1hT~xC%KD$Zk@(54M@f~TIIIlf+*xKM!gNlICz01Kw*XF@JIXvGJeri!hi35bNK=Lj!CbkiY9N4x#77=%({4w%_)hIFg?JNg{6H_Mpvx{R$2E;OywSr z&GQwtK#v4r{fA)=Ee}ad0hiB{D39IKm!CATBRGg!y7R$s7A`do@vUte-Uv#MJqHyj zaeFo93k;EWWnb~~15LmHuC3KoH->?!XN!~jn-oa`LxuyZ7pwaRLE;j965mOD)fmQO7$(!YoTD=l0A zD);f+r7BgPj`cmSvGCHhORz0$83=jMF*jcgY5RM8a6}Rr&y*9mv2p25e{kfvPM?$3 z*cfqV5L3aGD{kH#xnyuTFh#&LGkr@69qcVI+AGO{s_qcrxjgyA#yz7%QFjwC-skcO zfQ3zu5`BDli>vHMiQcNrh!zE#Wu+BQCKR=dy~@3SBSYW+;I5R<0p~X9u zc=t^cM7WEN3M?;!s2V?RWz$>GBl(FBl@?@|xfQfz8{J#xvpgxF$g0Tnm-U4ii>dfY ziG~R0==zqGZCCc-HL<7+9?~>wmZWZz`xZcvorRMnd>;u5XxInO%6TNIDlQkI7ZD%k z2(@A_F(rNB3tgU8O&}+>g_97Zl$dQGjAG3~X5pl<*j_JyPE}7-&q$(M=kL4hCdJ~8 z&rXar&xv&L>i)XiGcZr^XaFQIb>>Q#oTYU`sg?HH{M^&w>XyQ?W8e%s+(Dh?+*|ez z3K|jbd%OFJHtsN8?hi<(^`8nZ29z8$I_ z*iPkURV6|YmD)|f?8I2G`>jXFOrA1JAP5gTk4kfVpz1dfZj_^pIbz5w934Z7!GNMS zTH!i#yExx7GcEfl-hWOBHGU#lFm3**&Xgg+sqs(}4&XeD0y6(@BV3V-y@AZ&7spSU z%#5Q(oJRfmMwP4F-25ECo`^P!{tbbQAy2ni4c# zld(V-DYk@OZ%n1n3F{MX9sB4wx2ol_LC@Vvg8aF&B_BR8w(pT;WBtmvfb2?ZnP?c- z5;16eMAH75|7*(qI@*+)K(T6%You)Dr_=T5{P`MQ7TG={Rk&o0@KhjIPx)qsugjUS zCY}Q6;kcTMf#%^$tm4V}Nh&WjWOV~OlYFzTk`90sYP=nM*pAkq_)aJO2oB;96CjxR zhJHX*yAk0gTH=rl`#?{Y(1!NhwE9;S+)=Pnm;L;{0K!7=oDrXP>O!w zJAO{Cj2Z1FyD0D#N8Ha>Iu!gEaK>j|KxF1&Dx|52&GX@B>wB#Qk3tef|F|H#)Q{uMAUSR3UGu^E(mV^ywA;tMHtFh!Q zDil%u{bd6Ps8_Ff0}*LFCwDrp8K@-v4_%6?34KkpX55EV(L=jF?k!sD=~mO(ZDU$d zJQRp;>kn@grZh(^O2r;vJsD^?M}~_XcfdxT1FC;6AdFU4EKKRRK`A#eC&mp5+drYY zqPyp))*oW(e11?#qc-I;c$%N9QB?qCH#EMl|Jq~6OWX}enD}Hznh~P>Z`P3a$4Nd) z(MY7BNTdZ{r;3FA9bUb!Nn46?u;_EtI-QGW;kpxl8T+>U@ZROE;A1!uz$|wfxWz5& zqo>|)fw)!sERNc&2MpM=XAdMClJ4w|saSzo@iPItc^c1k2eImYiQlUMu-$(}_<{5+ z`^!N7+_3ou#r}$Oj~lmXHNy|S(h{0xVrF;xx_clWl9HEIwBmT*ln) zP$awbJLE=3whwTe&JMwsXX!*kl`Ksou(dDQ9d_*>jCSW)X=30*KIDs+1i*O??<|-cjTMx=sKvlpdT^zELrW8g5pp&buJVaDaayV=!KgF}3#22_Lm|ln$o&e9^R3Q_(e|T~$SclQ*w8n7|R*W6T#L z=<>&j;$^$7Z{`%VO?cR9S@%0`l*6-J`z7}R7%`_A8zMfX}rbNz|J-K)Dwni zB-n&Z!YqEK`1W-;V$6I`aCq?@;!CRIbH_0L;Fj=uYQ;jH+s=Fwwh;aHgw^4xPF zGjds^DeQG0QP3h0jeZ}&5d1`^YKU_D1aM@frCxEBFQSieC68W8zH@94iYt_4fGc>B z;atCFqh?A{^>0=q{G-x882$o~VTrLoQZ^O6be&&dHt2z3(elr6sd-rydKa|e>7ZS_ zm#4CjL?&ZlhpB6j2$c6i>d@{9E@XiT2oRTB2KSSFJ^1ebJ0!J>% zLk+#5Ld`+TlzGQQ{t|2fWZz)% zi486Z+#?mc_bnA#R!zb3Zdy9qqT!)Ta51k3B5A{@sOj*7q0P9`x>*_AF2gDpZGkn} zj;~aF!ndAEbM)CM>#{*Wqyex`s0bN)#4WMtQi2vMT9vGLbuThP*ZAcyvX8 zut&3H!b7%t$!%}D8U#v3Lk74mot!HQz0r zlxOa)BL|FW*vOe1&l|0TFtW{Ch5=C#f;a%EPAbV=a-p7q<7rY%j4)0LJ(WWN%dV!3 zRK+mUJo`4zq=h#pnaI5xAn5cO@emma<#y;eoDY8ckpt=q!yzh@Ie$V~NMoPB#rbm| z%V;!K!v7*+d@uFOet8#1`@{F57S>cw*Ak{cH}rGWUob>}Nd&9|){gNy+w5x z2hV7p1r8%Drmg%M%9!|yJhJI7d6hdB&ri=LMotirzTd)+tqu!=H2@*6F2m%Hpqx3rE@Hmua8Zl`T2WC*DC|OA_8^BIx4DjDPK7Y1 zBEUkMAMZ*P5BV8H<-{ZUEe-i$zqT%p@vvq`qA{5>KzGhzgmtPpgHLm1wKm6+gD}@! zUAd&l{m`u>Mt=xE$e55+P^iqNZ>N&9hHMBx&IJTW(0g1Wy0_17fupFq>wz!fpkOfH zRb0uJb;n=IKNN_)RoIqjYv#zM+t@uuenu~IcIyR<1Ey?ZX(`=_;=QYD{Te*S+`5&O z74X+G^g^|@wb{A3FfL8RLc*auGQxYQa+o*xtHW30^UWxUVbx-neDlIRI7#{z6W57! z{Hr%^=p5*GHSQgI8>^~(S>Xq-7xW<}6cFY)!}C`GyN8ECIdn=M#)2kua%`IT&3e>i z7~!S}?CrynxJNfAPLZp*kt(Gq^IIFqg53^8^9u{vbV$ax-A)`%ke1^M>W|hChMd`> zt2`V=y1KPYX`r8aZ=JHErlw{+UJtlYB-1BmuDZOPG8VWWaG4Y-xYFlQ?{sA0B)7E{ zlgUcVU=2Lf>VdmG+(cqpeOBdqvWz6~yf`@f>oYpEKd4!k)pVp8)|FW|?8}#BioK9) z;qIU1Pp@C0O^PH?MG>dPT9u_nyPwcCX6o2^e%Hdo-5?<#d8@S7oduHfIU<4Tg zba9(%HCZlWj&AI8sZdngG-B9SVm2kl_eqKliER^zLEQyZCPOLg7?PDMTfpp|dbH3n zFuboqhTtBqwCTN;=HN(RX`PL>;pp9R`~|F|l7l-#tnqq7OjR^+KhR@MhW^`E4qq~L)g#Rkx1 zFcm4_FEJ{^-M&pt9LK~cwr!gzdGk}Gf@ZGZ;qUvS-2Df;ChOizz=0@`-^>9JUkWPG zpEw}6{Goj5BYA>t%GTZeeG7Ka*n=d#98YZ zH;*Uk>punWPY+fE@=Dum7SI;YTSz+$*^REIs^h+X4fnohsPK6_m~C_>)YZK%5v9{< zx{7tXE82-mP7WlrYsOVid!6Z1^?EQ_I4~lDLEs@WGpdiz#5iy5ir;q6^>*6#ne^(o z)Q^`j5sd)=J&EX^KrAt3-vZ}QH|w*anVCwIdwgBwq@>2al|5MTb=3rp*+hdR6NCDN z#A8!_mk9)r`cH29U?h-?<`pIaECvev+L;?hO~WeJK`O^G4z69z8OZ5u&&o)k>#gWO z07_4ht8(HsFXdQCr1Di0OQ?~dPg^DR!YuV?>!W4>GoXM^!9@5v6RjmboUy0%DIfFh zrlglw!zb@u=+$`@);}#4cd_|zpoQIx-!k{f@juN`C=_7Z*7MW1q4$%d4Q}PkDrC=U zvwZyh^(aH(VXv0=bk<05{f=KuiLB&E(63RKM0r`T%>JEd-`YeMu{U3BZU~{JWvODZbrowK3KFi=t--Nf znX0_*OwP$5FsxmTnjVnn{Hv*K(yCdrzL;NlK|UlDR`h%0>>-c9?W7&h^B}ugfv#wt zOpU1e<9jgw@lBFJ>%x)LZPXsf59WiVYBVvwOLkoy$w%Ec6J!EV?acLyu(6P6-W9Kp zjLAONOQ;x3_Wdj4aLi zxGbqcg!NvZ*{92WiF4@E;~cy;lVf2(K6ew(*)J&2m>Gecf%ch$WRW-wNil*(K#;3n zvU37KEL9OUJ}O5NXlRdi;l0mM#mkH-!hJ)5uPe=SGDXA8N#lPOw|9I-fx$bCR$Y8q zO(TV!(P|y#p~jA+Vr{C;Yw>06QGGf{t^f`VS&$Z>zLQ&x2dzFs3pdMxu9r~ zKJmGuP+9kFA;3>dU!2cPXQQ*JDIzXk+7SPRA^kZV-XHmiL>viY=GYPmnDJ0=Sc>{b zI0{b*GZT{X(W_wssbCxwwJ8^VqDI<$9yEVNBu%9`s&6EJr_ge8grwsJ9k*YhKe5h{ z_r@bL0a@jUD-OuvU3sOY#Du)IQa(N;Bf`(xAQ5kG7!Vn2xt-##*@wn?4;-8*DQ_JZ zaEPdji=A?)0eExUH223gM&}w^i8|wS*&`(bVZEjoPi2h2JK@H*Xz4N+ z(i0dOqLmB-3m-7IY{ob&7|^rk6WbGovu0jsOikF(vaaW|64c%nxp49@vvf^1w|d*J zOELmLXSnO+dVkjS?2aM8sz3KfuhUSObP+aC0PxUCSc%4eH;O17-3&Mo0as3F)Sav> zYtZYm^>tX*uDAGFOT3SGWn!|^zGKB(;t7;Z3^sb@XaAg;r3gJjKWwqut?dtNO=6c>oR|?P1&({o9w1!Dg05s{nM)E4a zOu7!@_ZemnOs5O#tmX5}(*Ker#G_d`b|sX4qQE>*D*{Ygo-Tj6oxz!Vy~juxexzOo zje&zM_=f_Td<|=iCraS0dg6;>gJ%n)ONuiIirlt|0^^ zXAjbx0bG^n9@r`bld&^CTd4b>=e%|K*jiAg>y6-WyTz%b!6%irGQJ!xO{x=*Hr}lpU5GYFH+U-${NlAN=_36c%)K zYs}>%fEX@yM!&sqvHh`hzFDmazy9mixIVykiKKaK?qG$V=W^|TX=|J>V)Gr8aH0PT6eu|2%+XinGm$wt zIYEbmNl7n^h7`o+abX@sz|D{iQP1fmhle=e!1Rc)Ut}d>w{ONs^MGq4{c1z0CZ%xv z#_gPa1TuPNnu|ax7J5_|FCq}orx8~@HK~5^qEfk8$oO5%zXGYm&kj-**J;1bTd67# z%wl1Q#?bn7>`nkK0pBF~YH3U3jnfhpM=&)2Oc~8r^Mq~>m9*>t2)cRAvEE(CFwlWf zLFR>aupyB%qq8ynujlKBrv-0>ZTTOuL8DFI0#?_5%yP#lww+v#r+}fSg0K zWcYL-K6|d*L!?0MC5(fimrorW?s7EPhc2F-1H%tKZRFDuJ!_6d0VWiK$8_)!{)W5J zho@PxHRKC>oy)d}jKzHRInW|dFjE>{GL_h$&r51prN~0`+4xTFkaT_kW-4ap?Is6j zNC~a}jp+sw-fzA7JMwxbi;2xkr-yi)9=73c;O5(}m7B+nzm>U^062t%r<`(GslQP# zzx8f|Us5elfCpJWdVMN45g0X^DJy65aAI+El)|5W_Od#;Y zh7BYF!0>Ldl?(y0b+W}rqUTkKEh)KC*hLIvbz=E|sOc~UGwZYPo2gc^2JVPd6N3Q~A=CzgG}b=2i}3TB z!wychvhl6DRHoRlbo3pwL*dZ z#qRTN1P>c>d}R~q!5?IgTdgPUwBES^mXwnF+O0E!<9@Wb)lbVqZDr+P+Dz@z((_Wg zrKpPwuA4_FrW9XV;9Ba_HEG|OB-ZC8P9+>T3yUJR?bMz7hwq%DOdTbfTOhy8=eAcB zH)Z$sJL$shQy=x9%6XB2N9mX@U0`$GH)hf}MLqJ?T$X#V@4r4h(0e2~w{0b^HXRP? zvF!K}DbL%k&lKvty}- z%6bq)jyN|F^<}fBDipG1P#w`^ab!g*cB!8PBarPc#53++Cbs|DFke*!oh=<7TAw8F za}dzJ%+gUU@7VIPLCQzf&jek=oSaQN`;hRZ?) zQP;zrTD{x@lhmgH&R+!M*RQ#T^RiEs=Ht*Q!@aVbc3vA=#<73a^ON0F+5bED3zRp?(eJ{vs1s6t1jUQuu*QudzE)DW{TM1_#PSvIwwOk z@c_KsW7)id-GGs|Z_0e$uPvpi2q6?QJajVS=5#y%X=>wP{f$`lT;T7{1qsdG)RUK| zB_++9nT~D8t_bdrGBXf_LP&hw`VRCw=6V_~sv<*c$_BT(UE4fgwcaoBE6g@9YhB1Q z0sr3m0qC}MCXK^|z+ntmTzTVPSm{=in+^~!iz4JLU1U^stiNdlGoCNjG7WKd+Q8aZ zdmVrIqx$i7?Tn#GZslf7<07%E#2lRvVot#DEB@NOKWBSf?|>(N`LKf$07CENTfi)& zeCcYKg{X*6gQ@t{B(7~j1{4nP*lw;rQkax8LQzAsGVnJQJoM^ZU`6OJw@wefbS6dx zO{m;DAr||N|G4_17?0TDNprzD_2NH&x?7(~g6RBm2qI%E!y1J1F{w%Zs0iZ!E!XW) z5zZWqjBuD^P<8!Tqq_@9;0LX@zCGgRXUB_zh{E`OU+a__x0g<=Eafo~`q$WzSV5?= zWMW|yvC`;>KtcaJ2Ra!@ZXu{7fmnAQu)=#9b$6^aL zU~CrC{P83(c+y-dZE1B$z_UPmdx`bW97rg>rHoL2G&pq)$U#(OH#&oPW!|hnvx^o+ zmH;cWCs?jL_VuB8?OXJ{a~o|@{^07hajMk%M7ST`<)b=Y+!%2(qNJwczwo#<6J(GDYFWloXnPA*iIVH$|PGKJoXA;k$C+harVDeZWnn{ zXgs|bqr%(X*0iWlfcm>hL=V?))rh%Fh2&+02gzn+Zf~BS_!@b`tS^4?S zW6t$93;v$1*OVGerY{Z@Fyu-*J-8S7_^IBuYw#c=xQpGsKPrb#qiS>K=5S(^{q|5s zqrskpF9QPjP@U+ z_TMz0J}@@y+`FRQq@Fx(z<6El?KIr&Yu;>8qFbKZ0sL?%)0LLQ^H4^cXTvZrKmT{U zG)h=F6a$RMwyI{Yvd;(Vxz%f`BOz-IIUhyZ7`N&(Dc|mjD2~j(aZVOCf$1wD8b~It z?$`A_>h)J5t=yy5)WYkdzs_y1RG@S5@lYod1xR?8z0P&$A~;rEmOTyxLG7-so}mUlT3QU-tPM~AV*+fHU0`}8 z!a0fTV!^T8vA4Ue#qX0V2X)*gbRT6EsaO;5~n4S4*mKmk6huopR; zb2Ed04bc0u+e)U@#nCQu?yAMpBi}heKSIwa^ff8NKe&E4w4duHv*CaGoVi+qDTQN; z$;rtnr>rb{#ip++O3?F)f|j>QvC?Y3=5tPFZD`Ef9L3J}-*0ee^+6(M$^=LKHc*ew zo){|f)jo2n{Yzovn={_^g`0p#dBFz0CG%Ai<}M4MKoRjC?K>R|QoG1yoeof<_%$OI z=o}%ArVH9X_@G?WhEnt*neTk+$Csy}L=nhYUE^#}$n-HKqQ3bldkuU>dHh)nEACgA zd?=%bFMG^1Q8>619)!2Sndx-)Qtm}X`zKd8%)juch~3_GOKE;T12x}*&l5v#mz_BK z=C0rLe~^O$M5j{b_MEPMEv~6W_ri#lJ=@r8I}v)PB|wG-VEk9dZ1vqq5~wblv=gWh z0s#FTr_HIFhVgM*+V~iObn~M|Q6(?@H>(|)>lwqyv zTZc@<&F3|OV*Z?#hY$UqCx#t}2nYz;9kf*x zxJMFwALd+{6 zpx67x2C6L0eKpC5C=?C;QW36jreRYO$7VTCkX%Sc-N5ng(iRKg%oPvOsp_64`#I;UqmpdkBeS-fN5I*(606S^rkpE9y>ty2@-Tke+olcLED=XI z5zJQ$A}{H#o2?wUAdvvJHh0NDp*QzFU{yjAJ7dul{^y{;=7|%5c*q9mheA3EX!h>5 zPIu5P15&1=X8?$|&CQEv*bD>kcflSP#rFX3o zrBqGT_8P`f`%NU3c=2S9(!QhX`Wofkh}A`CE$5+RZaNp=zuAmFoHZ{i=7e6LXr_ls z0F(Wl7c`suA1WL?#rDwW`6zEsMwCHax2M;@7PVV+D>Zn^7ME566}z@RYD>W=Tu>-fixAsgh||lGiY+M1g#0GCo?IRhd&udn zh_JE6N4-Fl`f^FEIX6%QJTH<8U2V$0U{)K4Q=6+}HL2+Z;8|c3jPjbykvZ@9u%V!H z4W&I&^3vmZ=ChbdzHJGU3|2q~S%#NmEsx#ykSseea!~u|^9qcH*Nk_=`3ULi{?Bs1 znY7?^{%ie%X`7U{&K$T*Hz5Z1=!?(KN`MIQy*t;EOJP%3SpNF)5s#<&nH+cdj#%64 z>=pqw;6;v{-ot3r9VS;S963#5wje-Em?SrIc{h$yFOC2F5OKRy^?pl>YAr*8dJ+Bv z6>}UY@-9HQBw1nGlj(nRT=aSACuleUZY0d(@xK0!?MbV^WiiIRO2MgGgL4=rHZ>cH zQbFg;0=Ouy!k|k!#GMGTk&mZ0_2c}HBiEV5?>wyLi$Cv6bT27{9_*2!{mC3ZS3B(J z)Efd451#VK-)%Ggm9mxx{PNSir`uW2M}z7)`>QW1iw+QGEq78;lx~A%4Od=f?>M9K z*b-X6QFAm3<%xWMECtU}a@>TZ=j1El>&D8G#Z&cfP!##+fub>SF;m)Q+8f0M>TW3D zjn3ZuQfFe$x6-FL=DAdR?DBa`3wUw(xmpn!mOZK(2cSKXo|Z-i+Ha$c4y2a+PXyrY3FiJ%*`$6v_= zE(+~YmDLbEyb#nV;O`lQ$;p%Fv_O;KISR(YsBy3RHDMkn4f1^#B%F6A@ZpY++dUd; zpxNmgU^u-5t%B`aN+g2ZE3(ix_@ni|I;1nRRHgY1>Be+0MXA8qp?8x6(b@mzwIfOnMP`k@W z#0)=in4pCTSU*a#8|DpE#jubX_e3k7j#1yx5RpA`BO^r^EOz)r27zRCnE+H@H9DD) zBo&>H&Z)jidR*~0;5>`vdCBK@ZUl$~jbaG0`-xu;C^mg0h!fzmQT7K`u@Z1Pv^Q18fI z_LL}hKA&SdRyI4Ui-&e4l+l?Lmmx zK!q_mj!&DV)@Y_X92>i?T$=?iphLg&3a-cgu7|8o1ctVQUi`qk{zF*dHSzet;@%Y* z$uD~@foL67fBP;A({|?)dOMEf7SnlrT)wLZ6`K|~fg@X8^PdmaKQWfJ# zwm00bT6<`z!5lSxobIZ9<^rPjnEh-^L}B3T?8yHQVi>)5qE_#CaebZiFuvzq08~qA zEL0Rr2ch$?Jl9|6%+xY&VcwH)ea7cRq6PoJk4=y2#jbs(s50zx#DQ}|C&)OVet#MsKUn0ipz;GPe-LLeJD+t zgaG)yRda3oiVw^Fcb%aTBYNL9Ny&_Q5%irPHnRw^yD$dv3b8ZQjnV?k91uI`g(M3! zj72IJ-{9nk{!6+O5`Z=$DczO~?i>D|uXDqvS`{^u+^by<0brDu3H9$26v*JyJezWgkB>z#MRK%#wQDx?- zOE0c1nmy3#eN12nS=u1x(&%RYixC*)3f{%&>{?okz^RM-D=!s3BN}<*iI4EP4)@Rl z9t89r-=gj8c6sDt<*O;za~=L0@~gkxa@h0g9~)pQbQOuqw;(^C<%=gXmwvZ{|BG@z z4Ht%|C7jT>AN1k~Q>1}kW4^DE>9wJ4E5HmoG2qdiG2sdzO|+Ec{!fPj{sGSG$ukaP zN#^QLs>24KLTw2<*oxaGeyiZU7hpmKJ1oAqD_CI|1wyq!(SvvAUMg-zb2`T7x<|G7 zURYrC{GJ#bLC(xNqWwwTQE;-Rtj?$OruwLM#)W_>dm9 zf%akriBJHMD~A4DAjB^tK_xN?7u$Db*=15W z*9K8q#qX^yJf4k9O{sYIHv$qss*IHilYHCkTohmg#w_-C%TKS`)99FWFv4W4a&@WLl~uDAFA2a9XrgQ z6aYGBW$MeZuJ8i0n>_zNQ3jg{gJdMp!R+Jx}E>7|mqR-vIO6*d*yG$TJ)R zH2wp<5CnApelNL`6b=`1W7|Xo4AGTKc!;eF&_+HjcdnF5?9lmA<--c1B0M-;@1c#& zKS}g$o^vyv!R+E-_(iDM&`?q!@~$&u+i!miVYN|(^?w_W&Et+=u#vSeX^uULMqrq+ zV(kfw78RGB;xEWhOj4c~ad&ptAmdzr{X-D%7q=Wv@es>eGXpp$f9wssU)jE$%?66s zN5GF>>S^Q&A$H6fa&(vRX@(Az)yYLPQ+TRomqc+EAIwfxXFTfvr#oxH{)tm z-f#Z>A94q_fW_w{`T}eK>1%k0lth##1y@k+wJRk$UVR(Gr#HL82mkw~C?ZrZJ6~E6(dNH9XrKj` zfQceI?&eFLTFS9k%zwQAYHDN{2q+B-Ha5(y4}S*pA^_pXDr{+K`M&XHe}A70L9BBl z4kt^YYVTgf9>yL9L=?4XwK#M{q|x>bcr)utRLw3r5{S!#8R7e?gRPa7xU}>uzL>y? zRBd8*EX+^hN@5>19^;=IC<-kUZIJ^TjxmmxDBGUw$)WW6)m4KoryK zmt9TpLosO3gn!KtG@h4@~`AlkuY~{Jj@AyUA;$>d77LA&4ox+x}XgyOx% zlw~jw$*|0LMq~&44hEJQZ>Rw09Uvek8y0NRVBdGVge&R`CbOhK|FwnR%ZP!F+l>e^ zn9x8hV>okEMz{IofRJ}wmqM0*6Y($?`qw#38RQor2Xz8Kj@n!~u{4R3C`8^5BAs5T zH)*??|B?t_M->bIIVk+Z>EqT$%?xL6{`l%Y$r1zd70AQAg*uRbvgN7-cKhyQ($4Ey zd*ARx_{evbe|IuQxU7q1EI9es#y4d;?RUOQ`=MSHnu9g$in%2P@lpw|VLO53jvzC; zGGK9^`{FMB%S{{>&9RJ~B>0*BG81Aw(^n0{C_rnM(sAvWKBf%%-+^~GRPpP555)k< zIqelyMd*#KRJY7mWY~;w5%gjHP#wFTs%y0Es`^90@QV^50L((j;}taX{0EnT0S%Dh zc?AVV^EE{pZVS+I-}!1nnb+*W2f9z#%bs?W`v-&uJk@;XLs&p9jfCabW~c7K$v?`abb$?E=lEI*WHaMOLs0c(sSC}75Km4ftE}bTSbVwt0($GN z<8_y_OeZ8iOkM2&tpD2S5a;LyI%{(PVBW69@V5p}PU1{X5_JO2cG13%2Sz2j=bqoa zUH1GIfKsGv(gRVZ_uU(sOhH6syb&14`RwVF^i4<0b7UyV9XrqNrlDR_hMcAmdtHI4VTza$1N zeL~J1f3UG%qTfI10wfhapwMo@0<}_6PcfPy4m>DRz6o8;;dbIfG*|j8F|aW>KAR^-$YJtU%CaffX0@y_4&)bkt?8# z__4!w=2M^|{|V9Jbzr9c)&O3G<Rb%-^xc?*;GZbqKDsA%lPy$cB9weS!)OHmW%O zI%WU%gYrJ`#+EK`;Tyzd_Z@F9o2%CiyE*4E?g9h^-`HG71O#d>APw8!=Df>PH8q33 zGU~Oz*~y4D%g}^R8%6>(*o;FjdQBYN+^?*3$@A)=SEuH~M3E_Q{KLOF>{Ajea}va@ z-o@PU7tN##Gr6oc$Dwy*$pMq5rW8XW8pW1P(uVy3YN@@kyEu8S-)Cv)?ZcVSZc>1s zcLz0B9HpFBzfP`jRHK3*<58nbuK5eJgouiZV;gJs=KbbG>BI(o_dLb{iRjpLWi7ZL zlwN~CE4RnQA-BM6q_nNVp9;hRZ6na2!`VvY+S<#8KLe>RXS`qB0XG3#;I%kSN;sEm zV@)Tvesu4Ft>^{t22f&uke%<23#dKK*K?+VIAf>MaSOR&E##BK8eFw~|HWLxR$3&` z=X-!5xtG#=SUa-}IHqo^{+`^xSHG0lmxZ4LW}YryVPzn!Q+4C1*t2CJDFP=&1OU#3 zq&VILs!#1G=0L}dssuV`XsH3!*^NL+@9{&`N+X*39K|i{y)y%GODg^*&0O>+=9C#I0=M6a0wbD zxVyW%OK^wa?(XjH?ygz5!@`|6+54P#->>)G_O%xNq#mb*?wu`P$LVl_Q(X)h2~l7jg&U9|021g%Xk%c6p)H`gy6A3c;%uOew~?Tzh(@{n z*yL%O3YBCg(c|X8D+>5CfEf4iJ3l#82XE$j=C@hhG*ZcSH|V(y8cOCiTfxq^q5$rg zv#xirB{q;1C&o)A-SiG261BZ!?Cwoia_|o~krsME`Rafza^k8O-dKi@KxM2O_{P$0 z!ePYg3kqiaBAz8K>N)eg zMN+3Ph}W1UIk6x@^FzC^wR=z}#>myxmHjn)C|>xjR7))qklz$mm_hV>9UMP>&iws< zt=c>Fu9vj4ql3t-sX+t;Jj?DWV3O_H?GGGU8t(VV5y ze8Wfb*v^*^MX?0y>{;)VtSlE`nS$lfYSCF6Vdy{tw2B?1NDLiLSc2z*wnp|4Sd!hf{u=zs4+w*;4yuZP+4bP(8vGwI)1H5X-kdsvf z-*ML`707(UH6alFzyKXyzyh zfJRE!`9ZVm!WZ2cUbPOo2V%j9snLlL8jkOsv6eKFC~=SWKOxzJ49h7TC)xo#gop5H zOurREUiEn@x}|XHE=*LL3kARTLvDCzCPiVWl-8`%Ep^jBrS2d-P@K|B7*&RHc##~{ z7NG3uER<0J4oc?(aJyQ0JqEr>x|m-J+Hp%~KP!&os99AGGkbVwQ(FaX@Jw79o69rB zmp5NLcy7okZNNb^K3(8Zeccs+mb8&3z-39xN2ttubhL6()36&4qZ4OUI9l{v+i zRa69SoJolgIyg84c%&5q5~Hv7I=Q%#>$uGZ*hJN?bT0;jzdP)TrY^&ax!G~?F4c(= z>EBN-EJOm_Q&bq^_TC<9;~So4^bz4MSGlo2GV^t;kuCFF2=R^XbH zo^+~X=>0oM=YtBCJeu|U_wu^TnT=$&A6COv!a9tZZ(Z%kf`{4|(HVn-%>)@4gYk2W z(R>j;!XbPNBIWEkew<#LGgEkeDbzI5G1O)WxV?9uEH9p!J4=;Q*h{r)J9CSU&|tCS zSkSa%LVaEMROo;EVZJCvauTh3aaG6?tov8caMh7n*-j~1#fawFKvXH=_9H;NBVyv`U zZM0t3Q>gjYGUP8f&xUvnB4b~9R zY2&5gdx9`RD7onupe`p)5l~F#5`NEz77-Utzyf_fW~I5jt=!@1fyeuT#A3cU3Y1ZB zzZc*0R9bDAr;gi;eOwomNZ;}AgFxb(yZVa%O&`r$1sFCg+KU(|IE)&a!0UbomzTGL zM4Xzz^=Fn@Ol%mJnHsfaEn>s~L5C+J1F;}=0dyS~69d4zPnZbbwQ!yvpuQlQNgUnn z>y9gFc?cOAQo6Xftl4^yyW(G;tTe0~2e}41ES&wc0Av%oJ@DqfT7moupA=xZ$vR?! zeX=`CTQN^bV%AMPsZ0>7vI>^1nf2>9`S$tw!?`ksp}>_ba^cR=HWX`o);o$wQmaRl zK#yG6-x^_@Xt%%Y@|4-7w{pfwHmEeE06a1fIAL^S*Y%_Fekhq2?Y^U<&yG5zTV-n^ zv$Zw4Coqzk{pDb{4B}V1K7s@WWh#7U{+L>F)xbamkrtz|G;vcCbo8**`fTkJj+VRV zSKP3&)P&^Z;Y>ak=Qdim+f%EXr=O2%sy&7^w(}}mg(W4C!KT54G}Q03|Nel7$MH)S z)EN95_489{3cJ?XW48r@)StOy%=f2+>BG@BlA@pKuT%xy(f#Pja|h-MpcEo^G04YA zGBdc>r>Ki)?zEQBHXQTBvcdF}g%lNniD`yL_wmuW9?@r_XzxIGY;Kz>5T)w(mNC5x zeSLYyi(oAJgqW$EpHTS)Q|a7~OC7J$vP(zhDaf5MQalOUU*u$DhBjVqV3w9t)B({K zM5@cKvT!8v5bRln-fcOz=$^|3B=uF9jvP$Qz}Pj@IoyUz;IL?k9kpvz$- zgP`YY@#2e>rVSp1m+Q+8rPo>YK*nY?o>G;^ z-3q$%W|IlB_3PH>M9%)>lvSsdl`!8Qxxp+%TM-KH7t>e3+~Uvbq|jeqke1w6c`RQ< z4b;E4w3lmXexEM6s;v6r$_QXpqQ1aHT;%h@f?n&*|XVA z(n~3ee0ox^_iVo@ba3p$N&vqkJCtfXG9p^Nu*?{#3raX3pS3^oXjTkAqq;|W{yFF5 zLu-zdCLg8G^eygUf|akI4Uob7iZrug5sBZH)TCgaki1+Byofnhgt z!5k#{u)atKMwt(t=kH|_+G4J_2h3f3m*(#Hhc;&p+vY^coEm5tUw@@e{&%$y~$=eqqT5RdhgS+Tt?WM@RH3uJ9Q ze`mAm`RJp(uO16nJGNM`}lB!BDg!_C2Y)a=W1ra`m&p#X+IlryBpolvY&v3`r=?e z0Pm#Iyh6-?zd8J7+X>xo-REtW`~}lYN&_Tag@`<$Oqkhbtl#zI(!$VYcC_-)EXW_Oke~;bHSBra{xXYV9s{ z?9YiS`5Q!xoASiZ*HSCydqOx`gTU8fezl|#$Hx8miTfUBgVQ*~`h0tb0-9fW4y|^t zU;ed%jw0rrbPtSnrXV)@18CoV7u7+;0G3(Rn9sVX5JaG&$d9n8vs1u$V5< zNp2?bS^Mi?NWx2hv4R;jJ$-oJUj}vN$RfFk1iI7Q(^#NYiYYarO~>~k(K_wtD>N~m zPEaE2`y<>Tx>=1obo7XdHLVo;U;L;4e*Smf$v{*m@6{2vNHjtb?mmvaqub#OEEqO* z>=u=(z{&u)R+ZX);^7gxrtRoZJa$vI9RsXV)oUvj%ct1Aj8~!@;Yj0aE6F^W-+4rYAAd%)lNXDzJ1 z?|6N#iH%b@-{5ue2CSx7D_+2#w0^YM*u&{iAHLY}Vmo9H2^9wdzL)M|Wmc*4y~*^9 zMBM{G8@##qsH{5LlHcwC9Cd3Hh?1ALO;{X_QySmii{v8YoUmy4@;Beb>3lz@V`tL@ zCps>UMrmDihVB5jju_&&`F#twf7G0quar0~%j8Mdg~5lZ+fOt#l-c@&D}PS;k^5GR zPO^kGrM7yHfQt?|o*;cnb81ZI?@%0t_@7q_s!MqAyR;Fca?};$C9VHtnEpY)c$aphqF*a-Y3yy&QM*>%kATY@(s$f>i#D z-R?1A0r7Mh?xg#;;ckZoam%c_L{3K+ld`K7H~~DBTb|gGQ}S->$0@qC{W!}9#f6!H z65R!6T4)=ao&;T^&UkLo4ZG3g&v6Uk?=UXx+N#T93#k*6Ok>>N@(ORed8jhRX_{B=M5tr-+T{))m- zO=V>(HYwWjlwe8%Ew{{o-Z|K_6b^aXf0YeV}VqnqhVu;*4)(j>p?y=Rg zYv)K$Vue1CwRRoLD@1U>rb`>Grh>%sLSpTpm8IOQ0EgGQB2+*Uv#oVN+V7CS>FN5{fa2oDHB0mQ^xHzLl?bASm9o*FK zZ5%sROUNnS&puR1Gu`Mj3Dw2~YP7vdSr5XJi#X#7_`n%C9Z_wORauFaG2t{jq#_}K zOs7?qo z%r{uMNs*CPII$t7fL{R)?gd$AC;;iSYmZ`7IMvt8ZCt*nStd~d?Wq# z6UV5yHGz1YF)Zl{Ivo-KIqjYeY;MQV*wU3@5>7mc{Dr^(A5c-tA ziTNR2oyC8kza=r%B1Ra?k7bhXES5|F22xsGcK#UL-K`4|fXtaUcS*?O?l(Pc3xhf? z#gT73%Up z1rbKfoeU)Lx73dzAtC3F&tZh@)pA}x;r(O~%#yk5%h!khR+u$fKJO0lmr|yuOnh=u zXTi9<6qS-XvpG>!Qxks?)lOl}waaAbEJ&fvoQDqK6Apq#Ug&q%JSkm1O4L_QK4aK6S$BNknglS(1kZD; zN=4pSN(0nan`xDw+!KEAM%A9rROaio{#^@NarVam%pk`_>Q@|>!}PD|&*-8e;N3X~ zeL^CL|NPAZ|9)&JX8g3WgNcH||LN(8cm)NRa?U*tbOH)pI6AR!xvJ&}OTBX_phJik zfjyFYR=?<@%7x{}-KSQz9!=q2P93_UVKDmJ`CDIK|Ekbc4)eXlj3mO!p*tDAIS)$$ zVK8zKJ>Jq;VmV&JO8CH5me=bG;7__Pbj4VSNDB!;NR>2-bbkr47+7F^3+ZX%#z&Bs z0zT)n_@D_?$0QHWHOzE-XaxB*12+X)Xbx)O<2-PmBroF3@K7akEDV;PqxM0#SL~Xl z(ao`ytcpq>Fn}$|{uqE|Km~&#k>lF^0$nk`q_9wbBK>)GZf@XaEZKn_LX^u{5-o_o%}1NrvknVMAy2>$5Yn1)d!Jas_HKl zV7r$E*iUO^J{Frr8V@W#jSYN`k*gYH`-qSR`UPhK#o2QIxuv?n=ud2+k-j|3(_jn`9gwS0z zY1UT|hJ@qg3cBTb91ShiX!w0iV)2$N&{aIaIBj?!Hn~{X#Q-$t{|{`5xrpSt_`21f z)(^muD))Ojz!g7^&l|B~r3FaFjE1qC(4+RR-(}|KWxG99oeZft_VWZz6J4e@3;$OO zkUN@A9yc#Q?T`WmTK280@+i>H`}SE)Q172a-fyd$g#RE$Se#hu3%pePYUwx=4cTb< zH{(k4NU`6=S=8FN`5iO=-X9blF{Ff4Z5-ddfRY-o>$#@q)!(+Vc#r*oTkvG$nc2s2VB-$3GRdwb4o!ppZ0p|2srx33+|l|jgY&%-NN*dO{xLSVgf8=27K-6WUS-Tc9c z^j9#a%4c=AOqS&zhV2C(-oOxgGYr{`c$7E7fk&rdg6o?QD4FUbR~Eq{0}s)LSxP5~ zdCd8{D943Sl@#(&kBC*Cw(%aD88?iTS1FJy z3n)|+n7!`~+fNV<`1G=92V0w4E&k%+9qsqE6#xl?Vle}`q~sq^D6PzNiZFoFl0_5l zUr?}srruGa(Ji;Umf(=ct^oO|+EpXxD`%g*Qvwjj_to_kDd32LvL0gUxig4@JWy1= zkUJU}7)Aa;ID*R7jW)zRE|bOk`-`k261qf)54o?e0QF{}`bIj99Z7anaBIX+oB*D` zgc&=BA96CL4WVk-S^!AtbF;BPqBeQaj*k?6?QFlCkNCp;`drL0q$p0xlZbY~xp>ms zGcr9e#V~XzKYNZlM>+F=XY$4qZJQrdyvwSIY7s}{KgIk@)9%MsHyKr8$J+j3}thA6l8mDb<-9 zocAPrlZC2GQrS;42N~f$govdtHdDXjLH9 zIZAwr_%8IuUX}{{F1{Rv)Z$Mhq7M0}&m%UZq z0CZgxgQuuDmI_c-6_WYg7^dw>-5fiXHD#WcQy*Tc+n5jTHpK%+Xq|i}(X)ipB|*Zs zDtaLJ&ON?A=Ul+eye`G*S0Ffwb;99ztboBxeVf97uaw}gEEBcLOM@^*UoKN0+V39Us0kB$yn>V@3eacm57BL{n}9}f zKH#q{2v$*bu%=txa=k|;Q@vdAqlENW)No(2sP@Ff(KDV*L$zuixhJXQ4yAx2)L%Y3E&$ZLmsClhvVg5f`8F0yQsa3EP;QN5!IQYu>wI z$L@H;avbL3r$|sbDO`832Y*ctGLdz0p z&1gx_2>h@*vwa#a0vRMQ@4Hzpv1_F8$IAnpPqWm6X@EKFm6pU=7ae0icqf9t9y(3O76XgekNY(L3nL0+J9N?Q1GH>Apu$h8MGc zf#3s*AN$5{rUH>R1Rv5=I6dtK)VADuHGirxZ`%$@kHKHo>#~V!s6Pvsyb1;Ifu%uy z$at6aiDQies~apGJLx!+Lup=>om&9lHJ{;=(?uIm}=691}cinB!OoXcshf2qv zZ@r=DoRlcyc=a;n_)p$tmH9zQ+zRy-2u>6M~c-F2yf&I|^By*daleNDtUYrmk$Y^Ovsi>$lEGo&##au1v9;Aci6yn$| z2L}iD>+1mxc=z6=gPK~czVGzkpGC%BS>5@}JJft-)dSH*jkI$M&>nUiwUKijQ}k{q`2B zVKbFjdRadsEGPKio$Q_%5OCspUwD5&{%$0z)pX4fnwc;6r}9ucWUhN+gLjbENg=tS z73l|Kbo*uZDpT&}yR0!(uC@an0al@+UOl=DiDAe%MFlto|MPD2MX=3L$K?*pATh*( zyV9XL8i!5wU$C<(7cqjrv9a-fJ-mc~fV;bboSXpLDmNmIypmGNM3o|?Vw#2T#@X3C z&?A@N^C`p){v~_GozIm<)%(0rqLH%Pf+ZR+ZI~|mG6BDWU325fvf0Dk#)l(mS}PYO zFaZKJj970}4H1u}gTf!p0=aqd6@GIx0#_eoUszNgQ@kWd#BQ7E#Us%bxiAoNX5J!X zr4&Bj@8SXIvENVfCGzT2Q)~+}KLbr&bOm(Wf99OOxQ!EZZ+CeB9~R^c_Yi{->2>Es z&AECN0KM4GQ{95x#KPkWJ)Vm2X+WF^k)P_T{r+SQ0*8Wv!uIy|SKZzq%c_d(Y@dpb z7kqC%Ul>O1PH)Syiqqmi>f+)XLV`n&%sBy9Eoon4@r^DKv~+8ZO5S#FvK(8^z5u`& zB2COZgTV2-BhI+>jPYZK6@ru`-Rif+;p-;6AHruAh*f;%`(!M&0RUNQoeD@>Awd;m zb2C1Ua_-d<9C5?m>C98Z;xUyijc&&|KDpn#nms?Wmt&@{%MY&8#=#b3 ze8i%H8+az2-3lH+yzQ<>-I{2XPoN?8Xd(k426xAN0E?LOg}L4e-K|^YM!e6$FdmB; z(5;F}PF}VE83O&ShUQPJdk*ZO)(9LIV$7{g6z!ehrf0?or6Mc3fqt14lncS^>8X-b ze4l5Pr>Eo~=$>>F5}@WpL}<&u4jUK%Aw%z5sV(>0@x)pBd+i#jD%?xO$A*GrMLKjO zI`oUG){_+?S*VMnmN%D^O_o3;S!qbjDNR7HqQHWJ7-gYfiA>%7z}hk`bpoVHB5s&$ z1v7oJ=}!q~(O<#R!HXH>##81=Q}06#aY^ErrFSkL52EEnb8!~t;zQ=9=b&&7q?A#k zzorK~*8-J|(EE9HyTas6jf&E96x|r!QcBc_D}$3dO1}No9<-)x>5_ah+Usbrs9~ZB z{pfz_H&0QtXnuMF)>IYQIslOee`)zefV{lmz zvUI21^s)G|AlA>oLR$9{;0WpGwTyo%?292#ViN`5e?L*GD7qSHB;P(Hretueu>07At%(kpYczfmt?-Igl z-e;@!`5TV6dPE1T zl2W;FhsA0>z#Ylj02v~)e9s?lu>R(rJM^?V_@|*5etTEJpK7}5>9h~wW1$F=UxFk* zjYl_Q0W67d7{~YBB`Z3M&c{8}6gVloiB~Y0!iYeGyj8M0+(C^r`N<_?^(z5b4%jwyvaBZ-W@RPXyh=@{r)u#TH3Yp=B%WGYW zGk4#!mE{J2Z~lb{2Mj+d;5jeY5o99zQFWmB7Yi%HD)<_q6q*+yV<6UqZacZU$_f{h z8dAaW?~%SeTeHAp)IL^179Wz65Q~7C)J4jak|Ls;eopZp97B47Gsva!@kT}Ml_-vv zmmBRtZkY!8%Z&!Im-n!;rNAB4ZG&;%wLt>g!LZqz+u4-B`o*Y9ck$R7WKO*b8#nbH zQnj%{E^SxL<(~yc+L>|f?_B@j8o+4R{>-3d1TW=4<|~_#gbi@OD)mZv-!gsCBR{gO zfVNn1wnY=uNK>^3N5#=1u(ZCe^R+zt&G+?DjR+O=@sP-Di3dm82A|uPV$MHCx~>v@ zgQZmqC0NwJLq%>jkWH6QC$-0PM{(ABCT{SE-1CP$GNB18XM3Ml+ z_PL9*R2#CViHsCyskb)28s~)!MHU~qlGxKZl$>e72aF&S2bG|}jw@+AX6}`o?wPSA z&Gr--vhEntRPUfYU<08g_)qJJMR+`?78m!R6hhk;a4HJ)3RIC9CwGh;9h#$vCvIzP zl?<|T@NOlzg=nSFI6|a5uk4rWhMtrh$vuBT%4%3YPN=3&=5lNiFj;yhUFy6#7Scdx zz&`9Iv!@H%11a}DHjr3{xn5Fe+LbK7hc#l%^nJ^opemY@HFOixlH^xsTW#gOd?q^U zmR0m$pgSRlqx$&vco?+{hcwxKd%5F3nh%ZfvwJbWLzYLLt^A&hZovJa zPQPH2HlQjSi9U3%UuL^STN2tZw&c6yR0z>gZvE95dOG-KxykhX<85Fvw~>JX(17nd zVDX+gIF`AdU#p*cZy&Nsh5)Ixfq4wyltD{J$1X4HyH-+@FDUh}-ufrPvJIw9lcBG# z@6Mg4mHYU6+M?qzCJs(aa&kzvedI_7MW${=MVLikHB5Fvu4AwTjl<5>QbSB>KSq2k zbI9HkFBDIvzZ<=23inE|F0LOG_s=k1+ike#hwGTGK%mg^r3_wbUdBb9*Pez06hW{- zvF-F687x{KHz5x4JvtuCi_XHPQG+H4?z&sFZg$e7@Hi(+Y=!j6AoJM*jEa-li6^6&OTR3ogywnp# zCIGa^5 z{kI3YJ-2r|L4qr%?=n6`e^tlH!#@KK) zND^&t>8|g#zk4dXyD8cl%y1#PhYq-Uay1j!0B9WR>Z#;&{S;@~@;pQ!ts+9Z$j6gD zT(Aioy}}&1DLO8W+;#fqUl6Rq0@US1Wo`KO4c;K=3HdS~v>y*|VyycrqZ_)akDj!` z*LFrmaiBFAMInWoS{yU^o+rWqOv#PVJHDPP_h`W_{t60A^ID?)<&BEZVP2;xX1kVe z)hbCgbpL|$hYFdDmj)No?EBo&vkwxH zg^`GW6(^+Yt`$j?nf_{X$!C7Q$sQ9-LkJc6{&RRbujkyan>9@+o^vX{!s`jn`pY+R zcx@%lzUo~54VbgWY=|;mayFSG+(t?oIeLaLN0460zA1hjwHsqDef(HSf;{S_2+W$v zf1*<;!o;*WT+9f%b6{uFtx;s-!R(xp-Y209bCEgD`O#eZ>H6lYQSONM!M8d0Pl~ut zU&oKI$O}xOLI?D|r?!HgjpZBvdjtOy6xB4Xf%G0GGX4grp0Z6f|DnfBNAEd?(B#?N zTbmY9&NwKQv@G_|+tP^M01`F!Hw@;+adhbHquy2s({mJQu|kCm8|1=+@Jivg=2V3c+w?2=6(~bdjR$Fh!7n_`WmAOX5x} z=L}`NK&at9U8f&eWKGS!fC+O&M12BuHn;v5ZqOp_EQVB5f&`Rvb7V73|2#=%Qok*R z!*@qbd{J`~6e`OK(Nis#6qV71nvZm%*z&e(EVr3ta0A>ZJ_NVqMDYw=yE!nmZA^w)W)FevQkp>}eBIB9UpbDdhiw~w*M`wSpd63o+da9X#R-ix&+Aa+( zSH?m4J+}VDULr1WRw#H_65RkB9@~rqY#86Z7^tZ4YYW^z3rg>)^R)E1rUQ~x3s-A1 znvsUs^KRs=#HbDKP_-d*WwhmGodt>tzg!P&)TWw<;hyLWWN+5ZqM8=IPuW3Vhbc6r z20}P-Gvo`FMzK!ab?m-w!5cprOZ;I^Wc*c8@R*MhCkE^wNE-2xz(t_REX7sio=dtB zN3rAG^_+G8z4Z_muBYK;$Ci>r6-|woyVq3v-CnC)9{1bZ@Q#_NEED}>5z;Tv0r#bH zuRuqRUqQX!5^&;x@?cWBUPAo`Ka}j=5~i(SiP`&}FY));vpq{4Yomcq5^s4SeS-`D zag09&a1!el#LZ^L^$GN0&u=Bg& z*1D4Z^g0$8O#YA9#g*|(u+(oqcl&1STk{ZE8MqZ5@3fT7M@P$VuW3veFIxj+85LPD z(@dSGUolds(i^nOYk@0+U#bAn|82qHwkj$dT|%D4Ld(Kv98s@Jv)1w;nm=k<&mk7K z=~9ZS3&VVG$27%tW;=@WSz72K9CPSa9)~%u0O~utyt#eAs_B5(rkIln3*r�(f&C zJ>H5YxG7kV`bxB>`leh|;{^@NkC@Mg+Te1Q-<8)+?>bbvmgKNl_2ty)HNey5d`t+Q zRa4!(pc&IPFezKn!{?6KId-fpdHKiBtU)}lI>sh z(Ds8@&Uv|s#2OS!&NvHs*<-uMNjP!>Db(r6F(o*)9HEn_C~DxoA`h`0eJC; zpE!U^Rh(1joi#xOzQBGWFnR|A)1HrRM@ zO~;DuARs}1Yd~0cFpVk`{M>NEVnao=U48kKr;-|X6pMf%Y57E3|3i@awN)la0P|zWXe%F63&gF~4~dtm_n3^eVZLg}~O1y`S&MVv2=tS5712$-p* z9XzW>Mjncz?lA(Eu78bIgA*r=i(>#zp%Mhs_H-Qtz|w=-8!7SH3tF6Y9n>FdBH&D~&&$O*eD z^Bdg$+#8Ys{p0~ayL4OHiY&E@>9s`CAY69 zs{2N+w$&#BOb5=)L(W@qr@|2lZ-I5+=2>t7PD%mNci8)W{yWX6B%^=_`KTls2ey{T z01;eH0P~G}u{nqA#QdgfJ$4I?Nl>nqwC-!L6+!Lc`#FWTo3Sh^(U_3*=|!n(BACo+ z%3TVgWev2*O+VBhN%UPmM9K_xw%S%n!F&>U(ezF`9>OUpaMpAxE1Akc!NHC6D}r1-2eF@4v4;X@R5K;s*~IEXeFNJIrub?Ixbz zT4#N3-l05gyFF#`V8BJH-puO&wEOsg=m(fWp6dC4i%p+AFk7_8O4DJsK%DM&Ls4(F z5j?O33@Z=BW6HPP(+J(4DKNpkbt~ch8>ggcw9)QsoZL@SfV5j&Qm(x&v7l3B@A0)O zkfd~PC*gG*(4)#=0+KG=OUvd6y7Iq9BG3Rc4=Cmf_{Dx!_hg0u(Y-3Es7A#c*19 zsZ7&46g%mQMMfl$EN9Hi-)-!V7#LFdhWngi+g&ypuU9|hj@+IYNZqgj zOP0&ygX#Z5$JILw)J4P3JZ|}p9SM}w*<)Uw9|1!M@ykWR!eXrnB175TCleE!$Z#CZ zJ0?I;Ujs<3-$mcyx#E3a=rz7 z#R|`iSI@f-re>x|uOzI4{`S>VL)|fkKU@~Y>I|vm0>9Mhq_g8n&=TJ|CMO!`9lXPN zw~fs)tWxG?iULCV%576=dloM^&KNL-NNWqruni7IqB5~F$# z=jKJEpl$Nl^L(=fLnaw#3!zKjC_9G0d(?XO@>-GfS}S_L1Wz4T8Ja|XMa=iOjxe~Y z-GBSN;b-!`_(^8v=7t$W^7b9*4OF@$?KN=KJ;Avv-+|jj;tUcT@q$ z1j8W-8I;^>n0cq|kJzV=U0%RI@{;>qtOT2`_j~q(8@z7wT-XdQK_%tQ*-PI5vG*RT zmvedzReB##c@OCM0|hjAAK&Y5Y5>oxl_a`t)h(KTdtoy%Cg*y;afkVO^xS3pZ?L&-V#^)nX7>14fz4ovyeg@An{KQEACTH;<)@--q{CpJ_Qt^ z7J~oLm*BE&`K{9J=_gJ)#An6I zf*%>4UvFQNKu&yG7cS-Z^y=leccUbP2tD{ESD3k>!Rox?mQ%bauR3V%ZsQ_C=W0r0S~ANQb2L?bjJkr zFMdkb`K--jN*sJ=lV0deQ8J~4UhK?>klM_#gPtX zyoLvQ9Y!nlL{AS3yqvqP%Z(s8+e?^lhG9+KKd3Rg&rqZOfY3n)`aL87=o{3Y#Y}hn z*$i{3YE9q%J9coB?Szjhv$Nzw(6*zh&EmjAvi&Gu(ubD$H}}I+RuMWt&V`F$rpyzS z>X=ovoaoAt$9`v;8@>X0qy$NfF}irCv04&Oe5vphYU{YaG|FEFM&eqZrf}4%9SXiB zt-nY?hIQ1+7d2hFWR1sZcb~fALOdQ8bFog%u@~nuL86PS+*Cq`HQblB5>n{t5be|s z12EDvl4H^8Qr*E4TXTo@V^=>+XOPjGSZPC|Z?!yYk`{*k^p}SpS?ljj?qWtq*ppT- zuMNUm_kF1}We?*dq11$O46IF>q==1;1@~4L0Jd4gyZKV9$`T?nr|)xhK)>aSOWPD~ zVyxVd5HScqmMprwp?lx03j}OI8yWbJ?620^k~1_u*-;6sQ9v;W0|`b{R21Zo^D?^L z_ISVZ`r-#T?U)rEfp?1Q>M#0ysme$Ia!S&-owu1g$jCNUgBKTK5J2Ar8XB6=jp7K* zhwCG80pFD2VcKzCuWu{$9zz%-FM|nM$(r;YkA`+@4mC|%<3i}RPTwZO&N0Ov}v+aI4Tb!H(#DexNkVT51*x8q+ZX8hVzr=r{nc+-5b+=r|6zh#PaB0#WLFI-1aRmmh4g-zq%S zg&?0rGXt*o?A1+Lfw5NiXP3q%U5zXI^7ci!WyV4g3E0p+q&;X{H)4JL6p%N{&Lm6cV#uK0d+5Yv?_Y9yzi5F>vxEzV##mRwi*YWxY9_AvnHwT8~TZp~xt zc&d(R(v@%I%^y?ks4iyZIQ*vG zkCl@AiEga_qG`)jzwJ#jG9(q9OTHB{gBke*SWAi$_2%_yyN5(qTa&xy{qhV8#0NE6 ztYN!n@3*=$==z3-KPiYB0JvF_`miLEOlNb6iY#pZ67|3o%FK?Q%!7u346$F!|^2s^xWqwa)j8sI~V`JQq>3zp8rFRS~jtmnr+b`;o#up6q`0I z+H4R0VbL)ArzT6)lOB?nJO0ZbKDX_L{if`_Xm`wQx`}*7Iy}&6-_UuC zAezguw8aJ-3V5lxhV(!qS>oEBc~WdzW`bm$a>Iu1nS^g+5!v24i7%2l7M|tTmarMH zkyJTqM_p&PPHuoWplv0fa52{uCUA{|lh*gR98qpEQ|9lyYLYa%o#n7ZgRP|{LJZ7e zA{VK@djZ9iE64D0`ZRVK8E`q1^U&c_5Gcu>_|1FzYvOxd#&dX?%9c}k|4Qj@9&WUb zOELX}d*}EU#Sh*TjMfi1jycTD*G?VI`(rO`pn#lugt0Ctqt`97*QoJN?JtWi|A9d?sKN zMhV zR>zXdtRBur#;%;TqwNWf{KJDJI4YX{pqTmg8XQ&=r@&aVNT&Ff|;13ZjxRoo@e z`#44N!60Ul$DFLNY$=UVLvLhaeujYMEC?hg~2YXjW7!hE0*Y5 z(M=+TPuu>`!qmV4%io>#rXH*>>`cF^SR?RdFV`o zf+SAsgCzo>0QUI_iSkMTH>1q@C=bIxCwKm#QnP~Moe$Ub0B@CU-^M;L`sA$Kv#hVW z#qk8xWTd70c#{TnM6k8B1yts=PV03+JRZxtn)paCD&Xl2_g}cE^+iCrY(BTL8nCC1y=lFNs4}*uetO}LjQt;6s;N9Ir_Yx_@Aa+Q z$EKHn61_(Nbff~kNyY&Qc{eM4hhG@d3^d)pdg-7iPxaUb0Mmh=?C9=hE*UPpI#i>t zoW8g{(I}Ry_mN~gAa4z1*!;S71lpb7Jzbt~y`=D-?^k$*j2WWriHnaoAC)OAo;Z$o zXjiLixA}|Mf+H}~`8hC)@ga%ik>gL~mJ~=L{Fah@P5~cE9)|jMfod+YA4a-hm5I0x zx0)FKqU>Vlye{kMUm00aTnI7p*m3P;>zR_FE6Z#9O$ZwV>)YtTb*8CtaOwM6R{mo+os)_UB8eIh*v6=#}KvI$Pkw+-|&L zyxFgkjKU43vRw*!NY2V$qDWRB^{^Q{0rc9=U*4Q9CK6hP59Qq>{Ffwy63b>ZnNS7% zDPoy=>JD&Qe=`r6HR6;Ouwk5t)7@FIa!=z@&fpfN6=kb}dq_D&yc>;bp}wAzc7?09 z9kpiFU}{TwsEsOR#0;V$P})z70z{(1fFU$Hl)?OFv0NiR@b6!;dr^Z`OVmrNX5Z0K z9`|#QTGca?p?dGz^SM>@>df}PwFdd`ZGxRufb(eC0L-Fzbct@5l6u+ux$3az0rZeX z1-iD30LJ?XzzDSjFeQLJd+6zYVPN=X;$Q&v?3MoYabzILY0F|Xl^xnGB(09bbF|Ah zP^HnB*eKP;we+bBfaf`pEEi{9{|{GR6;)TXY)OIzcMTTY-QC^Y3GPmCx8MX12RJwc zf(A8$7_xGSko+qAd zoM8EQJu5qVcy4^2=+9b3;}+|hQEl8Gp!C@sUAK6QiT+hmvAOE8-hZUg^cHkBS()vL za0icHZj<(~H&&nwNI^y&RfjE5HoTaJa@i^GuxZsU&*oIyW!uErV{{$wAvvG_#T|U2 z%IpEE8|Ue|O&x_E%FTs?KU>p4y5Dp&cgDJT%=WN)M!dUZ>mXQabv-anET)2KsQvoA zfXn)ShY8>3vftYm{D+PekMkXijmB)ilDVX8p-U&bxBq|G4oilW=9(TcfD8p{l;nIU ze4*&Sasi$%gND3Jgotc(+ZbR;iPwYvK=5M#2 zHRsfPbKM&%8zs?aa;`ybldj~Lfi=PRcSyXHi;X1Lq(ojj#(lATu_=z_fW(p(pum*L z;ZM-8-cEvTgJXWYz|wj@MGWj&-1R;0^acj0y}TfSLL>~3YkB_3rILJ9*#QcIbn(0> z?6_RAb<91&CzH$^+M+<=kVyuoruJQpzr_JF_hEPQW!qYBcTiDzPVcjycWW(8*00~R z>HAL=@~GiEyswL&Zz?NmFUvJr&x?in`A-}TJ;{LeK0*43Z9(FjGDtsOAL|QtwxZ}# zVv5hYc#BTFs!Bh?q$JFs{T>!AqdxDoSeayQd6FuQhV5|L8m0J%e=3a2Q4qZ_69Qb5 z8a3wL(|SjA2F}`o4ea}RNqB_lbdIA-O1hADIy_EVN0-ps1M>RI8ljYd$;~L2)(59? z4+o{C?39#2U3c&4-xP zk`a<>Dw7|>P8+_R;9j4`_za&|pEfQa2Dsq|zFdp01HrePgZAJ`1!Ede=1Kw#*n7Ys zxsn1maA+%ZkqUTG?U3Ykl?t zhfb|7~LsK9pwx8~Uc4Z89oy$tGQ zLrL6lUxC@t3VQB)^XZhszbjfU5U+m-@Jlx?!he zJLCBAb0rrVsbpPAn#x!{AJz;v2MJlJFEWhX0WMZeo6X1*jrZ7pc~}NhlAi=pe=Y1F z+t@MKY+OZHPFqUc0Hc2!4x1Uyl|>`!zYk`UNx0z*q8qIsdV47#Mx!JnpQ1rJ8sxGH z-x&M@0-swo$5hlwvDVb!dIWxU84<*JJO^$U(Et^*=jUA?T^)3sGJQ^omE`yj9A9=E zc29?Xod*-FMUDlYb#oloZk^}lG4>oJJH0%urbY^z!kuLE8n4hUAOo%jGo@!k)U zQ}Lr)$E+@)BHO~)xB;PTU0?5E92FL3MD__7{@Hc?WwS#tN<|S*Ne5Kc_BAE%MB=Jl zjURv@`pwNva!Sh0ujaCnl1L$TelU@xk%k7AV0LAR0g+JZQ22f9Mc=AOvs}_t-j1hmHhR#IuzOp0 zMRs8@92#n+vTO6=Cjy4DtF-B*kQT9)6$zH}#(A>Gr6eV-ntliwTNkt0brvsfcHV@+rHnGP39ZbpFs#r(UI9)>>TgJX@k-dx4plH85z@?HP* z>TLOHOX_|*qd{Wn@88)K6&1WUK4TAr!!^tbYH{}XF${Eho^5Mrz3^J3QG943)$~KR zj@`5KJjYv;Ja_D(V2HwMdkk-Z@t`~SkUCQ0C&dm#Q_KtNx@v{J7f!q}XFyaTMI~39 zl!qpUil0V=3DT%7*lqV-6AI9XrRh>9_-as?@uUf3mY%g!@Pku+rY->^!%%=-Sj;v# zyk6w?I>v)ZgPJo-W7F)0K1}aWOYV0Z&EI6KY<7ihl#T$Gg-a&?i>vPpg3!SqzOcq5 zE+!igA;L(lOA4DQn6u@CJCcDHCBKTA0SG`!jd3qK#UOdnCi$~REMK6ViUvi?&WbQkE> z9hq<6{tzs)@oW#m#Tu9#ctjQUnlKqFBredTu*2T&M zU?vnq@p7BiQOxGO#mdy&HVsML6i3@LzzP`4Y##l8JmkEH4Eo&7Z}MJVctL?icBW9e zx~$9E>?|l57Z%}Jid2S`9#d;}keFCl3bJSt1VXu8(e2}mP3jrm14I6ChpM8*5sOZv zP}vmEkzLl&A(7IjqW;brXt~N8zxuGFdn5X{Lq&6C8rI`>7Z9hrs_3IKkzvsB{X0aZ z*hl#Y-G#rM4AJfZDgqBLFnO6=xQIj53bobHAA`E8k%~wHw#cRs1nld-FG|dBHQzFw zM4vvZq;Kf?zExLQ*BG6=lP40vP57U0yZrwTzrRO*2NG?9mNAQ|c{N zh?Azj%By?2F#Zl}^O0IroC)wpAfdR)MCy;c9&3!qAsqDd=sZAXHo3cA*E|Fh%?(v6rw>wpTHXy#9lYNmM)q_u4p}zYgPMRS9{r<`+w3r2V zg1))$z=CS{n7ltJ)MAf6qd8C1j<;-tQtm@9Eo#Q$(Gc~GOo>VJ7a^66>uIejw*z!w z^#_If$x8Pn*04+b%Xq+BvKi}Oe`QDW1`jL>5owBSt<$D2KhYO{eUDX**4L$s=l+0a zEJ(9dfa2 zw&S095x+vc(_t0vh3p)yS@_gKq%S)X)1>{!kp`QYKhro6lBJ6-1^*wud4XP-|i2`QOw3+kye zBW_)A3q5Z_HkkvN@R#E3_K$&;elLzq+iXS~)7h_|1uqliO3KQ!Q9;%-Bgxz)Fa7v5OsrUAi zXbLzZQTnwts;At8Gh`|A)r>?zzUxKPjE>FQHbfh?(~HGRFUr|I~wMxvpI)j2x5Q3OCw9rg9RRD$K&)iK`#F0+BH!QTPR z!_giSXB!)ftAp{CE~Dq((MElYWsUX6IW3*0OLd~RZ8c!QSw7Gr8PoogsN{E;)O0C0 zxYJ26aCPrQ{jYNQ=6X!cMp;y_izcbHfC`jXe-2f%<3OKRGO2Kd0g_o_K1pmD2`g!@ z(0~7UQDV|6PUHg!8l`4Ft-T>2OkN<%8lsHW4UERC1mKe&vnDL!Hi*l z%vmMe*z;&i7i#*dhNfhNUeaTJDLk<~t$d&`OauNmd4cdKrDdb~H=>dqWN|M8|9W-z zMQO9zY<|I}N(#VuqgcV`JHHqyJl`0AF>sY>5m$Y)ZO4=O#VVm7&;@zQ>M^<0kV>>d zdJJFsA~_z4*F1fClSrVhc9Pw(HPiiin*|FbD21*8^4B%DXH!2Xb}EI3X%et_)|Kae zUJK}A1P}|g9Ju9KXK4@RSdloHhr01T}g`sl&>fKRWojio-SZjF{^Gu>fD6!t8nO5P>A5PLKt+1t=!e} zT!Cl1G(j5RfN-v5Jw>iI{Fyw%gR%QeX(mZ#qS6;Ktgetj{Q1rhe$fUl#ZYaHb8~2N zKcSCW1(dZV>x<;0l=6*#V#BSj1%$|iNUa}`gXX-hvlJb44MvL~U!cdEEX-3@oD)l= z+86JSlrY_!^gLF3>Ng4YUG(`8=+#bbBY_xTbFufSE2)H11@k-iOFS7rg@Tn9l1WiW zXvukei-tS0#d8+@=9!yjN%WHzERZjliN|T%1qVC0i4Fe)nvHV&M3#Rr!Q>d?QA&yF zpAmbmYOs7Itd?(=fjxBJaIwWX_)chcG#6G985bn|QZTb6O@FOVr)Aq=4r4ByOMcd3 zs5I;TT;7DB7dg%#cS^37WcRRAkMp%^9y^<{S<|ijb7bC^Q7ZlIOu5!q(F8ft1Znal zp*aGIZR@3hW3~d(3aKnu=w{3ouJNl$iYI4qgSvy^g&v}Am+=I9V zF7!XPlFTd;XM`IZcVg~9)h6zg$vz5%Dyy`fvJ-_96OIV9#ZXynF5xt}>Gvyf&hx`} zKHerCD3t_te8%F_yjlX+$dh$#LXLWBWbtx?Gvf215I2t)x2%p36^-An_o>C&#V}aV zJqA?a-_Mm+TL5ju36uL-m9IOvqG0f=1^Tszy|w@H=aCXfczBVtu2s1geGR(L>mzQG zhm!NSn(?%?*54E&RI`)e$@{Ur#APfjgqeHJWPD7GHvPW7$aZ<>PI_uh6_8n{wdrf{ zv5gX-T$j+(-o?`io3mw`(1e_I%+S|}(EDWXAt-&RjDBJ?phuA(oavb`?mzl5!iH*e zXW7j+9~;*FYj>&Dk1(1C+x$hj-Ah@_&7qFE_5f}6 zsKWTkB%S{1Y`<$>n8Yg1wf_-bPkoXw!_6fUqbo_3K;T$#PoAED83DNXNy=(d!0M6a zR|tqNbpOi*m`z5DT}tQslGe3N7)Eu)!B#b#b?#ilJyBd^;_j4r6RPa7UaKIk>Ya-+ zhT86Bot0^jAg@qkvAt|KK(8$Ry`{PaESx+E2p~4eCG5Jr=Ya;eI-ash(w>KMdZB80 zBMz258?ZhalcS84BMyEFXdX;gaskD7;`HXI3J2oTFia784L0Ui|xm_)ZoOHyvLMo*;7%*Fsdc{j$)fiQ^_$-q$0$>GNN48#PTY7 ztS+}&oJS>OAVy{^p?u1Lp+ym*)wOiJdJ9FChVt?h*g~^Qv~f2RM9ld1IAdXt=yUHO zNw55Gjh~Tp6sZ8qoerK1qq?}a>Y9!)cP}4Y`6eOKc~*5F+7aKO-z0we&pO6fmz>@8 z=fcbcwz=)m1~(V?SC4$LC#Zd0!YJeFpa6b2!$jM94#@MSAn|o(i!1aY0onrL>;`o==zaX^R`n z9slCMqZ*yI?m_mCpeAvb9SsPB3AO^hbUETi+Qh{=tj!B8+FTHJoj-G3q5?3 z2^UHlm^L33$0}T0FNPjO-Aa)Lv*?1}9(Y^dmj1;HyO-MOI z$nmKP$~b}k2W4&L!@~9odK@uZ0fwG!XU$45lSd6y;GqX+90R}kVgn)3O9g6uEnRry3|r{rK41{ z>8MD}6+*+nj0-7upk7}LOyKx5DP#Dk`}X%WL{o-AS9g(%DQ2)DNbXyD+ZaU z*x(#sQFti{(KC*SSuaB7vH}`{uN7@oZ?RoBn7{!8pU4(Lh6ZNG(4|zi@OV2U`D4f< z!Zka<1-stbk-aM5TwgD4WfhGrKNKeo=D9D`f6(W-`DWgYe|3D4@T6;^hAAk1b8AWU z8hU4lCG`DB%{%!$1m`QOdQtgtguV8)>j(azYx-=$`RbeYl4(;(#s$f-s=^Tx^5TAU z-)N!2+vOc@K1n1~T_q2t2?>`KULPS%IF`J!T3V*%Q>LDWyv8stu zYK8%=(nA<2-?OKuUrTb33Ep^!bYGn>ZT}b{xRdRxn;t@d54aH6OH;=@ z?4k!gG!bJw;geEL#_Dw!bGg2znuolp-qS|UXlJc+@29MPn)9J%I97wkX#798@N9Nh zhiS2yqat4RBxwX?$ony*!1o1Q1SCw^-+B^TEM^PXtSwr$$EYidLn+;RI$xoBn}bT1 zx714JN81fk0{kx^ZlZ^qyd@|kBg3Y1Cj@`)JTY-uj(bA*ZU#d)o$wNdbtcOJYmMz; zeEv>^4Qzu#yN)NOn%CU#C)Yb$&AXg@*Hm%;1_Oe6 zS=QdGyV|`VdB48EZY(e&0&gPaBun!mv;n*a)6KZH(PWA6T;+RpQp`7sFe!#{XG{A} zytt~OLft(ti9oQ~qJOpRWMUrrLPXZRO_!Q?H;#|O!w_H6XBv*U9Z1TiF}gz&T0_y@ zVZ^f;f1N=?;>I}M7Lep$L2ef;EsFh40{Aj!}yW&Sw9c8)2R;cLBp zDoubj{@8S}Ebv`{tON=(2KicaH z!*8?EM|H7z@e*CDOw9A=GAdM@Vbchce~pj4IdwY&a*rlnY>7;-@W`iimj3+y&YM(c z_gQTaug_-^^=g8KLib|lkxZS${^FIDnfIm9=^ajC{D#2eoJ^S@{S~+G-<@oc3~M?$ zc63(V#qOJ5O<%60X(KOLk7aaDzqdV8pBR&Q8AI+P3&B8N#ae`Q%=1iMV>YQ7rDz#JZpH4$l`U_8KZ~XMQq^ zH#7BV*|08*$`L~FjZm5*w*^|!0HH7?%#=8aiMpr$bGx*oDcRaF>wBEhuxX8sT8 zyBz=4NFpzJAVRy1$5wAYKfSttj7&bM#h&y)96u>9FCWR`^BDQ|o|gFIQWm(CB^z$M znDw5Z^R^iWFn+}?-*NCzA5X6X-eB8di?MkMdrWvGCq}g-h5@I6_e;vmz3Xqv*rM zr#LVt=5SXoh4C_i(*wizH^anIfj$N$Y(e*{w6IOjI^sPrDQiX_nL6c~)~k<8(=H2H zf^-o(78X{hXuc9Apn$2OuRj8))+!C%>ofA*(gM5A0Xixbe~~({U&c@5Tx7RhBol9w zhA-DxzlLBRhtZU?Yr2H2991-zq42!Jz`>itB`H1XV_Gu4Z&4rV1~-4wft`Er;3Iv} z6yPN?{PqbG3cRNHdVCn{H^mEUDI2S&k;!Z{cv*v_JI*d^E5fIi=7njHu*g58$+fFZ zDu5dr#&tvLW&g|WR2tIQjIt@+=XCcirzfBEGqIBg5x2vcM21H6E`kZBR!*9_N%CkP zbx99($zSRcg8FvIPiC*Df=+>Aey~3Ss*FvHFB5B>vOBsY1X1Z!8RwzP>erbm)J7%qpaAg*MVp`txa1m*>iQJ zUh@829ctCBCld%)BhU!4>4cZchu@HnY^c&dia&)7=JE_7ex>~)J6l7!Rs!{Wrv^#PeXr~82@kNkCfiS(PQ9s1a91%(iw*4NG5fPjbnbohXngjZ6AIPeE-s)+8K51M z*07k_^ww5MV|r`p1-oyp;Rz)-&RjxTsI`7OVm|C#0EK%I|8^t~pZjqRFlP(`fs!z$ zcfUCTw%Cg+gX`$v(oH{lI9bKX$k@@ylP~ zvi7HmDej=HjNG#18ld=y7SkFZl}q7_kQTBP+FKmm=rV@R!oz3Dav$6n?;>-b>m8BSZ$DUpwzYdz}Ou zk~3w_C+)3mnu``~1B`<9ip8XN>-ipyn+ba(9w z{tFrq6u|rz<4ME^udbSz7d|KQ1vWDF6laB`=X|MM1J$J8XyAb!uDiK>sz%{A>i>kT z_FFudKTIZ55e=FXVc|xG47lOb^4t4-(7j!#?>@XaXt!kST2tE1!iOI*G+O`7R=A@PZ*_C6*b0m4@V_me3N zyhXyXKJW~YPphv|qk^wTTcohSTg?)-z*@>29gYW)d|B@0Cd%n?hyMfcakY?<=fatu zTj^RVY40kXA=Xqf4Zfy*UYKU{Pan53cu&#`xDJw=SO(RN3X&$)$U_T*i2fZF)Sr;V zSqVL>2-NGyiRm=z|MZw=#QvzZOF&&2ez|yj5{n++uhZt8=yq@Dk#!H0VoR)GMc33F z)-pv(Ae)r${;l484#H5_>!I@K&JFJ8*D9yE7t(W|XU^Flx}4K!w$%TdW8=O!E2g*X zRn7rMXm$|zEmjl{Tup zbcbx`D{iKY=a?fCT25kMl30GR<_reE7lR|K6@a z!XxFk{@jq{xl^c*2vvVRlUDyv7CzV7w78FKv_4IlbMcn#o8f6$lKIm2?sN3!0psL} zMMDJ&L9Ux`qGpLY3Q4#I7}c59!gVOkpf6J>2H$&3%YAyi^UWBb<8E)T#NGX@XA>+L zZX~Jp+YM?Cdn^nT$dgm0p2=sG`mFtIar7+7lWO%;8g zdThz#heNe1hb;jE?KSy`1a;?Y59~Mu;iJFmE8N~K&YW%0R2d?L)VK&*iN6dHr5<>w zi$q5t;!72q$Ju?5H8CaPPMFgpWQu@vN2g0|D`DB{^+r3ZqyX z?{m&|Rr8rA7($CM4*u-8b*&Ua8_FeOeCH=qRPD3%L&%(&YsgPnM{lUCJPVB<$CKGQ zByNfh`%NY)VX2-8bPT~QRXJ^D*$1RHyk2xz5Ta&SX_rCRx4l#Se#dQXZP#~qraIXE zZ-X~Ui|duH%z0NQ4JHq>-+gaJNS~{bDW{F!&t96V?&Rd;SCaCt0pg5J>1br)hicIP zg6}2UiVL+o&HYL@w?-RhReXLur8v{BhhFNAKJbnOu%kxu^M}I+2Cdk`PMhqAzWb{K z!J|UmSu7tsPP{|fi;2yhPT{dz9}%7z_z(>F3|TWc1F+aALuuAF-%w1XZb$|YB-gxgWASXrE!2&?j zv@SHBmS>dTEom^9p|Gp|Js-z!%FiHl_TQ2g9DJsRsf)4lOU#j!5^$^SV5mw?lnGo- zp!Cc1gv%-nG^ut_!TY`cN$vcxnKgaZ4P6VK|dd3o7KRc}4cTX^?R4-!lWm9H%xec1eg$!>Z|k~hNWQbv~eZG5HXq4E`- zO+B1!Dy{+yoAeadaZW>m2of5g2w6nyR!89OKj7j%PGpp%j=bk`;G zCzF;Q-PtcK>q0s=;E1!)s{is`mU+7!eJGP3%(_a(x5y5VJY^XT4f+{m^S8Z7`V@2* z1dqbOu}kIlrlF-3Ps1PQsa9-rEv}krD4_3)CvC+|NMmCW-JeGTfLCKKB{YnPiNNoI z--lA`_l|S4RStIF$`J@B7bvg%`PRhqy8^39OKqn7v|LSHGt@EDkd3}5(_553rz^%Y z3BjD%*Aw5tIIVmW(r`_Uz^fEwni-+^L`wY$bNT3TmDv_M^Jy|hD;XNQ{ycfNsDr|7 zY!@{)@i^-{r_YzuiV%qVCHL*&Q!xtjjF^rG_SPGEdLGC<#*i$%V?0g->THFL=5Y+} zaO+tE0|Q`sL*_1hsG6`BX$ooyQjR<^ot!@b1KfJI_IIq8qK6&216oKRdFLR`PO>#D zj~dDVeq9eMbJE=Xfx>N-B{mKDdU7G3NCnQIoyF77z$<9}-&)K936HFNcUU_{Q*d7J9dkgpGt^;^HGheW z#>86B7lh@cw&OOz`oqkyB+Y7SBXU(heA=CJ#BwYXx?q=u6Un(C>F=kmn2Jc^z*NaE z=Q}<&tKT}Z=8o|a?f-pFG{vM2k|xezBgAO$tae7EV@W(_6JT}%R zB?y}(l3nJ5xMCK1gyOualEG77%`I2g-7R!q$7sS7iy&F!*O;e4%m!41+uVtvQ?hZkBMmWnITbdUj zv2}}^yjCXmJaan*3mw`I0lt6z?X0@;`%&gsDJ;hTI9HrCvw)Hn%$k%;2B)+)FC=37 zg7=g1SQhIdImqS^fv8ITAqug=>QY#9}(1*X0Z3y+odQJ8I zsspF;3y?vh%Lrnig7fayq{-v(5gi-;fA2F=2gX(1wps zjR*_oj(%ddkf*Y`?7Dt;ga^Z6GgL>*J~L|0LrFa7OWGw;xQPvTGnfx=HiPUdDa1lj z<-a$ww5D!+dQJ_uh-A`7j#Bev*iGvh)uBHDKxrf|&L8q9h!}1viWZ#TBj3(t%1%FN zYo6L2{h}fBABO!v_zrdJR7LjdCN(9ttB{R{kUhNpx*CSruR}jsOF>FTf zkMHlX{bm7ZUIl$2j`h_gCGoJYcF=8#u}@(8p$DGTH&wRcu}d|BL+I$eWY037!CqgL91+RdkdK*7;ElhMuCsh+)Iban!Nwj@Td$Y9 zs8*hLt9vO7K0iT=?V8LnMo&}JB!#CQm{$=fCWNIhv9~?r5_k$R{Z!phTMv!WT`s8c zOyvi^JHP4xcbA$Ywe%L)*4e=G@YCvXkyN^YKb(ON`RIS^J)AA6ksi^wUbb7eTTZ2{{6Qoy8FefAEFG2c?8rY)9vn8J+)$5v z8Q}jw^m)y0cTMi-{!;D9rL#u^rL9|Im~PSS@2zW8(Ko@X1x8WB{fJtMR65BZKGCqWb3?MOi; zn>6PCMEQSbvX!1gedRGwCO|iLV~91`b`&{Dl^rSvS<#;|zz@8qXrOEZZF{ zH<}gxfJ=R44A5YZU`nV%;)H^1+m`K9>^{iTS4+vzpzS3y?e^{}7YPY^7@z=UI}(FC zF%^!WvrcNjS`Y#Ln&teBahTJ>k++*Ols{^~1n3YJElDxtBRiEobz4bbn5fVywgo!U z<1&&ICjKDm?$38otRE(+lo6{xBwVnojF+bDFKwv_5*Zoeea+eVZ$1p6n3>n# zKfWfoYVWu3^co%x`Thcpftn~X8>-Za^rd7;<4kn8F(%=}W5}RF50Zbb;>+4Z+`C~? zFS2$JR>rq&;o%Upu%9TWOG9QyNbjJ;G^tsNyhISmAeMO6#cQkMaP7W?ci;g-LYlmss*J)i&4j z2`$9(CtAnC?ota!uZ&jkC^Jr7AEnY9-FqD!1sZKG@$;elGuS@xsk8GbS84lWyG;IV zWK_cCLVtqK44abGEMcXsD4)DFyqpM;LbljktOqaJeq3LF3QFV!#^O$i5iWQ3pV%2LHE(GbLFBY%JItANtW-~?w(4ew@{-E4L~^t2;I}A zA^hi-!(IR#Yu)F7#CAXR{A3uLjc;eucKmpI@+4~}l)uURuF_~|Rf*kYyg954I5g;8 zlYoa6WKDJGqHz)Q_)qb<7~eLPN}(6!$9A~!WJ}O27VEBi<^bGi5HrtI<*K4fryZd( zC@sLUV@?MroBa)9_@F+&nIrA*vRB{sz5#^=@_W1`#d9aD52KG;snNGVuA$st`Eju2a$ zxnrh3-8FRJKSx*tVtthr3o9)3SQnO~+(^m(WNck!*{JdNBF8%@qQM4dI%6i{N%m*Q zVjHZj0dPmLIw)VQn6t6NJ4zDtXlq18B=WQAysbPwSd*Z4M(s%&E zv5*a90gY#il*Xvw6UIDk=%C(sM{NMb31w1CQ2Qha&}ZhdbwvQT4JM+}2qNWYg!M40 z3pCgGkP1X$y{+7V&99@GCg8>sdEgHeB|7)(dwGh*(gk5<>iJe8(X+#GW9bgL3qAHe z38IV1jnte$Zrlio%<2Plr2)CKDU;Nh>FfHokkV5H@&ZkNN#gEGasmr=vK$%Z1{9K7G?4WTOe7 ztU&Kln<#WzAA-0^p=!78wW4rCvJd%gkJwJTJxyK%8lox|n%2gXdw7*4OYTe|h@K}t zRIGMt8qh8+$1qX-HG-WiSw;A7_?}@jEo+-Rv0D`2RnaK;4;px1ReYiFsJ}PUo=B^& zWEJ8|9vF4{u+l>6q^kORI($%Aw3sfnsEtX+NStc5d4wEuaRFl zBsm>}bSH6td=L%PeGswoG_+gZ&fs6&)-)jRDUPlL^nWkW0c@E(kzW!O9AgFwnZ^32 zh-2aP16J_y%i2|Pgy)bdAcIsZYp~*>8>o908Hr^=1F69lQG^I2Y&Cxby5>?p#kL)8 zrcKPI0P_=uUlILP@np28rktF#jA>RP2efI+1E*xqaS)7YIxrR8!;_>HxmjJ()#{)GBfe!MM*wQHiofSM2?4OIgw3QIKiN{! zufy<4Bo>hC>mT{*7y4xjP%2)J8@tu(FgwSJo++Hhc9oOUzqH+ZPbMHMg6_?{d~^2A zV;Kb0KABvlf@hp%fVg9Zo=C-y$G5B;a1KQ?x}p3Ht7wbFP_MJ?NGRrzhL*baNj@u$ z@5r~`UOU{{9KR3Ot)JbHcS6cEURF%c&7^OmDN-uTrXvZ3o?eKD!mT-6Om1^Rm2}Q} zO~-1*-fDu?yKjm`0^}h_u7>`eJu-%;(>2Knyts=DW?8=IKCP{Vf|DEB&bYGdWMN$X z>kQUZW6RXJ=|J(0AYQCJv;n-1&vtsSETa%6&|32ve?*9y{mmKw{)>Y?P_p7tv1Ts~ zmKeEQ=Greb%3HJhZI~8v`i(nsAKD!QD;vhAr)+g|EW(^E6T;oD^J)qz8+b|25n4=Hq#oxVx&3h6vt@;jj|GV7@t=y=+XA zF2a4!C3QyHt;VET+no8**BZG?<($jQhs)=|?5+WC)_=u@TmamU@6eZuzPDLx)(TG= z`;!tdnDX#SH(Dn(m0r^<|92CWGC`}dY$x|f3)_rB13H>gpET`kS$)rA8chz%j88%L zvE_@l%eA4HZs|lL9}(KYys0_+0cb##m92kt^ON#Lgr{Kv4(AKJC9Q~cJ9FI?(tQRo zHoWLT0V*hb43!e2%7^ZFi8fTjcEFZ_S0RRNh1Zwh?dn_O(bL2{MR2_ihgds5%-E|U ze~2;PgusjMg~#L%KmQtNqT=6wTid=iy`Q!P|6E|yA_qiKSiLh}q?94$*4>njCT&j? zPp#KRJ~29;QmPXxw`S3nSZpWP+1Y-1{U@L1Z6HyhvOsi*L~sW9+BNB{UsT;_0u-R(NKgAnFTZ8dGahB3Ylip%6pT^t^$82CUzhU*@P1uHs$159F)F+e&S-I#KC&qs8l z&!;a$U+Agfn}TQn=4TH;LwM+dzLHuoY-hO_zi1-F%h>fVJOn{)+ucrDT`?Ux)t>$@CO0?(WzoYPgdw{n;4@)OV-8_fom2*M(%P^w$;-t$S z?AmmH;sl>vyb|P^I6W8LN|sSNTt@XyHM!7+q_vRhXlxZaCs^7ufbN=3RXe&LG{*@x z&!r~#Q{u__TV@vVHguMZ4F*;jbL^2KpndVg!6<0>`bPDLg z_6&T|1UDwuf*Z5^rg1eN<`AA17t4dmeEx(hCxrNnUAwL>7FW5PmTj}fsa7HcB$&Dc z%=u+)HCT%}en-u@WJ{43qen@JXCMtitrGquq9a0qhA}>^O6o%yuq>nTN&GW0T9B@W z&J6{!p=VVzH%|AbD%mpf=aR==FV)n3pOx2k&L9{a%R z;c$p5X!**w8P__+Qtt#&G^>n`=i8-$oV zg0JK2!oBOI@@~q^Jex8KxelJ#en&x@UDMy0M$GtVq>@hnqY+o*{Z7j3=tll;y~S~> zZ)4H+6xGxT=*ZdmR*rt@QgwQB=eI$;?QmJZlao>?LS#I=1=g>Xl$M97jiJXsd^GKs z0+PN&5g;>25>tgd>#@u#6Jg0PlStx8at`bnHymmigne7p+9g6LmS=rkdamb?;Wpo^ zEB?Ii73UTErhQ$CPZC-@E?6Hey+rN#ye43=P9XS&r=a(5Qz~kq?J<|L>Z;SJ6*~ABw`O z)~L~c=k|~7%_IuMd^Y~{OTef<*inO_Yxld&0~m?8kP^ygS5?1$a`1XS)_sGvx%pIB z`KmY~OB;xf@p0x)_ID2>8iKp#Bk{jYmm#mPzARqY9;%q~2!Ab@r7wtvG zMPxUcRr&!Nn`4cun9hE5FV-Xl>=C9L=e_?S?ybY3`o8vI6eUDjKw2f0?gkYQ5d;P4 zZs~5AA*4$hBnFV~kdBcY8tKjfM!K7s`5iu=@Ar2-*Y*DSzSr}P|A3r1XYaGuUh7`> z+H0-!Ps2}qG%1=&m*K*63Rjd00x%OfI`cXe#8NA8-nck-IS$C7y$@hq5y^{)mkbQ# zOrPFLi-taBe&b_*mh3{8)4c+c$|IJ#(37_Vf8in% z!b-V#r*7epg*S*`!bR34ZwtQLYlGddO8%!Ic8s*_ENbF}o16g#An2c;Tkh!3td9Vt z9ja~n@vZLBC7#$)R1l1X_ZMf|k{~|zk=IRK400g?W#a_-^x;((BK2UIi4RCE>SN4i zSs1{irdzAkSk5XaNCujW&ssqGXx*3I=`D8P+9Oh4V3@?P=VJr+44|CtarSj0oS(>WpgIza&58|1Z4- zDErxQT8N(p>BH#*ACtqHV(~#PxA_j0<$E5kpKuhKu*z5}85VZM;rm)O&p*8ySOGB6 z9oHJt81-riv?3n5h$QuK_i6Zzu|W0Zd0wEopZ6{Gd$!E`t=y{bms`@hl!Z);vQ5WQ zecWgCmG$IF#{(SZj5|S&MkD4Di*(ye>pxsO&gCkKNfjs;u4a^Cb8N%fON()e&hEl^DW+wI?s8^9^R-oq42Gc*Wze3f~_r!sD1eQPwzlYo^Lkq%yI5h%&cTJcyVFI zEo`Qg>iMCWqxX8B`it0fm+R|mps32o=p@;uG<{PjQ^CaOq(Al)X$Jl~U4TISZFP)U%9?#apmJbBJ|? z9z+LKRZ_98T8}1MM!{}8|7N3UoT+VXX>b2RQOQAgh?8Dr`rii>v$H^1S!#pJw2H$n zQtHOFeg%<>g_@XIYbDu)_aw;09^F*EwnG<(b9NNmBsu50Wr4dOG;On~{f}qW%YNXj zHPZ zDE6q7Y%rxNRWrX6-Lyk0eQ$2MjCp|0EO-BZIY?WtqAOlS@MO#7haBiwA=7JHWNuE>p)sA^pe<6z zt~C~q^K#{vR2CK;DE>zRR+8)$*Aj8l+fL9}j1Ufk!w`{{vY)_rzoRae&cU!d^RUjP zFAIhS*;t6lem-3A01cbAaM_L~K6$51{o~6IwLlw##XGpi5ks$)HkS>lG2;FHhi7m# zAb33c6WJ3Ge`|4%P^NWtAT1uy9|Qy**zRKz-}*rWsO{A#{?Rd!@yZ}ZqO}j=ymc3< z%4v%q0Rvjg-;uu@x|^P^#2;+l$6)5DK=T(aDbikwZ+X@-KuZ(y*-tT2DSi4>Mm7ZyY$33gI2ToIa!CI-t^~_R zC&x@BHhhk<#Is6r5nQbL3<}(;aZgz}`7I!7fy9)#&Sz!`LqB94iqj2`2M(B3!dU?~ z92RTJ+uc(DDag)v0>~%H{+1ZH`Dlbt$X?DK-F&t z;mciw9b!^rdF29nLy7=up7VGQ6}>a$j=G7R4wXQH?6lN9=2~2cV|0BuPcpD5L|^PK zmmOP-%x^eLD!{_R@FHqTK~|xkt-Cd?&Q|MP)HNj@$g5h*7S+DPAz}D=>zn(9UaLpt z1-$+)jYh@ihw1LzWN(JMkH3fRM$3uR@9Mnd(Mj6PMhl^%skHga{$+4%4ET0?;aaD@ zZAcG%1(ejH&;YG>e)%$JJY6nvdy9Y?4J4J_uE(M~5Y+Q5z18eMF>C9V^KI&MP|qW~ z*0XmLG65EXZN1C4XXuS3Z*+8wp6|vli4-6u(Mqt*B!WAbgPT!MJfs;ELqY&z&`9OjAp8-;Zu+xbY=-R`)4bmLR^6r|NIBQxsBe04dD3l zMFjzg{ZR!Ys|(ZRU!|K>bKr z8k+Yri=Wt8j~ZuPU{vHuo-DACq%T35vCAJz3-F6so>rQRwU{AO!!5|DVyuha`dd`W z;b4o^{T}8xe|p^9suj*n0^I?LJ+)q9YU;>~tK5{%8DFfUSm`#1h!bX0Hr?JKk-F+1 z7l4okuk8V;QEEff`gI#l5sop8>5U8Ll05jatq&T<1YVmeX1MbBtnJ5MBzC?u=k=cJ zlI%TT75Z#YaCCT$*In?Nb*+3pP)!AwgG00YrTK!yFQ6o3?0Y2=?+d|L#eB^gbG707 z&maQU8Ylg1Ix}rZBu`M5`|{$w-*dGV1l~4R$9X`@r8;{sI;Cy%i}7gr8+Cpehn_u> zOOP`0&8;Mzxa;$Zq$G#o67cJ3R|2h<>RNj2_mZ?)CrrC?Yt0d1l)D`pIW*8Zg8Od@ zQ`nd*dWe$%#R=pBtESvQFYLSEuK+z+;n+=h|GVZ$Iwu88;K;Igax4KCv6_5^8 zUrtH+5XyeOJd#HD#W6)V?O0eJq7hJVzaIE3mB-+uhy}6%Af6Ql`YUeqVOhmy(lY3V zrE#lA8K@F<`#oYRL(bB&@7Dri$l&T>9%Zaq3isrw@-d>~)9EVTdx>)_bK&rVVFe!S zHNcg3UQ1M2e?6Kg9`VzT^5d5g3ksZ7ZM{aPjLa_n zyx%!+18^)%0CmlR?i^F;z-F~pTpVNd@q2AbmQhWmUdG1%fIhr@mUan*6LD|R=jP7&I5lqjm%i8+#;q4r zH9zX-Q)RA7o7G*uV;<=mto$7;rd5iSO(4q>m(;A*PVtRNZe4eZ@Z~2Wr!;={p)XpE zJcf()!=9pQq+{e`!2$ZW?%to%Rd;z;&T*>#ksDsXlwnLvsmEM;xOrMz-^|Qg+RA-? zVm?w%1Je5P1Snll14h3Y+d2gl(91xhoS@cAxToIcg6L8|NKzb)k(>wd--n;hdE(BP z$0oaY;_E{Uy7;d;;}|+OG^{v^#IU9&rcb23>cZepF!vwsiS0fU#~8$e0K0k)%7QOW?pJh=*79= zp9e0Af4)2Y67vl@0OAt@Y-}v;A(Pf=4i>Tn(%!o;!ae_BiWNe_---26OSl|k*6tny zKdjv9pP2L9v+<_emLg~(u$M4_p1(J9wJ;#iSTG(TAt0mua9o`T)UU{;VECehu68~$ zXrNo3AL*>4=!UaN;b@>94i4eroG6S$tFMR2+1z{8ycZU#O;F$gri1nh4}@F7!gt+} zaOkv4<>PU_hXd25eHj2v@u1L=nBwx5!J_AekMiVbW+qRixEQE~=pZWB7Hq0;wB~r0 z(rTWxw4@KnKw*sak9Qgx6x4z?!j?Cl=pdjK?bb8(xhTHEwF)sAE&0c zg2F5n4ZiK}qx|OhiC8=!Z|qI>goDsG_CW2hAtEb*MU^7GaqTGE&bf~{=NfZ$U-v$^ zVU#h7Wz{to>U^(_(Ei{CG&s&l%~A_Xh2Y&(?d}u0iOFufavLRZx)6O5{K#$OCo8nXb66 zxnpRc#=STYNg_~l5SA5lr0@nSXlU#@5&`xZ`-upR4dA|74o*e3 ze5n&e-#!SrztU|sfrTA3iM9O|G3OUzjt_6X7!_fEacr__*)(?#eB4*?glvz6m-jv) zAuOZ@6PrdgendFPbJ<@#URjQSzT2vH6ly?`inS=yH+FF(?6dhO(k*MhdCWX!lAJEb zX6?m~;e_H)Fq!n}Q!B~k`4o@prbdTf5TSkWHJO~=V;}FzU|+Mu7I0T8t(m|O7Om26 zZe9m+De(7QiiXtv%g11Vt^DN zb%>P9Um{LU&eQ$Hn}Dv%%OiL~QqtN)^J5))>gA@ZwY3LyoUUIy&U?E5rfjKg&tj*g z!Iwi?<}PHDMzntYY??cgolo6{y5(+fTWxP|Q$fWws3)*$-`FoJs78hbl~1cE2`%gico^CG zwC+z3*>L59^&WbsdmYE#Y%I`<`^^X6I3V*ZGFOdx8=SNtZY`}=N`9^7JdUgCYQgbh z4!M&4@b%q7e^Uo+isX3rU_Ni@%Z%vt4;wr%xZUFK0a(>ybXoL^MG3F=Ke5qN%4thM z6;hHSszqB*s)(NxNz7N#*c=eRXyKCcGr7Sk(j7&KUE_Yw9l+H;XBx`pKD!rs6T(1} zJ~!(iuhRfFFcvAmda`WnI-%lZ%@U;}sXL5MnI#X2A zHmatEZ)s^sFq<0q})sTP@Hdq*MknGvHN*N?vuN)4`7NlD0#WSXn!U$QMNu~=ZG zTbdvDZ4KMzDi53YyNK;BG}a7R{)xWVBiVMj7J+voa*?Qu^=1CIojp(N1S7^!bZd`I z7p}w0n#?1C;^xMxvtpDDzc>Gs^Kl*By(|MYp!{~!!Gf1lQER{`XP(bZVxqUuPM;i44u8@`uTQn} zG27C}N*_(kVa_k-bni2=#mL&E=jbb&v=?j=8`Nk!uNugtuiiX1!!A0f>$*^S=i_GA zUS=cpCBu_h%XW&I=7jpqh4c+~LPmO<+Vqg$iK7lR%T6Cy{ZhaGLStD`cKP|I(^hyv zIr-0n#0LF?5KlCLcpEy{E{~~UDpQ{{XKJW-)Q@MEFcU}29d2CKmDkgAv~TWEo|V=Y zV>hh~YXM`cwPG9Z&BknJA`KDyCvF6El0GRvmU-9JUHoXd1MK<#^ed8_nHk5{K# zL(q!G@OeSRn$Gt7zY`6@M?Imdr!*4Z9qp0Hb065yPx<{KCEH}k_u>1P*oTFVr3iFW zG6kL-;YLabUo8aOw`3=ID&+n%+(fs>=$dNxa$f0^-h9+L3H_3U3-24=8U3DGSL_s( z3xVnB{m!nfzIf1-bZK2{Kw4}0kcn%FcyQx3+hCAEZOWj?lJ~RP#Yo}Vox;7%VMluK zu9DIh{D#PF)V*jAEjFILrQoVuq6^H24eCh(^HJ#89(0$Z7wBZxGNwM&<9=l)t|}Kn z5HdVmfunQ#SF|HHN6T;_>R=(VsH}?kxg^sw0b+scE6A2%zGnRvQ)E3K#Z@nU}we0mWWpr%m^W$XxljT7Ly-80d9>6pDeFIz^{m^D70mbyeuWguedC5`-)_IaF7AJl_~?G(*_33BY# zWCwb(u%}cYuboG{(a*mE;T?nRk?Ng_yb=FEb|}B!m?+F9Kcjq+^<<|rKeILT3MD4! zeYWPzr3_N92@z9>V0GAz>NXQb64EkNol6awbKkqYPpI$G9NyXBEU!mdKaGq*o%fr| z=k%Nl(u&t5dRS@8%rANI7R+aFx<-pXqLtnC7AY%ml*n0JLahis;fH}h=&x-c2e|1% z&w?V9&(24LhU1=vMH8(=&qZG6;URTs-_S#w$%c#LJZ0~s0X4#J(%sQ{fs%p40Fk-1 zB@ZkeC9f$CbNm*}mvmzCt)5=Vz(W53Ev|-vfdSCX$NlqJw$(hLhL%=XNC@oV9(b}Y z+7i+`d%agmX#@01O%TrW)DMK-KqL-PUP*PS&!_m zI!vdo%H4j{GOXD4Y&@%iQleV5>tY7mVD-#(m9l;Z{FfC&>=v5{ohC-9uQ}LeSFZVp zBVo5@MFZmT%2{Y^ekh!n8i{Y_vum+(+=OHl26t}=Fh$+m_zq2xTX5gI{M7f8MY1Qa zHelv#w%})Hz!s$&o@t)gHjBWdeI-2F8Z-E}U-q9N4D>}2yxd``n2%yyG7+hmT0#!$ zo**rt20z54<;yQD^M?x-mN-5kJ8;Lf(ZZew8p~UJ!noAiEX@9*cP|gyXy!f6Bi9N0 zxw9bnzMF|%a7npX^ZA8K`pJ1Q>2Dr6+gI^hyW1}ty+uJSqN{pl!xpWth>5TIcv8jX zbx`j5pe;4Y>k)G&RP#eFF22}Up`-a_r2>1YWw$*RLAMuelD-~iwL*Qpmn=Zh*2k1i zxM49Y4m)L}qU!>(GBO~=48f~k;v5;fyD<$1ZK<}*nHcvmD;S>{vQkT35~Am;;@Qeu zN{<>RG}Mi2wZmFlJ+r%$@;}15ZsTI=ZA`YtG(0v_b9<=dn3%9Vcgku{celEuR;Q8c zCe{GjI^{}Dp>le)$7*6$T(n-?Y2e(|m%8?XC0c#P(a||?YV#ZECo6}wn8ge36HNm1 z;!4P0EG5^52L6g4V5_w)>VnM2b;+v%Cg*VjMGvc}nF;Zq-iI4s=6f6TTkoFI5aYDo zbQH{fa_f!#-DvvYaoh1}j{()+2G#WVlRlXbw)jtc_TO*&Kq<}-L>qm**C$@cP6J}Lq*-5 z`XIR$&Kl0Jw~g z{>;nGZ;OtIz#}6|6i44iaM)20gM3YLwyGY2SA;Hx;hEl>B$Sj&5XmK{J9C>51Dob+w>2xd7RI(h{@`O1y*br zihcnnSMZrau)dFYr=mONw@$S+VUF#2!-;TCy9PHQxLa=(Kd}&tm2$)CLRmhFEZxD% zX!Dc*1zF3e)Fb5rs&~l;>vK)MPcH;t^6qWm`g2(K%#O@PtJaeWH%|Js(-vfXYT%@Dvk6j`au{4|TqOuz5Ke_HHcblI1pF-j zUPhIkku7(?$A!N$O8si*;1y!D6L6>B9OS6l*)AWVld%GpH$H(TMwy-GYf^X17Xd@Q zgqjxc4h(6W^pYN8Jz{+E&xhiDfuf|I+w31cat8ap@6VGD2MyxpYHh8@^YwIfQv~-2 z@;^5JRL$hRL7me{6x;4`X8JbtFPviyb3C~rXvRlm?PMj}!y}n~e~#Gdzs@tKlW?$n zFYeFKxW?C8^{u=Wmr_Fr7MV+07hNK^zCEQWRMdIS_3yKth@I7Na$C2E7oGDHJ}m z+=%wu&x|&WEt})9E@HAs(Q~09tbs?H{h<8M6vv(?_{6#0v?ZtB7FZagg(G&P^ybT# zaVyXVQM<0UKdB9ag|L*e#JvQ6dN(YZiN! zgpyun{par)>2b}k_M4i$fK|g>9PS42TYC?Uq6|1)nY&TrlE~%Kp zkl|mlF-8n$oc_MYDZmCIzavQe2HMvA zYK&U6gZ1@o9y~DQO}!g<-SM&0qeqYa zPFK(aevQVw=$m;ApgnPPK>8VmnB$;Z`IitnfK-7 z|0_}G**JpHgLREAw;Y|Ea)46jF2I419GkS}<{Me}`J0GVTa3jfS75eufS+Axn~`O! zq@)D=2uy+d>EnOaBalwQOE&>oF*hYrf&W6!v!zH-R~(t}-_ibR@3X6)ceNz@uZ%`# znYsV33a>8NeG!8@dsm`fW)1`#C#I&>4y3;8*k}A+f5{^(<~}5j$PNtgJjx@pnQcjw z4k3U80N3I0J?Lb9OV?{_rDje8v_wD$_Id7oL4H&KqkADL`oU2wu(UMhy}^}Wv)f*F zetvKp(gzn0354#wDfIO>l_=`ru$U8|sCku-NtPe@4#o`$X?rTd~p>YqQ5dT%K- za)f-z%NieWakGE@ip-SoCZQ3JqvR{dJoB7Sys5ZzMa{T!d5pWrTCV5t!gcU6!6;dM zy{h6#gtRnx*m;C&;-R_>ykAMAe4)yOrK07Q$^di)|1EiZhGq06tC zn}(v!>@$%A%SDBSD~sSGLPXiV-WTuS z>bghbDC~2IW8V_nE=O;>c5}ckEF$9E#{`m*T~yrNBXx9i+)_D+z?!cIx7+V$VE;~v zS%w+@o!Wv|nw_ZAc|6OWL`1z~NUOiq>0`Ot(rOdWfTH)zt}hM-_Ue^$)3T}4gnT;; z&`k+IiK+P0E(SeLX1QpZjs50>PGgbY%cE%?mzg}xHZFk6VXw29eP^}3{FIzKSC3*G zcri!UJNB!qtJw{;e0DL&Ep!#gmwd_B%Xy~^`rDAZ(sjp8y-fS#aK5r;?KWAHyZZsZ zW6k{DT^GkHV08Bb%h>Sya{ZhHeggdl!K08Xg{h9Tp{FIEb7DING=|5Aci;NAS+W z)6;dd+rBaKnqAQDma8Hc6~$JL^9k*Wbw?N`CL5?h!B`ux{(GTetzI zP2wmvv5kgfpb|O)B|-?_-0c6>i__BTeFD6L^(Z0U%k_9=&%HAW2Y0Cx;k?}SXJF8= zmZ1Goc!=BuCuK-@_#<;gC`7{h+-cTKI#t}0L@+HXiW)tzw*I6IwG9?;<*pis8KK)S z2U8rS?q1uRbztM_H8+4{W*T@xEHXVax!QUUfHhQebrq10;o5dOh%nh8!}u17q)axD z-2H`Uc5TMM6>jbonS(-SB$+*0d{s^>_7W4-qGG0}rsn7y&_*|fvG84;I-4CQNBIxh zrt|_%>ApihzyjX45nRW6Kjrl7p*QwY8$_lg z1=lIgfnVkqVC8z=yFcGW*T^jk4(aRbFYPvJXp~mRP>BrFp;zc9!5QsgqypzJ+mT~hnjanuP!cE){gZhW^f zxN2!R6I`Ts)3kbRFvq~MU^g;^9F(*;mp=c@0Go{nKX)$@&dtr8yBjt7Bpz&e?&0a# zD8YM7@Xrf+DCX{dEesT7H~HuKWlzMSc}#L5Z4G*DQ}drI#N79{Ne&|N$vJh6(X%NI zH?;N2^>-7ntDS4=x>+l$pZe$K22jbJ91+1upEzfL)w-FOm~gr{eK}+vCoCq$x6_u2 z(bDXummD4jP?)V?@M|c@w{fpd=ZA#*q#DChv2%-c#GWDG7h5bOLzkZ|X`VfDzkQ|c zXl=0&O(W4SFdey`>Ai1tUgifFvHArkQjBKzy{#`*H!vwd42R|22J;wi&(u^Wi^r3b zlPzdeh(#zgao!%%i%EEkrK~(+KJqd2F}C4y8|F52_!);S6ELmK3&O>icvVEpU}i-F zZ3*NGXs$v;y!8s+KO!|^0$jS_GLHY4)=Rzy0YFd8_`Z1eWj&zdrWCxbM^7pV@bZU& zaqU(+7M7H46qmdsqPW-rhT-Mqglwq7*qv)F0K|Nz^10%VC*8L70$f9*KSRQ*Ou%lV zG4Nr;O-}pyp26X{j`h`9#X*+6!7}Gr+t(C5lyvedw+Bp+_4JJmU;IC7vIpU8_Z{5b z5kP0+Z55u`+8ydrb?TpT20`)hwSYO=`d>bVovu zn#MD0Xr=c5h7M&QI_>oIXbpU80|AAUe)zpSl!;ehf5KaddX>{K^l6ItBpK^OA#O96*48CYLB5 zygA-6hyTb2&`ra3F)@vVc0UT)ZLy$XYoyWU6KPl;l_piIECBB3n6#+#tYd_3LlAW=n5ML z2mElA0WgO8l_sh~EFu=z=M?bu^=}pSknv1s01X&O8JzFV4jjQ@T!V9Cc*OCT%lCez z;C#LI@D>~4bmnw*SOB6Zy&ecXT|K=N4{Otcu=&_dxhiej(*t)DTwPzky`T|tGRQY# zB6zV<56vh?m5>Pt)QV>|dT#KHX!GF9wcnYYwFgu|8gCWE3MKqaY}D zcXc!)&5mjW6M8cglbN7)EI+IT^n1bz%$_8~!B@>^V=HM=JT5A(euQzM{z%)s!r+m* z*5@M9zjEO^fGFTmaIT(=oB*BmyeWO;5id=3wbUkeTiuF-^Fqihi0gq(fXHmX#&B?S zTs@y>;q~pwINgGm0q+8TU@y}fv}!p(K6y~j%Pp#|BrD6LBWTaWxj%inhfw&uD!Jr| zpugR-=K^Rc;uw}84FqeBqgeqJ0HD>U=@kYZc2M`cAv=TT3>P!N9do*Sl2TPnYi1bj{Rak1d#P$%SXycfi$=o%C3n16FDqWR z^8C7%x3ie-!T@*)IXQOB{HF-7iaI{KurS<}2ie(D_UG*@=VEs7=y{973NSLw0Qu?% zKpQC{e}?wl&oI~Z%+H62_i!Q$Q0bYK0?^z@QJsSP757;7^baXV5>zYz?WA$GW&$MPCYkLM@4@DZ7_T-0m zO-7r;%C1o;<(GEeP>vUpsF)`d;E{lbgnUPJy#O8zzEJOcTsXcQXhC0p5A*uy4?W%2 zTQtCud6=R|yQD5z#+E&BEi4d9PQw}AXf;(;>9v3$56{#qr1V9}vQz{JT5E6QeSW%D zH~&IjK>-n826=VG_^&*8O&G+m=};d`NG;7;jJnCtNM3&>y#qx6Kx$XulDddbIU6H( zY(SoeMWH3#O4CC?=%*3n5+U{O2jtc{Zu!FE=!FPiyvlROP!n#Ko$7;#SO!cTpM~!3 zpyvI|!KpGx^;x_!cw#;NQS^WaHUC!(x8@9BTc!Vx7#%ditI1lf>%q$FqYDnnWru<= z08YBzcQssSyFF&xZ?Yy&R8wn-yY?wi6u)=6JxNL!v;SmzdeBP14mfAdEq?3O6G;hF zxwm_ScTt44dcnbw5fHFAnCVM6IeB{GO7h>c0P5^Xfl*S1h^{bFO?GS%r}aWWUwe69 zU}-DAivVSJ@uH}RxaO_vIxTG0Vv;{J`4U0 zD%fJDJ~xyAA!Vv+r#2@e~Ez-i^*E5e`ZFnpuoW zb)8ADz)>Hp5kvf_i%&+Wp>;N9(#ixeya@c9F)0L`B8sK-uDl6FT|dJmT| z{(xUgxkkUB(r_zwH0$I6RiB-jlmg5P*MT+G9? z4cfry>ROn~GhW-=?BN1m;Stb@cZnU`{+xBD2d57JR-K6I`oz4}`Fx)P@Z%R}Y+%j1 zji=IU>MBm(0f#}TS;E`Wk`nCk^&Va3qUY8I)8*yq)@fBl*cB6oLUP;KZd!U=Twu%D zl(rYJYkCgj$4l7&Y~1K9)cw&s6`w}f4|zjyIHph`xa3;;25|9?Puy3ww%WgEC%ep| zO7a?pGK3C3-Rr4jAwCXFOZRiv^1RMv!wF?YxvP1s3#<*~hSJ^Sq7m zvQavZaBWMy%{4W>4J?3YD+1ymBeUSABP1kw?LZ2o4{Ve-v+%j^mvU0XA0I#3Z`+Xs zcBNzICHMXN!A^!?9yH2b(7YyfgDpvLKyaB(ECcXVj;H1aD7De6sK28s`e0<4Ij-M` zl6p9$^|NZktIg4;ZYULD%% zrJXG$^)X&BLF+dS1Mh5wi<`VWvjt($3v*{eLD1z$56nQXDjWd;6Dundc63%0C_|Q> z05RJ@LR7?}9pVUva8{W3*wh>TRd;r)ClRPvyAIIsop$x4|4$Yk*j-OGG^Pb_{@mWu z`dqH|&Gu^@fZsbFT_}cMF)_#gq*DYW1auTZE#^v-|M7<7E?9y~+HrJbdxZyMD{qJl z4c!6$B#AtAzOzsO!{BV&`xICZ!8?&@uTp7$iC z79%!*NCG!Z0|X_$sYyHdapSix7-_@V6s{gGGjlwP3) z+@4Ec0nmsx<}p1;1bvObQZLmEYBzlLw|KRy`=j-=klyg{FuVTolM)Sd#M#agVwt)c zfSV~p5)Fdu77Lyhk4<_a)zM4{4m)>cNQa+4(gU#v{+@v(o}5%4+@#YL;{4Wpkoned zCGpX((CcKSSuO+54YQVO?MGakI_*7CA^`n2RsrF><8q@HTwvYMn*qSKFHrqE{}GYYsyK44x*F1ErcMtj0!2Y_qKxKw3Tq z^h~Pbc*D*9OA6m?)Cn>686g3K+{3%%3UzjHAb=0yQuMYxi;M4p#b@rmjRRI3dgn%}uyqtiG ziz{R3zqI^kz;o(0u4279RYh!U1VJE>;jjPU85{p5BqzIv!M_aszb3m~aPvIbQmG}v z{h3Lb<8cpG;i;MFK^qBs>{@+_^inhj*L%T{;mpr}V(iE|GpZkx1e@?iv2^A@OYWF> z-CKZw1jzi|(uaMDRB7+am09^HdPr?J8NjZOfTBdb9JhXZBT?-*3FaSJOvkJ~1TS{% z1|2<6dGF@+=q2A<)=JROoMqc1pz^H#hnM;ulw^;9DaMv7-Ld#v>mT~G4xcoBJMQSg zS3IczduTto2bbgKLRwbGPmye>z1Hrq`rwlf^he@-(XuU+Muo+|TqVjB*kS2>sPTHf z{lip3wxSW)&Gfsh+q=-g`|12UdUkB?iZ!3s;@3KIFP$&piLPMlBKc6mFt;OJQ734lWG#A$h}uzOL_E zsZkZ4b0i(#``CNgZFJwxY@^B^nlj4pS%nt0*)Bb%m&(W!(MGV`34?_)9}w)wjX!z8 z>PGOUKtliB8~=xo{U4;<^9f543<*^gsv(cl!MVgkV z&Ql(hcXr3Rk*Ju%x4IZ_Km{WV)#SOnOrk+Kf0%5v$b+~({Sb)wF26-2n#N?EgTt*o z^NT;W>!ry{&Dsp1pFTRXQtN|hzaw23c;pAPp6uc!iSw#vnuARFeQJY5exR8Cnbn(r z*7$8{f;M}CX824}W^aoPj^5=1BmFXagYTk{UnG1y1IF6AP6%pp>1QW>{WV1ND|nhS zNm}0w6lvO0DNV^W!=9Qvu)detC^%?T^+0Qk5~3dK){sK* z@EM0d#i-|fXhPO$HsTQBNT^_nU#aWp!Cn$TT!;WJbdS3 zo0)C2j=J`o@Ob~OWcf2#rCh_k-19z9wc7O{fI}!gfVpJnzDb96xq#UagXh?jvDE(JT4nBhBa0vr)7|^fP)(mo zAnT+ED=?w4hnAbbUUH#rTV>a(34ak0)UxX`wi#vSGVC-}5h}sd zsQO_zKwgGxn91)?UVGav2o|qI*V?;)$sWTQp#s z#?*Dv!CgbH{xJ?eE>yvy_(&v_l9e^5bn7Q-lU#r1?RDwOAoIc+QnQy9-HjM6l zQ0d`i&C7dt^Z8G%pV|rXhSzr1N$_K;>|A73?hrR0JAZoUOwnK_)f=`(mX?m>N|3K~ zN!dCxkln#J#rRN&`C`ytUlbcS(;?IJ>@Jv?yTg6W$J_zodA<8fc&(#{&v;3kBtQ7X z96p%;mN#FH*Tj`_6;G(fO6d<}QaB}UC~o)nOliARGB~iwfRuDr%pJ`^uw+TGr zJu%W0H}L_#ZpP=R%?jOJc>OkHqvoX}dC_O;P%#l-TnuPxiKgUaLAT%y)>BJesQWew zX`)rf6rS5+!^_9pH#q#lY{^O`&P21;QMN`#g8@)T9aJb-_@vG6*UyZMjI^!pRKr;J z%RAdC)l@9tIYCY@W`E-hwS7>IsgtIr`hKckF}x)w)Ed+*<`Wor#E_g5z)PdnuF&^` zx-W8)sK$qn&3IlzYo3iY+z^m8oo~;bDI`ta9$`lMe}0g_6rY$J!cqSiTIg_#Q3B!m zu8_SkaV>B5L>prnV>taM)w{8tsID}=6z*%u&5!cTHYVct-f{n+h1yqL5%HTVmh{nE zr`-sxQ9qH@{q)39`L*SgZjaIj2BWQ?G7`jD@t607O-tbM(zQTDxwYxddwQXV>cbI8 zx#BBC!7>Cmh@0OR6#%5dI z-ox{#^2P~;J{HSF=h^2?NcSON_GQZ(09tY3{zq7y`LKu~E_hpfW2yX89lH&z_kFWW zRYm)@uDERkW4?L%3xsra+1PA4+_#y{&Cjf+59BhcDq{))!gwbppbw8`A5`XM*TwSq z6~5+sSyW_*!&0=Gj5N{Y=S%JVMX7kC_ueN#s;639g<_CpnW#UGQO>w`Zdm#>O0TDS za8yxCJgBR1h@Fb`^=;16F_t`1iN4=Cmzwt{gQj{lw%u^d>VqfD3T`*N}FM|Udu zqxiMZM~6YxZ=KYc<7`h%HYc*&7&9YfN+r6UXZYSJ%7D8S9ea4$Q*Kxz64X_oD*rJ( z1pl{q=d0t}gjhhe48Gh-xC$cOonJpH z#L!H}%dUDLF#P)X#cpF%>4ow$9#^_}6&t@)qtzNMlsSn7-&QdSCrrCKy+|bfeJU>m zsGt@as@a|-@P<)5<3mRRzm1=#CTo>~q@j`Sr=30y6C!0B%@~chFN$#a-q+lFdWaYO z!g46bye9jfBM5VyTF)4-Sx?A^_qmJaeNH+gQ^E!C9F;?<(i^iB`U4fc1biYbae-`g?l*l`RpdR{+x zhi~Bzzfa*Ms!W(!Y@?I9&ziX=!Fle$M6GlUOxe4qC?p*@xll3b7>wRDD1W?RYkftT=U1pFJ>PX1`I=!GRM8g&VYN6$`m zm}9v#^A~Y>#d9y%d>;GZQywgK<2@|diW4>_3zW-PxKB!xY79->p6QL@Yv`#7;i=qkCh+wSl?0MBjMeu zynTArLHLgMqUd|;n7%of^m9J$w0my1%JASvHoH{(L_+tVX0rS~HnkRij{+Fe(lauN zJF_=I&yZj}DYXWdTUs7OzO_TnXQC(QnI?2B1OM`c?O~BBPHJ)#zSLV2bCMnF+|p+e zr>-y8%LlTvX3t+BmRJH3Th`zAFmDDZv+~pc)iubgEYGaQ_Nf)@`pgA;uq}B5U2s20 zu4*P#vgUZQRN1OC@3L(+Zi)u*vx(7K1>QBt$U9+byAAXC}8_e%2=uK@3;AZ%gGJKn$x55{HrjrRb6{)gR zjQY|2x7S5w(}D!*NJSOaXIWsb&s(G7=D%z+Wv@T6UY;r6#T8|GdaFy$VoCyiKIJ%6 zW6CS^oI34!0+Y@Z$#g@PCvi-!l3`cN$Dw)(5riahsi!iJ1P?_({7j1^YQ6e4w@8s* zB5W|RXnn|T4jtyAukCh$nW2ADWQ4!#K)y~ESNrUG-R1w(_LV_(Jk8!XK@x%lcXxL? zxH|-QcXtm?a0?vV-8Hy#aCdhI?sgzt{?Bvm!>xKhy<5ArwLP<2)3d*s>F(+2x&2WP zp&uU#OBV`6O2cQZb;5lqJ%*h~mEmwz3)QkmkyE5Rn%|?}7#eMz2|>QL*PT5UAP+fi zEwiN)0GZU8Ch+qVI_X^Qpr@szhYz}|YhXDiCar`5(b}UB=vyqBU`cEs5rY0aV}EC7~{5 zO%kg0PYo2-Z9q%Pn+q?lsG-)!H@IS=}Q&-w0 ztbr8aMRTAWey;EkWPen90GN zvS)@Ia%Se-vH><|+-Q=SS|@efx~dC3q2YM%)%aJf7dVV<>6MmYa2HGFiXrYw%=XqU zM%ZcKGxR9paK+_i`N!-Ht+UxvXI#V5_OfVYIvEz7|uOulG(tgtqrM~h#V7R|Vf zeb;Ix_oD&--Xn<7c4*@XtXM zspIn)n2K5x#i46j0IKciIW{B>3ATqW+tIL4Ha?ncAp`2MF-if7Q1GHz!h0_!X4A}? z781Vd(JjspLjx8Xk>!w*y3mJd#Og7IqWdh_9Xlp4w9;$F^o1uD&|_Id47M_-phN>& z>;yAJ;S*Fg1?~^yNsn`(^Sf~(1w%8dO86`AmRa2nPcoR^4kd~EOr2=TxTNvh$oM=OvQZflYdeTBsvH z_p8NQ7nt?VH<}kZ8exjZYUUu=U7HaqHi4`G=@nZaBR&TOBy#@+=T|iITJ9~mr#`ds zFU)&*hPDFI4eW{W&#lX^eZW@cY-WSneY3>2r#nqt7QR?26U-S*wz+K0-U5H~QAAcx z>qTQsMcO_OFyym z!{yS)?bE2%!U^^zaztGXirA0x9~MoK&7OcX=tx%|CrTJnVbQaTi4AF85{r#%_iK60 zI4X`LO+Mlig$)6d&SQE}&iu#Wa-(1OwF(blzoxQ(1^~J#-^3u4=fV=w(qz5eNu0EO zs7TL^@cYrHVyw2B^-Vl?C)BOcH0|yUAb?iCG4S4kuc=}Z`+Q}%i>4SxTzqq{P=>l= z{%hY2~4OpTYYPj)*I6}94UQ<5deC+WcmQ!%NDXE;;$%8jdmR8nW{5Td(TLiO(V zvLu5D_No_X7^2&wB|sbl+Cneut@}?ySBAC_kdWq&0L5p*)mtIiMdyYQU$####XU9QLs(P zi0!5I_cf2}2~n$_f)Xt}x)zpxHJP$HsAG!pOQ{G!^a=)64fgVtsge?y^kH$oej%2--lYO-sHv!U1ma-i#sxvLj(k%g^8DRQEF9?;pqM zY{mTjwcRIcEzzuWVoZinbGLo#n>i$@)xTPM@b|fa6#K<-6cl_}>$A%*ZNj02lj(|# zdGsq?dD3>jG>ZoZ_s6PY zB((xd>}I^?H&LgGYzYPRCo1q9QbUPIFf;K1?}Drb;MraG5ylNP!k&M9xAnT+AFtqX z>WwC1iF3O}ABohH(A31hG|ehFq0Tx?>k8mxdD>yCksY3l*$qBt2#7kw!VN>2$d+39FJFN6;W2tk z&cF{-xMdN~NFmk_68_lQFq=3BdL#=f7ZVxMB#Y-OU0-ZSbIH%yMF>$VgUsek(RMx~ zSphIR{f9|88Nyv0>u=}nqs|go-cY7g2D*e`XTZ5TJR`U;GenyOO67>4M#+SO> zTI*qu zC<1P&tvt0Cu{OF&Tj?h+AqjLD!ibrWMknGv3!&^_Of|}V9&|J;>J{XAuODl!LMQT_ zufzR+0mOlOpjmyJH?YC`%j5fTK;PkLua>_R-#IckKRq(hCg{O5D@An3$dEPA#WO^# zKuF%h$QxzW`_4N>MX>iwGa1qSBBT>bYGs~=o>2v+QTh_-M#;cPgv)ayi4XaWbp;ph z5k{L+hEwB5wtxwvVmX9mSM$1o`sQkYzFa=QYovkX)adpoL?9WzU78twQZl3RGl4I0 zwM7YzteTQ!oZf6W6f(DivmJl0re!kVVnwWER42=oUOd>o@AMSRN!L@j)xYc*a-x8< zVUvI>;i%NFHm7uh8dgG9_|uN6DG%>)02=ecYs$5@yDN5X1xP@q4C(U(yQT{7%z}a4 zGXv)79Y@ZqOik%02_bOP4gZD8hx%d9PlId=zgmrbg><($tu4Lqqo*s1_y#!l6rofO;Ra23=6V_Ww1&l4CA zva~DKoC<`@C~`Z3<}tuV1YcSuxp*E%tV!t(P$kR4<^@=zJR9A3BV869`l4)4A}fgO zYtI+8*%3C8jj6t%qiZ((4o$$mCzRfj@Io!bgiKNYqh-pEkClUQBQkM_?6mNg`(D^D zt)~FoNK5D6N|5E>tN|mq_}=eul!~9X?8R|}8gy*|sl=c96t}-p)3b7?<(i*aSpBJG zlg_iafStm6v7#LM)BeDtQKNsLy~TZ9o^jQ(W+7pD`NuhD3^|5h2mE~cZT16f6QIBv zJVdJpHvSXwL_0oNHpn9Oiguw#7*$Zfufeh8rB4$I?x{FaAz(Qwe-56Z3Bxagojnhu z)2JKoqwd=9;QY7yIOl4|aUvr#OS23Mx+Dt;lx}p^X11XcJw4yMk?;2F*+qq+&xa^$ z{M;J;uA>$G>*-sA+q#A$0SaohgA+nr@ro+*k1=$!?c3PcJ(qzScawhwj|a)0DP2(- z?OHWbxPqgkYwqQA=)m{#5-J>xSCpqo;pZ8=&x7or)z$@5%GRJQ4+jWjMIM6C5YsR&ljrxAiVe zmlZT-5-hT*G$uBz#}yBMqh(Jo0X(8|7;5n0FLKh`DwU~~LaAru_&`TR7al&}Bw=WG zGl*A}fsTp{owTgJ+3@C=7p?l=jRkbFlrP5m=xu3H+%e1W z8Wb*a)9}HcY87Z&Lp8_V);|2!&x6iX;91g^&5#w~p%Nb~5@}SSFjzpqUxja&Sl_wC z(D)yBz?O{h+#5P4buplNQ*R$Np}oo(L(kAP76^0=`+DG}qjUbyp88_uBMG73aPqJzrI;%QYj%bXgUs$kEf*$T6t(=*6u ztT#M7yD+)kS-jo&s*?PSuETKrmL>U}GAeX%-}I>n0LeB*doW9fBzZSq*$Iu># z#I8$3BxnOe4|se_R$IT1lXQEPAUYW)jT7 z8f#*TEH2i{bk3PEyS{hDk3lQK&E-8#<|$dmFWoOqmSN|n=d(EF?o2ne<(4Y`W4g2Q z5hm#`uE)%m52z|+D)wnzP#w@O&GX2Vm(9T-PcqY~KB&HT%@k*!b6ffRY`dgqJ7Uc) zAFs4nfctsp#E=ZXFfpKuood}{%P!O);`=e0onsv7gp+(L`u4{)H&Tkp1ED? zmMEXh?PB+uyzbACgKv3UAJO#*tG$k|cW`9*G+_3e4Ma%&kE_L3!1YaH;{iw7f(m)`9{6>;!!e(z`ndLzz zXT9s|)_hRqrVZbiYxjQ|YPhL~tO z$U34^iy*v9V`_X_ISixdE!rpGQiQj5?=ZK5h8mQhA zJ8u#Z>bZ~$=CxDR58>ANZz}5gZP-%oGcIf1X`pg%4_MOeRIs1yc7HOhQ$J$*(JaE) z1K@$#$Bq1LHsNb86B2v21KYvZDDAJ48ti83;XXK9rCCmOOEeTMrI3?C;cfhwT%x$P z`}Kvg;Yi#bJ^MW+y%H?a${zVtw}xM~)gcFa1EGgpw94C}#?@ix`M_=^4~TT9CCl*M zI#zh}qeYYG4PaE+zoaH)2qM8GjYlq$_)s-=C0X1Sww@)wt`s2e(fC^F%QQodB_h@a zX}7vEeQLB@bvlgtw1X0S)5U0f=bts?Z|^6BciU;-kk%?;Y*SZt26y_+qLXAW%fNW# z4WE(IQNp2q+ChYq2y^NQw_r+PeMF9T9O@wOV%zR$7*cadUz?%s7sg`xvV^4?U3zXZ zb>>Jg59deV)ZqPz|KY1AvbR^b9HqQ7iwrHVh{J;Cbuq&wuLPbt^jWsC2lhnqLsyOq z!0R(X_-vOC2d?Yye1i2EjJ*R}bGJY$S$mb>J<}ohNbcBGsr_and>~9wn3S%GXMogn(%ve3 z;LKsc%GJ)DhiTL67>!$?qp%LF19_3kwWP}%bo9LH9NgF#a+jWa1xft*mNCjXI-ky9 za@N^=x{DKfl7}$agvLr?IAr>^Z2^iI)7i5uh66S zOwf`%UsH$UCB2A0!HS{hI1dNHh7)7|Dvy-3*c1HRI>W{*pREQ`A|{JzVizUky`@n*k*YN=D(HuXs< z#z7ur)M&&7-9(zz+^uZAF*DA&Mq#~C%?Jj3m&tt}AD$ z&o@ns&gZUNSsY1@hqZ~Aw4lJHMEBjY`k{R8_8AO;@L1hJf>Gh@*DTlKxaNu@bXauCnk-Nzq zy1M6AHU7RJxujd*RavcHrPhc|Pdgx=L%BUCntKJ#jWc$`{Ve)UR;@xylw#r+u33I# zUF^-Hh{LRTp0>rcQNEm}>of9Ta`_i)JIOsDTeCTGM!>6)xO;G-a7~I0oy;TbVnNpG zBET(6vBq^~VPP;u(FZ5i6h)7Fj9&jTzrrI$gtJd$2~b>75mjBS*eZx$kQQI3YNET-uxW(ZV8KmUQE}()c0Kq;ualtT>vYTm zG7c@cjMF!~Kh^QQcJg_p%W-m{cOTx-Grx{n)OA-6f8LeB(y%X9#7!&uWZ>#H)2=Px zOj-HRd??`lgglMp`)2i#+%zN9!j>jseb>ShwH=g5)zb+@$CI=e7< zKd7qEWSzdF*na%7+rusB(ap%#<+%Md0-3c-x!*tk_&PEbEknbGgDkRGah2?VJ(AyW z`bw8L9CUIsD<5i0DqIZbe%Tw^or0KR!2!$?U ztxpjvns=l>Ml#KP_|?;n@B9jUw_xGBKEr9-!Uzf!_gd=Wrq;xlC~wkJ$@>JFkoL1k zu+;Hi2OAWR4uru=tEo|Pa(QigFD&DuuL_N!#!Hc}48|(1IqWPu!(gD3qj{ynVTp#D z9oda$7@IiTMf#g0={m+s?fjNUcn)vCf3I7(!_En~L0uv%x&<^ZwqS#IO$0Bcs{y#eO|W1f%EA$cAbstpDv7#WzMk?c_ipNcV)`}MKQ38-i_HL zOTatw*PL|_ah6nq1FjQO$O@f>aa~z_S$rmNh|{pY@)^h3^;Z-K385OE3;V2d za9{u;9k%=#W5zNptaqe3KtjthNnNZdH=BJ5;PT90G@f@Q_y#U^r-E?Pq%2-YeIY@k4TqfAJ%(J~>KB zu&);NtuI)Zu2T%M9KoCvakJIT+x8(Ttm0(AEKICT$geajOqnU3@y8EbzHpsHjRsSj zZR~006wE)JETZ}Z#2v5LCFQEplE7@DeF>|WRK-rsRjb7&*9b6NOzQ!STKIt}*vT&( z+yW=f(sQGacZ_ttM6=iJa8W1{y&_h2-s7M|4XGwIrJ!oZ6y8g?y&3tEeZ`LM)&|&_slG=l-X_{@KH$(i_UHZ5llumrn>~va*QBKbCWN(U$PskSB zF!f6k%b&iW&8(Ku)Yy~|;GcQ=x|?&VAnRA`KKjhqN?6WQn`%T_q+@5QA}?#^fmqH#Hg2G3S`LM;{Uix3itVVB;LNNV5Xw* z39C2>bq1UBR6ck!ClRMZHA2R|By84qd^BJNi{I{i!j_J>=g%g(luEirzh1TA(4AiF z)zZ?id+cph#wa4lZ`h;Xg@;ejA}QDYQ03QM_Wb zC^e9wg2LQ_LW;zTc8GR7_eSe(hc__^mopAFo(p_9_N=?$s@lWwzF=fYzaS$#DXMWr z>DC#XE{&HRPoxeyF(7>%<*~7&^10w;D>DkyWAehR{TYis@O)R%O2$*X>}y4Z`}3>p z6uKM)wsMpI1wCLp`_h=CZ!ynJUmFWyVrcp2WbF(LviVsxLiT#+*fem^h2iNJR~cNe zn93~-o1qd%2l}b4VjD9iEjc2I^EL6DJf8i>4upZWkDS1<@K54F;~MFKIN%;KOiap; zH3w_tr%aXOlM+%zha-(T4%p-Fx&=t|8%T~%WIaWP#y7Q%ong2gioRqrmIe1`U00Mw zI?1Pa;&)N=PAcrL`ecz}VVYgBWB1-mZZ?K74!8zv1k$Luia+Y`j(P+*m!R(A%}6<< z*tryIA-|+TCm9=Q zc@7C;sGXGMkU2Jr9vJ6VwgkwggZh60n`v#sI#!fg&ETjx1>fTQy z+jc4+o>RNs?L%!Gqxa*C6ED|Lzb%wWJKK;fpElH$8?!7M^YItHr}~H)7=lz-yFb@-ny2*Xmd~`5X_1-ADcln$+ZDs(A*l3?u#!Y z*|lx@&x_u|M$*`uHVMOA8G5~3UbB8EwOcFqy4zthVmxmtLcTaojRyiN0RhJOAq!ue z?vKefv+_>Gm>^)$CE=yP;vLy!32Q}v3y_J&>&|$pr+eF9i#P6lvlBBE=62?_Zp$uF z@gp9MUQ33LDCeDZ9RZG>5gDO?o1A9&$K(fXo+anuA_WKrZ~p*9w8Fi!i}R6!*3 zkEpO+T244!^ZZEM(u;z&t}Miqk>FmAfylP*Sa{oRS7sl83@tACLNiNSlQLn|y2a|YRgp4l-1 z&rV=q`fu57r7ECA+5AsahkYqAmlEXF7+$BYTZ9TE11=1Y-s2m5p6*9{$Pd%~TQxe- z-DYz~6REQMle6IDc3M=bqMv5%wx0K7%VaozG-c$3srKjC@eRv@xZfTQiBWHw=-EYf zoK1^-ct}lKTG6gH`WN(D?N8{*N;d%TZ55r z7arYv{U`vj@84Gra$vHu>MDGJfj=fLHN7-=kyA*0ofrnHZxOZx@I-KFK6S5hxgOaF zf>{+B0%8FC_kYh46yL85&|XdcDmvJ9&;`8bJdbUar)nde*4cab=*{YGy#n2)>WUF` zTArTVoA+|^u+qBRgG;YbnOdOn{SgHKLq~ozjZJ{M3p=!S{}e;FWQX^sCnckC*1FRi zbldfv=P~$Y#HynPh5%5*!_oAKHIoLYLXJORB(tFRWTGPa=q#F%3H5=8u{r0SoC|&^bh;qvzv7587=Iv}6l5MpHgRTaS)Z(BDTi75m36)<$r zw*D=BTD0BST_aS%p9!1D>SYp`emC)M5x$^lF1whPZ= zA0Awm#1`(ItQb{46xDu|uDPXWBF;X^rTV(W0{(X8(*1ElmXfXIII3^pkkiGdWhCk% zm2b=GHRoRHrTblMKcKs{4Rb@Ud#C$tts4!NrUTWz3k-w244A0GORl^r^QW7PM-`dm(B{6C|g1BNL2f~{^VYme(Ykw zU2s84PG={=6`mdBO4LxZD z6fj>C{k_k?yEKscjJ4zXH|HAh{kZM^aEMJWLHt`tOEh;5%+8q&dbK27v^h8;F7&EJ z^Z<5eNsNcHPi`~N#mEaVczkqNw90v&B4Ulw2vs1J)VU71I2x@szgtjX&oxp_0xVl zu?l+{&XyV+iG&q>rOcb!ys<%@$MgLLNNhl*$1k9-wzvL9o5k~Ys`p#J!;@kFI5mGh z#DJXWeZqj{ndn^sQ{+kdY32^~{pk|klmuo+Bjg|c`Q;c77m1YH1pBumhHmdkM(8m8 zIp!PY))5bt-?nPezK#f>Hy~vgW$QKv5pwGW=Y80fVBj9}s-?qOj2n8wzAR>Ev$b*R zH#!stF>H&*fTZ@zf1fzrxS|u*XFd5~hNlMh%Wgx?w{#s2opaAcJzfcNztLD9tkx}8 z!!5v_Q&YlIFH9GRq!=VWuYaN#3dULA{ilUrD2>N3-xRr*gs#Gisb zunj7o!*20M&p_^5%n#ceB?eW!Vgp6ZsQm;}y zbxML)!YVb0i@0|5cUEG;Iima<@S9&)+@j+U=cDK$_( zdb%!`udmQ4ME9zN|H9j<))Eo@8J%ZQXT8v?4oApge(E6uLt})isA+w=ZLKyH(-{J;%a|yEV;nr#p3Zq5`JyZ4;^NH=T;$ z6Z^!f*@Ol1oZ*>)*vECweYkN>BV$b5Y|(K8+85<5wFdvN{xKuEBDJ6NPbEwi zq_#+z(^i?*sU*NDO^`=$VDto@JnYn^UyV=fbYPkmKGGqo>f3rJxU$Q9p+Yj6qPegC z5H`b{s6fDOwrg%elN^xCX=#CwxBsa_JVT;<)j7 zPYw>)%7MHC@v*Hh4}lToj1WOqzr(?9f^@U=FRWQzBsiSt-^0f z7?MeMcb0|>p$yZ^bH-1xP*P{8SL`cJerXelO!ct)m6Cc#q*qmeXDic@C8#q)bEBKCf1d zUDk4XT+ZGXLNHrB;m3~!A2tzjgXnP?*#>NA{U7t7?iL|VnW96AExfpAC83#1aW}6h zT~w=Ou_jvJ)HP)ShNO<^%(Iw0r*DfNW&g#nFn84dInbMZRDy6RRqOM}r=l5$=e<3` z!novr<7G;=NM#3n*?rW~{dUd=&?tK*|5q#eSL(+ECda`4lTf(*`=8t$C~3a@;7zL2 z|9tgz|KCvj&⋘E`B}w2b=$W{vQNS3mZYdt6q3D+HbTeKhNs_J1_C4ZtijOTOXFG z|KaeeALaGqA|3gkPZGb9K7Wv&qq;pGeR?|^{`LR=;r`!%IFhs$MEGC1KRJAM`u{+P ac}K~vks8GPngR#$aY&0Rh*gUi2K@&KDhl}k literal 0 HcmV?d00001 diff --git a/doc/source/rllib-algorithms.rst b/doc/source/rllib-algorithms.rst index b2f804dd05d0..3d3a020c4f8f 100644 --- a/doc/source/rllib-algorithms.rst +++ b/doc/source/rllib-algorithms.rst @@ -16,7 +16,7 @@ Tuned examples: `PongNoFrameskip-v4 `__ `[implementation] `__ -RLlib's A3C uses the AsyncGradientsOptimizer to apply gradients computed remotely on policy evaluation actors. It scales to up to 16-32 worker processes, depending on the environment. Both a TensorFlow (LSTM), and PyTorch version are available. +RLlib's A3C uses the AsyncGradientsOptimizer to apply gradients computed remotely on policy evaluation actors. It scales to up to 16-32 worker processes, depending on the environment. Both a TensorFlow (LSTM), and PyTorch version are available. Note that if you have a GPU, `IMPALA <#importance-weighted-actor-learner-architecture>`__ probably will perform better than A3C. Tuned examples: `PongDeterministic-v4 `__, `PyTorch version `__ @@ -47,6 +47,20 @@ Tuned examples: `Humanoid-v1 `__ +`[implementation] `__ +In IMPALA, a central learner runs SGD in a tight loop while asynchronously pulling sample batches from many actor processes. RLlib's IMPALA implementation uses DeepMind's reference `V-trace code `__. Note that we do not provide a deep residual network out of the box, but one can be plugged in as a `custom model `__. + +Tuned examples: `PongNoFrameskip-v4 `__, `vectorized configuration `__ + +.. figure:: impala.png + :align: center + + RLlib's IMPALA implementation scales from 16 to 128 workers on PongNoFrameskip-v4. With vectorization, similar learning performance to 128 workers can be achieved with only 32 workers. This about an order of magnitude faster than A3C, with similar sample efficiency. + Policy Gradients ---------------- `[paper] `__ `[implementation] `__ We include a vanilla policy gradients implementation as an example algorithm. This is usually outperformed by PPO. @@ -64,4 +78,4 @@ Tuned examples: `Humanoid-v1 `__ (some of them are tuned to run on GPUs). If you find better settings or tune an algorithm on a different domain, consider submitting a Pull Request! +You can run these with the ``train.py`` script as follows: + +.. code-block:: bash + + python ray/python/ray/rllib/train.py -f /path/to/tuned/example.yaml + Python API ---------- diff --git a/doc/source/rllib.rst b/doc/source/rllib.rst index 5f9d94681650..7dbe039ab744 100644 --- a/doc/source/rllib.rst +++ b/doc/source/rllib.rst @@ -47,6 +47,7 @@ Algorithms * `Deep Deterministic Policy Gradients `__ * `Deep Q Networks `__ * `Evolution Strategies `__ +* `Importance Weighted Actor-Learner Architecture `__ * `Policy Gradients `__ * `Proximal Policy Optimization `__ diff --git a/python/ray/rllib/__init__.py b/python/ray/rllib/__init__.py index cf0f1058083f..4d6575fcf6bd 100644 --- a/python/ray/rllib/__init__.py +++ b/python/ray/rllib/__init__.py @@ -19,7 +19,7 @@ def _register_all(): for key in [ "PPO", "ES", "DQN", "APEX", "A3C", "BC", "PG", "DDPG", "APEX_DDPG", - "__fake", "__sigmoid_fake_data", "__parameter_tuning" + "IMPALA", "__fake", "__sigmoid_fake_data", "__parameter_tuning" ]: from ray.rllib.agents.agent import get_agent_class register_trainable(key, get_agent_class(key)) diff --git a/python/ray/rllib/agents/a3c/a3c_tf_policy_graph.py b/python/ray/rllib/agents/a3c/a3c_tf_policy_graph.py index 00f630d3b0e4..f368f1470ea7 100644 --- a/python/ray/rllib/agents/a3c/a3c_tf_policy_graph.py +++ b/python/ray/rllib/agents/a3c/a3c_tf_policy_graph.py @@ -1,3 +1,5 @@ +"""Note: Keep in sync with changes to VTracePolicyGraph.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -77,8 +79,6 @@ def __init__(self, observation_space, action_space, config): ("advantages", advantages), ("value_targets", v_target), ] - self.state_in = self.model.state_in - self.state_out = self.model.state_out TFPolicyGraph.__init__( self, observation_space, @@ -88,29 +88,21 @@ def __init__(self, observation_space, action_space, config): action_sampler=action_dist.sample(), loss=self.loss.total_loss, loss_inputs=loss_in, - state_inputs=self.state_in, - state_outputs=self.state_out, + state_inputs=self.model.state_in, + state_outputs=self.model.state_out, seq_lens=self.model.seq_lens, max_seq_len=self.config["model"]["max_seq_len"]) - if self.config.get("summarize"): - bs = tf.to_float(tf.shape(self.observations)[0]) - tf.summary.scalar("model/policy_graph", self.loss.pi_loss / bs) - tf.summary.scalar("model/value_loss", self.loss.vf_loss / bs) - tf.summary.scalar("model/entropy", self.loss.entropy / bs) - tf.summary.scalar("model/grad_gnorm", tf.global_norm(self._grads)) - tf.summary.scalar("model/var_gnorm", tf.global_norm(self.var_list)) - self.summary_op = tf.summary.merge_all() - self.sess.run(tf.global_variables_initializer()) def extra_compute_action_fetches(self): return {"vf_preds": self.vf} def value(self, ob, *args): - feed_dict = {self.observations: [ob]} - assert len(args) == len(self.state_in), (args, self.state_in) - for k, v in zip(self.state_in, args): + feed_dict = {self.observations: [ob], self.model.seq_lens: [1]} + assert len(args) == len(self.model.state_in), \ + (args, self.model.state_in) + for k, v in zip(self.model.state_in, args): feed_dict[k] = v vf = self.sess.run(self.vf, feed_dict) return vf[0] @@ -126,7 +118,15 @@ def gradients(self, optimizer): def extra_compute_grad_fetches(self): if self.config.get("summarize"): - return {"summary": self.summary_op} + return { + "stats": { + "policy_loss": self.loss.pi_loss, + "value_loss": self.loss.vf_loss, + "entropy": self.loss.entropy, + "grad_gnorm": tf.global_norm(self._grads), + "var_gnorm": tf.global_norm(self.var_list), + }, + } else: return {} @@ -139,7 +139,7 @@ def postprocess_trajectory(self, sample_batch, other_agent_batches=None): last_r = 0.0 else: next_state = [] - for i in range(len(self.state_in)): + for i in range(len(self.model.state_in)): next_state.append([sample_batch["state_out_{}".format(i)][-1]]) last_r = self.value(sample_batch["new_obs"][-1], *next_state) return compute_advantages(sample_batch, last_r, self.config["gamma"], diff --git a/python/ray/rllib/agents/agent.py b/python/ray/rllib/agents/agent.py index 314ea2b6dbb8..0ac52cc8b8c6 100644 --- a/python/ray/rllib/agents/agent.py +++ b/python/ray/rllib/agents/agent.py @@ -360,6 +360,9 @@ def get_agent_class(alg): elif alg == "PG": from ray.rllib.agents import pg return pg.PGAgent + elif alg == "IMPALA": + from ray.rllib.agents import impala + return impala.ImpalaAgent elif alg == "script": from ray.tune import script_runner return script_runner.ScriptRunner diff --git a/python/ray/rllib/agents/ddpg/apex.py b/python/ray/rllib/agents/ddpg/apex.py index 93c0a9e3d983..d0508463c712 100644 --- a/python/ray/rllib/agents/ddpg/apex.py +++ b/python/ray/rllib/agents/ddpg/apex.py @@ -9,7 +9,7 @@ APEX_DDPG_DEFAULT_CONFIG = merge_dicts( DDPG_CONFIG, { - "optimizer_class": "AsyncSamplesOptimizer", + "optimizer_class": "AsyncReplayOptimizer", "optimizer": merge_dicts( DDPG_CONFIG["optimizer"], { "max_weight_sync_delay": 400, diff --git a/python/ray/rllib/agents/dqn/apex.py b/python/ray/rllib/agents/dqn/apex.py index bbe18e174ac6..b120a0fbb75d 100644 --- a/python/ray/rllib/agents/dqn/apex.py +++ b/python/ray/rllib/agents/dqn/apex.py @@ -9,7 +9,7 @@ APEX_DEFAULT_CONFIG = merge_dicts( DQN_CONFIG, { - "optimizer_class": "AsyncSamplesOptimizer", + "optimizer_class": "AsyncReplayOptimizer", "optimizer": merge_dicts( DQN_CONFIG["optimizer"], { "max_weight_sync_delay": 400, diff --git a/python/ray/rllib/agents/impala/__init__.py b/python/ray/rllib/agents/impala/__init__.py new file mode 100644 index 000000000000..087a568c31ac --- /dev/null +++ b/python/ray/rllib/agents/impala/__init__.py @@ -0,0 +1,3 @@ +from ray.rllib.agents.impala.impala import ImpalaAgent, DEFAULT_CONFIG + +__all__ = ["ImpalaAgent", "DEFAULT_CONFIG"] diff --git a/python/ray/rllib/agents/impala/impala.py b/python/ray/rllib/agents/impala/impala.py new file mode 100644 index 000000000000..bfabeb940979 --- /dev/null +++ b/python/ray/rllib/agents/impala/impala.py @@ -0,0 +1,123 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import pickle +import os +import time + +import ray +from ray.rllib.agents.a3c.a3c_tf_policy_graph import A3CPolicyGraph +from ray.rllib.agents.impala.vtrace_policy_graph import VTracePolicyGraph +from ray.rllib.agents.agent import Agent, with_common_config +from ray.rllib.optimizers import AsyncSamplesOptimizer +from ray.rllib.utils import FilterManager +from ray.tune.trial import Resources + +OPTIMIZER_SHARED_CONFIGS = [ + "sample_batch_size", + "train_batch_size", +] + +DEFAULT_CONFIG = with_common_config({ + # V-trace params (see vtrace.py). + "vtrace": True, + "vtrace_clip_rho_threshold": 1.0, + "vtrace_clip_pg_rho_threshold": 1.0, + + # System params. + "sample_batch_size": 50, + "train_batch_size": 500, + "min_iter_time_s": 10, + "summarize": False, + "gpu": True, + "num_workers": 2, + "num_cpus_per_worker": 1, + "num_gpus_per_worker": 0, + + # Learning params. + "grad_clip": 40.0, + "lr": 0.0001, + "vf_loss_coeff": 0.5, + "entropy_coeff": -0.01, + + # Model and preprocessor options. + "clip_rewards": True, + "preprocessor_pref": "deepmind", + "model": { + "use_lstm": False, + "max_seq_len": 20, + "dim": 80, + }, +}) + + +class ImpalaAgent(Agent): + """IMPALA implementation using DeepMind's V-trace.""" + + _agent_name = "IMPALA" + _default_config = DEFAULT_CONFIG + + @classmethod + def default_resource_request(cls, config): + cf = dict(cls._default_config, **config) + return Resources( + cpu=1, + gpu=cf["gpu"] and 1 or 0, + extra_cpu=cf["num_cpus_per_worker"] * cf["num_workers"], + extra_gpu=cf["num_gpus_per_worker"] * cf["num_workers"]) + + def _init(self): + for k in OPTIMIZER_SHARED_CONFIGS: + if k not in self.config["optimizer"]: + self.config["optimizer"][k] = self.config[k] + if self.config["vtrace"]: + policy_cls = VTracePolicyGraph + else: + policy_cls = A3CPolicyGraph + self.local_evaluator = self.make_local_evaluator( + self.env_creator, policy_cls) + self.remote_evaluators = self.make_remote_evaluators( + self.env_creator, policy_cls, self.config["num_workers"], + {"num_cpus": 1}) + self.optimizer = AsyncSamplesOptimizer(self.local_evaluator, + self.remote_evaluators, + self.config["optimizer"]) + + def _train(self): + prev_steps = self.optimizer.num_steps_sampled + start = time.time() + self.optimizer.step() + while time.time() - start < self.config["min_iter_time_s"]: + self.optimizer.step() + FilterManager.synchronize(self.local_evaluator.filters, + self.remote_evaluators) + result = self.optimizer.collect_metrics() + result = result._replace( + timesteps_this_iter=self.optimizer.num_steps_sampled - prev_steps) + return result + + def _stop(self): + # workaround for https://github.com/ray-project/ray/issues/1516 + for ev in self.remote_evaluators: + ev.__ray_terminate__.remote() + + def _save(self, checkpoint_dir): + checkpoint_path = os.path.join(checkpoint_dir, + "checkpoint-{}".format(self.iteration)) + agent_state = ray.get( + [a.save.remote() for a in self.remote_evaluators]) + extra_data = { + "remote_state": agent_state, + "local_state": self.local_evaluator.save() + } + pickle.dump(extra_data, open(checkpoint_path + ".extra_data", "wb")) + return checkpoint_path + + def _restore(self, checkpoint_path): + extra_data = pickle.load(open(checkpoint_path + ".extra_data", "rb")) + ray.get([ + a.restore.remote(o) + for a, o in zip(self.remote_evaluators, extra_data["remote_state"]) + ]) + self.local_evaluator.restore(extra_data["local_state"]) diff --git a/python/ray/rllib/agents/impala/vtrace.py b/python/ray/rllib/agents/impala/vtrace.py new file mode 100644 index 000000000000..ac5abf0e6592 --- /dev/null +++ b/python/ray/rllib/agents/impala/vtrace.py @@ -0,0 +1,300 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Functions to compute V-trace off-policy actor critic targets. + +For details and theory see: + +"IMPALA: Scalable Distributed Deep-RL with +Importance Weighted Actor-Learner Architectures" +by Espeholt, Soyer, Munos et al. + +See https://arxiv.org/abs/1802.01561 for the full paper. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + +import tensorflow as tf + +nest = tf.contrib.framework.nest + +VTraceFromLogitsReturns = collections.namedtuple('VTraceFromLogitsReturns', [ + 'vs', 'pg_advantages', 'log_rhos', 'behaviour_action_log_probs', + 'target_action_log_probs' +]) + +VTraceReturns = collections.namedtuple('VTraceReturns', 'vs pg_advantages') + + +def log_probs_from_logits_and_actions(policy_logits, actions): + """Computes action log-probs from policy logits and actions. + + In the notation used throughout documentation and comments, T refers to the + time dimension ranging from 0 to T-1. B refers to the batch size and + NUM_ACTIONS refers to the number of actions. + + Args: + policy_logits: A float32 tensor of shape [T, B, NUM_ACTIONS] with + un-normalized log-probabilities parameterizing a softmax policy. + actions: An int32 tensor of shape [T, B] with actions. + + Returns: + A float32 tensor of shape [T, B] corresponding to the sampling log + probability of the chosen action w.r.t. the policy. + """ + policy_logits = tf.convert_to_tensor(policy_logits, dtype=tf.float32) + actions = tf.convert_to_tensor(actions, dtype=tf.int32) + + policy_logits.shape.assert_has_rank(3) + actions.shape.assert_has_rank(2) + + return -tf.nn.sparse_softmax_cross_entropy_with_logits( + logits=policy_logits, labels=actions) + + +def from_logits(behaviour_policy_logits, + target_policy_logits, + actions, + discounts, + rewards, + values, + bootstrap_value, + clip_rho_threshold=1.0, + clip_pg_rho_threshold=1.0, + name='vtrace_from_logits'): + r"""V-trace for softmax policies. + + Calculates V-trace actor critic targets for softmax polices as described in + + "IMPALA: Scalable Distributed Deep-RL with + Importance Weighted Actor-Learner Architectures" + by Espeholt, Soyer, Munos et al. + + Target policy refers to the policy we are interested in improving and + behaviour policy refers to the policy that generated the given + rewards and actions. + + In the notation used throughout documentation and comments, T refers to the + time dimension ranging from 0 to T-1. B refers to the batch size and + NUM_ACTIONS refers to the number of actions. + + Args: + behaviour_policy_logits: A float32 tensor of shape [T, B, NUM_ACTIONS] with + un-normalized log-probabilities parametrizing the softmax behaviour + policy. + target_policy_logits: A float32 tensor of shape [T, B, NUM_ACTIONS] with + un-normalized log-probabilities parametrizing the softmax target policy. + actions: An int32 tensor of shape [T, B] of actions sampled from the + behaviour policy. + discounts: A float32 tensor of shape [T, B] with the discount encountered + when following the behaviour policy. + rewards: A float32 tensor of shape [T, B] with the rewards generated by + following the behaviour policy. + values: A float32 tensor of shape [T, B] with the value function estimates + wrt. the target policy. + bootstrap_value: A float32 of shape [B] with the value function estimate at + time T. + clip_rho_threshold: A scalar float32 tensor with the clipping threshold for + importance weights (rho) when calculating the baseline targets (vs). + rho^bar in the paper. + clip_pg_rho_threshold: A scalar float32 tensor with the clipping threshold + on rho_s in \rho_s \delta log \pi(a|x) (r + \gamma v_{s+1} - V(x_s)). + name: The name scope that all V-trace operations will be created in. + + Returns: + A `VTraceFromLogitsReturns` namedtuple with the following fields: + vs: A float32 tensor of shape [T, B]. Can be used as target to train a + baseline (V(x_t) - vs_t)^2. + pg_advantages: A float 32 tensor of shape [T, B]. Can be used as an + estimate of the advantage in the calculation of policy gradients. + log_rhos: A float32 tensor of shape [T, B] containing the log importance + sampling weights (log rhos). + behaviour_action_log_probs: A float32 tensor of shape [T, B] containing + behaviour policy action log probabilities (log \mu(a_t)). + target_action_log_probs: A float32 tensor of shape [T, B] containing + target policy action probabilities (log \pi(a_t)). + """ + behaviour_policy_logits = tf.convert_to_tensor( + behaviour_policy_logits, dtype=tf.float32) + target_policy_logits = tf.convert_to_tensor( + target_policy_logits, dtype=tf.float32) + actions = tf.convert_to_tensor(actions, dtype=tf.int32) + + # Make sure tensor ranks are as expected. + # The rest will be checked by from_action_log_probs. + behaviour_policy_logits.shape.assert_has_rank(3) + target_policy_logits.shape.assert_has_rank(3) + actions.shape.assert_has_rank(2) + + with tf.name_scope( + name, + values=[ + behaviour_policy_logits, target_policy_logits, actions, + discounts, rewards, values, bootstrap_value + ]): + target_action_log_probs = log_probs_from_logits_and_actions( + target_policy_logits, actions) + behaviour_action_log_probs = log_probs_from_logits_and_actions( + behaviour_policy_logits, actions) + log_rhos = target_action_log_probs - behaviour_action_log_probs + vtrace_returns = from_importance_weights( + log_rhos=log_rhos, + discounts=discounts, + rewards=rewards, + values=values, + bootstrap_value=bootstrap_value, + clip_rho_threshold=clip_rho_threshold, + clip_pg_rho_threshold=clip_pg_rho_threshold) + return VTraceFromLogitsReturns( + log_rhos=log_rhos, + behaviour_action_log_probs=behaviour_action_log_probs, + target_action_log_probs=target_action_log_probs, + **vtrace_returns._asdict()) + + +def from_importance_weights(log_rhos, + discounts, + rewards, + values, + bootstrap_value, + clip_rho_threshold=1.0, + clip_pg_rho_threshold=1.0, + name='vtrace_from_importance_weights'): + r"""V-trace from log importance weights. + + Calculates V-trace actor critic targets as described in + + "IMPALA: Scalable Distributed Deep-RL with + Importance Weighted Actor-Learner Architectures" + by Espeholt, Soyer, Munos et al. + + In the notation used throughout documentation and comments, T refers to the + time dimension ranging from 0 to T-1. B refers to the batch size and + NUM_ACTIONS refers to the number of actions. This code also supports the + case where all tensors have the same number of additional dimensions, e.g., + `rewards` is [T, B, C], `values` is [T, B, C], `bootstrap_value` is [B, C]. + + Args: + log_rhos: A float32 tensor of shape [T, B, NUM_ACTIONS] representing the + log importance sampling weights, i.e. + log(target_policy(a) / behaviour_policy(a)). V-trace performs operations + on rhos in log-space for numerical stability. + discounts: A float32 tensor of shape [T, B] with discounts encountered when + following the behaviour policy. + rewards: A float32 tensor of shape [T, B] containing rewards generated by + following the behaviour policy. + values: A float32 tensor of shape [T, B] with the value function estimates + wrt. the target policy. + bootstrap_value: A float32 of shape [B] with the value function estimate at + time T. + clip_rho_threshold: A scalar float32 tensor with the clipping threshold for + importance weights (rho) when calculating the baseline targets (vs). + rho^bar in the paper. If None, no clipping is applied. + clip_pg_rho_threshold: A scalar float32 tensor with the clipping threshold + on rho_s in \rho_s \delta log \pi(a|x) (r + \gamma v_{s+1} - V(x_s)). If + None, no clipping is applied. + name: The name scope that all V-trace operations will be created in. + + Returns: + A VTraceReturns namedtuple (vs, pg_advantages) where: + vs: A float32 tensor of shape [T, B]. Can be used as target to + train a baseline (V(x_t) - vs_t)^2. + pg_advantages: A float32 tensor of shape [T, B]. Can be used as the + advantage in the calculation of policy gradients. + """ + log_rhos = tf.convert_to_tensor(log_rhos, dtype=tf.float32) + discounts = tf.convert_to_tensor(discounts, dtype=tf.float32) + rewards = tf.convert_to_tensor(rewards, dtype=tf.float32) + values = tf.convert_to_tensor(values, dtype=tf.float32) + bootstrap_value = tf.convert_to_tensor(bootstrap_value, dtype=tf.float32) + if clip_rho_threshold is not None: + clip_rho_threshold = tf.convert_to_tensor( + clip_rho_threshold, dtype=tf.float32) + if clip_pg_rho_threshold is not None: + clip_pg_rho_threshold = tf.convert_to_tensor( + clip_pg_rho_threshold, dtype=tf.float32) + + # Make sure tensor ranks are consistent. + rho_rank = log_rhos.shape.ndims # Usually 2. + values.shape.assert_has_rank(rho_rank) + bootstrap_value.shape.assert_has_rank(rho_rank - 1) + discounts.shape.assert_has_rank(rho_rank) + rewards.shape.assert_has_rank(rho_rank) + if clip_rho_threshold is not None: + clip_rho_threshold.shape.assert_has_rank(0) + if clip_pg_rho_threshold is not None: + clip_pg_rho_threshold.shape.assert_has_rank(0) + + with tf.name_scope( + name, + values=[log_rhos, discounts, rewards, values, bootstrap_value]): + rhos = tf.exp(log_rhos) + if clip_rho_threshold is not None: + clipped_rhos = tf.minimum( + clip_rho_threshold, rhos, name='clipped_rhos') + else: + clipped_rhos = rhos + + cs = tf.minimum(1.0, rhos, name='cs') + # Append bootstrapped value to get [v1, ..., v_t+1] + values_t_plus_1 = tf.concat( + [values[1:], tf.expand_dims(bootstrap_value, 0)], axis=0) + deltas = clipped_rhos * ( + rewards + discounts * values_t_plus_1 - values) + + # All sequences are reversed, computation starts from the back. + sequences = ( + tf.reverse(discounts, axis=[0]), + tf.reverse(cs, axis=[0]), + tf.reverse(deltas, axis=[0]), + ) + + # V-trace vs are calculated through a scan from the back to the + # beginning of the given trajectory. + def scanfunc(acc, sequence_item): + discount_t, c_t, delta_t = sequence_item + return delta_t + discount_t * c_t * acc + + initial_values = tf.zeros_like(bootstrap_value) + vs_minus_v_xs = tf.scan( + fn=scanfunc, + elems=sequences, + initializer=initial_values, + parallel_iterations=1, + back_prop=False, + name='scan') + # Reverse the results back to original order. + vs_minus_v_xs = tf.reverse(vs_minus_v_xs, [0], name='vs_minus_v_xs') + + # Add V(x_s) to get v_s. + vs = tf.add(vs_minus_v_xs, values, name='vs') + + # Advantage for policy gradient. + vs_t_plus_1 = tf.concat( + [vs[1:], tf.expand_dims(bootstrap_value, 0)], axis=0) + if clip_pg_rho_threshold is not None: + clipped_pg_rhos = tf.minimum( + clip_pg_rho_threshold, rhos, name='clipped_pg_rhos') + else: + clipped_pg_rhos = rhos + pg_advantages = ( + clipped_pg_rhos * (rewards + discounts * vs_t_plus_1 - values)) + + # Make sure no gradients backpropagated through the returned values. + return VTraceReturns( + vs=tf.stop_gradient(vs), + pg_advantages=tf.stop_gradient(pg_advantages)) diff --git a/python/ray/rllib/agents/impala/vtrace_policy_graph.py b/python/ray/rllib/agents/impala/vtrace_policy_graph.py new file mode 100644 index 000000000000..0b9c46c9a842 --- /dev/null +++ b/python/ray/rllib/agents/impala/vtrace_policy_graph.py @@ -0,0 +1,217 @@ +"""Adapted from A3CPolicyGraph to add V-trace. + +Keep in sync with changes to A3CPolicyGraph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +import gym + +import ray +from ray.rllib.agents.impala import vtrace +from ray.rllib.evaluation.tf_policy_graph import TFPolicyGraph +from ray.rllib.models.catalog import ModelCatalog +from ray.rllib.models.misc import linear, normc_initializer +from ray.rllib.utils.error import UnsupportedSpaceException + + +class VTraceLoss(object): + def __init__(self, + actions, + actions_logp, + actions_entropy, + dones, + behaviour_logits, + target_logits, + discount, + rewards, + values, + bootstrap_value, + vf_loss_coeff=0.5, + entropy_coeff=-0.01, + clip_rho_threshold=1.0, + clip_pg_rho_threshold=1.0): + """Policy gradient loss with vtrace importance weighting. + + VTraceLoss takes tensors of shape [T, B, ...], where `B` is the + batch_size. The reason we need to know `B` is for V-trace to properly + handle episode cut boundaries. + + Args: + actions: An int32 tensor of shape [T, B, NUM_ACTIONS]. + actions_logp: A float32 tensor of shape [T, B]. + actions_entropy: A float32 tensor of shape [T, B]. + dones: A bool tensor of shape [T, B]. + behaviour_logits: A float32 tensor of shape [T, B, NUM_ACTIONS]. + target_logits: A float32 tensor of shape [T, B, NUM_ACTIONS]. + discount: A float32 scalar. + rewards: A float32 tensor of shape [T, B]. + values: A float32 tensor of shape [T, B]. + bootstrap_value: A float32 tensor of shape [B]. + """ + + # Compute vtrace on the CPU for better perf. + with tf.device("/cpu:0"): + vtrace_returns = vtrace.from_logits( + behaviour_policy_logits=behaviour_logits, + target_policy_logits=target_logits, + actions=tf.cast(actions, tf.int32), + discounts=tf.to_float(~dones) * discount, + rewards=rewards, + values=values, + bootstrap_value=bootstrap_value, + clip_rho_threshold=tf.cast(clip_rho_threshold, tf.float32), + clip_pg_rho_threshold=tf.cast(clip_pg_rho_threshold, + tf.float32)) + + # The policy gradients loss + self.pi_loss = -tf.reduce_sum( + actions_logp * vtrace_returns.pg_advantages) + + # The baseline loss + delta = values - vtrace_returns.vs + self.vf_loss = 0.5 * tf.reduce_sum(tf.square(delta)) + + # The entropy loss + self.entropy = tf.reduce_sum(actions_entropy) + + # The summed weighted loss + self.total_loss = (self.pi_loss + self.vf_loss * vf_loss_coeff + + self.entropy * entropy_coeff) + + +class VTracePolicyGraph(TFPolicyGraph): + def __init__(self, observation_space, action_space, config): + config = dict(ray.rllib.agents.a3c.a3c.DEFAULT_CONFIG, **config) + assert config["batch_mode"] == "truncate_episodes", \ + "Must use `truncate_episodes` batch mode with V-trace." + self.config = config + self.sess = tf.get_default_session() + + # Setup the policy + self.observations = tf.placeholder( + tf.float32, [None] + list(observation_space.shape)) + dist_class, logit_dim = ModelCatalog.get_action_dist( + action_space, self.config["model"]) + self.model = ModelCatalog.get_model(self.observations, logit_dim, + self.config["model"]) + action_dist = dist_class(self.model.outputs) + values = tf.reshape( + linear(self.model.last_layer, 1, "value", normc_initializer(1.0)), + [-1]) + self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, + tf.get_variable_scope().name) + + # Setup the policy loss + if isinstance(action_space, gym.spaces.Box): + ac_size = action_space.shape[0] + actions = tf.placeholder(tf.float32, [None, ac_size], name="ac") + elif isinstance(action_space, gym.spaces.Discrete): + ac_size = action_space.n + actions = tf.placeholder(tf.int64, [None], name="ac") + else: + raise UnsupportedSpaceException( + "Action space {} is not supported for IMPALA.".format( + action_space)) + dones = tf.placeholder(tf.bool, [None], name="dones") + rewards = tf.placeholder(tf.float32, [None], name="rewards") + behaviour_logits = tf.placeholder( + tf.float32, [None, ac_size], name="behaviour_logits") + + def to_batches(tensor): + if self.config["model"]["use_lstm"]: + B = tf.shape(self.model.seq_lens)[0] + T = tf.shape(tensor)[0] // B + else: + # Important: chop the tensor into batches at known episode cut + # boundaries. TODO(ekl) this is kind of a hack + T = (self.config["sample_batch_size"] // + self.config["num_envs_per_worker"]) + B = tf.shape(tensor)[0] // T + rs = tf.reshape(tensor, + tf.concat([[B, T], tf.shape(tensor)[1:]], axis=0)) + # swap B and T axes + return tf.transpose( + rs, + [1, 0] + list(range(2, 1 + int(tf.shape(tensor).shape[0])))) + + if self.config["clip_rewards"]: + clipped_rewards = tf.clip_by_value(rewards, -1, 1) + else: + clipped_rewards = rewards + + # Inputs are reshaped from [B * T] => [T - 1, B] for V-trace calc. + self.loss = VTraceLoss( + actions=to_batches(actions)[:-1], + actions_logp=to_batches(action_dist.logp(actions))[:-1], + actions_entropy=to_batches(action_dist.entropy())[:-1], + dones=to_batches(dones)[:-1], + behaviour_logits=to_batches(behaviour_logits)[:-1], + target_logits=to_batches(self.model.outputs)[:-1], + discount=config["gamma"], + rewards=to_batches(clipped_rewards)[:-1], + values=to_batches(values)[:-1], + bootstrap_value=to_batches(values)[-1], + vf_loss_coeff=self.config["vf_loss_coeff"], + entropy_coeff=self.config["entropy_coeff"], + clip_rho_threshold=self.config["vtrace_clip_rho_threshold"], + clip_pg_rho_threshold=self.config["vtrace_clip_pg_rho_threshold"]) + + # Initialize TFPolicyGraph + loss_in = [ + ("actions", actions), + ("dones", dones), + ("behaviour_logits", behaviour_logits), + ("rewards", rewards), + ("obs", self.observations), + ] + TFPolicyGraph.__init__( + self, + observation_space, + action_space, + self.sess, + obs_input=self.observations, + action_sampler=action_dist.sample(), + loss=self.loss.total_loss, + loss_inputs=loss_in, + state_inputs=self.model.state_in, + state_outputs=self.model.state_out, + seq_lens=self.model.seq_lens, + max_seq_len=self.config["model"]["max_seq_len"]) + + self.sess.run(tf.global_variables_initializer()) + + def optimizer(self): + return tf.train.AdamOptimizer(self.config["lr"]) + + def gradients(self, optimizer): + grads = tf.gradients(self.loss.total_loss, self.var_list) + self.grads, _ = tf.clip_by_global_norm(grads, self.config["grad_clip"]) + clipped_grads = list(zip(self.grads, self.var_list)) + return clipped_grads + + def extra_compute_action_fetches(self): + return {"behaviour_logits": self.model.outputs} + + def extra_compute_grad_fetches(self): + if self.config.get("summarize"): + return { + "stats": { + "policy_loss": self.loss.pi_loss, + "value_loss": self.loss.vf_loss, + "entropy": self.loss.entropy, + "grad_gnorm": tf.global_norm(self._grads), + "var_gnorm": tf.global_norm(self.var_list), + }, + } + else: + return {} + + def postprocess_trajectory(self, sample_batch, other_agent_batches=None): + del sample_batch.data["new_obs"] # not used, so save some bandwidth + return sample_batch + + def get_initial_state(self): + return self.model.state_init diff --git a/python/ray/rllib/optimizers/__init__.py b/python/ray/rllib/optimizers/__init__.py index eadb38620de6..f7ede66f7287 100644 --- a/python/ray/rllib/optimizers/__init__.py +++ b/python/ray/rllib/optimizers/__init__.py @@ -1,4 +1,5 @@ from ray.rllib.optimizers.policy_optimizer import PolicyOptimizer +from ray.rllib.optimizers.async_replay_optimizer import AsyncReplayOptimizer from ray.rllib.optimizers.async_samples_optimizer import AsyncSamplesOptimizer from ray.rllib.optimizers.async_gradients_optimizer import \ AsyncGradientsOptimizer @@ -7,6 +8,7 @@ from ray.rllib.optimizers.multi_gpu_optimizer import LocalMultiGPUOptimizer __all__ = [ - "PolicyOptimizer", "AsyncSamplesOptimizer", "AsyncGradientsOptimizer", - "SyncSamplesOptimizer", "SyncReplayOptimizer", "LocalMultiGPUOptimizer" + "PolicyOptimizer", "AsyncReplayOptimizer", "AsyncSamplesOptimizer", + "AsyncGradientsOptimizer", "SyncSamplesOptimizer", "SyncReplayOptimizer", + "LocalMultiGPUOptimizer" ] diff --git a/python/ray/rllib/optimizers/async_gradients_optimizer.py b/python/ray/rllib/optimizers/async_gradients_optimizer.py index 397fabba9ba9..fc7fdb2488a3 100644 --- a/python/ray/rllib/optimizers/async_gradients_optimizer.py +++ b/python/ray/rllib/optimizers/async_gradients_optimizer.py @@ -20,6 +20,7 @@ def _init(self, grads_per_step=100): self.wait_timer = TimerStat() self.dispatch_timer = TimerStat() self.grads_per_step = grads_per_step + self.learner_stats = {} if not self.remote_evaluators: raise ValueError( "Async optimizer requires at least 1 remote evaluator") @@ -41,6 +42,8 @@ def step(self): with self.wait_timer: fut, e = gradient_queue.pop(0) gradient, info = ray.get(fut) + if "stats" in info: + self.learner_stats = info["stats"] if gradient is not None: with self.apply_timer: @@ -61,4 +64,5 @@ def stats(self): "wait_time_ms": round(1000 * self.wait_timer.mean, 3), "apply_time_ms": round(1000 * self.apply_timer.mean, 3), "dispatch_time_ms": round(1000 * self.dispatch_timer.mean, 3), + "learner": self.learner_stats, }) diff --git a/python/ray/rllib/optimizers/async_replay_optimizer.py b/python/ray/rllib/optimizers/async_replay_optimizer.py new file mode 100644 index 000000000000..0037ea7a0738 --- /dev/null +++ b/python/ray/rllib/optimizers/async_replay_optimizer.py @@ -0,0 +1,295 @@ +"""Implements Distributed Prioritized Experience Replay. + +https://arxiv.org/abs/1803.00933""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import random +import time +import threading + +import numpy as np +from six.moves import queue + +import ray +from ray.rllib.optimizers.policy_optimizer import PolicyOptimizer +from ray.rllib.optimizers.replay_buffer import PrioritizedReplayBuffer +from ray.rllib.evaluation.sample_batch import SampleBatch +from ray.rllib.utils.actors import TaskPool, create_colocated +from ray.rllib.utils.timer import TimerStat +from ray.rllib.utils.window_stat import WindowStat + +SAMPLE_QUEUE_DEPTH = 2 +REPLAY_QUEUE_DEPTH = 4 +LEARNER_QUEUE_MAX_SIZE = 16 + + +@ray.remote +class ReplayActor(object): + """A replay buffer shard. + + Ray actors are single-threaded, so for scalability multiple replay actors + may be created to increase parallelism.""" + + def __init__(self, num_shards, learning_starts, buffer_size, + train_batch_size, prioritized_replay_alpha, + prioritized_replay_beta, prioritized_replay_eps, + clip_rewards): + self.replay_starts = learning_starts // num_shards + self.buffer_size = buffer_size // num_shards + self.train_batch_size = train_batch_size + self.prioritized_replay_beta = prioritized_replay_beta + self.prioritized_replay_eps = prioritized_replay_eps + + self.replay_buffer = PrioritizedReplayBuffer( + self.buffer_size, + alpha=prioritized_replay_alpha, + clip_rewards=clip_rewards) + + # Metrics + self.add_batch_timer = TimerStat() + self.replay_timer = TimerStat() + self.update_priorities_timer = TimerStat() + + def get_host(self): + return os.uname()[1] + + def add_batch(self, batch): + PolicyOptimizer._check_not_multiagent(batch) + with self.add_batch_timer: + for row in batch.rows(): + self.replay_buffer.add(row["obs"], row["actions"], + row["rewards"], row["new_obs"], + row["dones"], row["weights"]) + + def replay(self): + with self.replay_timer: + if len(self.replay_buffer) < self.replay_starts: + return None + + (obses_t, actions, rewards, obses_tp1, dones, weights, + batch_indexes) = self.replay_buffer.sample( + self.train_batch_size, beta=self.prioritized_replay_beta) + + batch = SampleBatch({ + "obs": obses_t, + "actions": actions, + "rewards": rewards, + "new_obs": obses_tp1, + "dones": dones, + "weights": weights, + "batch_indexes": batch_indexes + }) + return batch + + def update_priorities(self, batch_indexes, td_errors): + with self.update_priorities_timer: + new_priorities = (np.abs(td_errors) + self.prioritized_replay_eps) + self.replay_buffer.update_priorities(batch_indexes, new_priorities) + + def stats(self): + stat = { + "add_batch_time_ms": round(1000 * self.add_batch_timer.mean, 3), + "replay_time_ms": round(1000 * self.replay_timer.mean, 3), + "update_priorities_time_ms": round( + 1000 * self.update_priorities_timer.mean, 3), + } + stat.update(self.replay_buffer.stats()) + return stat + + +class LearnerThread(threading.Thread): + """Background thread that updates the local model from replay data. + + The learner thread communicates with the main thread through Queues. This + is needed since Ray operations can only be run on the main thread. In + addition, moving heavyweight gradient ops session runs off the main thread + improves overall throughput. + """ + + def __init__(self, local_evaluator): + threading.Thread.__init__(self) + self.learner_queue_size = WindowStat("size", 50) + self.local_evaluator = local_evaluator + self.inqueue = queue.Queue(maxsize=LEARNER_QUEUE_MAX_SIZE) + self.outqueue = queue.Queue() + self.queue_timer = TimerStat() + self.grad_timer = TimerStat() + self.daemon = True + self.weights_updated = False + + def run(self): + while True: + self.step() + + def step(self): + with self.queue_timer: + ra, replay = self.inqueue.get() + if replay is not None: + with self.grad_timer: + td_error = self.local_evaluator.compute_apply(replay)[ + "td_error"] + self.outqueue.put((ra, replay, td_error, replay.count)) + self.learner_queue_size.push(self.inqueue.qsize()) + self.weights_updated = True + + +class AsyncReplayOptimizer(PolicyOptimizer): + """Main event loop of the Ape-X optimizer (async sampling with replay). + + This class coordinates the data transfers between the learner thread, + remote evaluators (Ape-X actors), and replay buffer actors. + + This optimizer requires that policy evaluators return an additional + "td_error" array in the info return of compute_gradients(). This error + term will be used for sample prioritization.""" + + def _init(self, + learning_starts=1000, + buffer_size=10000, + prioritized_replay=True, + prioritized_replay_alpha=0.6, + prioritized_replay_beta=0.4, + prioritized_replay_eps=1e-6, + train_batch_size=512, + sample_batch_size=50, + num_replay_buffer_shards=1, + max_weight_sync_delay=400, + clip_rewards=True, + debug=False): + + self.debug = debug + self.replay_starts = learning_starts + self.prioritized_replay_beta = prioritized_replay_beta + self.prioritized_replay_eps = prioritized_replay_eps + self.max_weight_sync_delay = max_weight_sync_delay + + self.learner = LearnerThread(self.local_evaluator) + self.learner.start() + + self.replay_actors = create_colocated(ReplayActor, [ + num_replay_buffer_shards, learning_starts, buffer_size, + train_batch_size, prioritized_replay_alpha, + prioritized_replay_beta, prioritized_replay_eps, clip_rewards + ], num_replay_buffer_shards) + assert len(self.remote_evaluators) > 0 + + # Stats + self.timers = { + k: TimerStat() + for k in [ + "put_weights", "get_samples", "enqueue", "sample_processing", + "replay_processing", "update_priorities", "train", "sample" + ] + } + self.num_weight_syncs = 0 + self.learning_started = False + + # Number of worker steps since the last weight update + self.steps_since_update = {} + + # Otherwise kick of replay tasks for local gradient updates + self.replay_tasks = TaskPool() + for ra in self.replay_actors: + for _ in range(REPLAY_QUEUE_DEPTH): + self.replay_tasks.add(ra, ra.replay.remote()) + + # Kick off async background sampling + self.sample_tasks = TaskPool() + weights = self.local_evaluator.get_weights() + for ev in self.remote_evaluators: + ev.set_weights.remote(weights) + self.steps_since_update[ev] = 0 + for _ in range(SAMPLE_QUEUE_DEPTH): + self.sample_tasks.add(ev, ev.sample_with_count.remote()) + + def step(self): + start = time.time() + sample_timesteps, train_timesteps = self._step() + time_delta = time.time() - start + self.timers["sample"].push(time_delta) + self.timers["sample"].push_units_processed(sample_timesteps) + if train_timesteps > 0: + self.learning_started = True + if self.learning_started: + self.timers["train"].push(time_delta) + self.timers["train"].push_units_processed(train_timesteps) + self.num_steps_sampled += sample_timesteps + self.num_steps_trained += train_timesteps + + def _step(self): + sample_timesteps, train_timesteps = 0, 0 + weights = None + + with self.timers["sample_processing"]: + completed = list(self.sample_tasks.completed()) + counts = ray.get([c[1][1] for c in completed]) + for i, (ev, (sample_batch, count)) in enumerate(completed): + sample_timesteps += counts[i] + + # Send the data to the replay buffer + random.choice( + self.replay_actors).add_batch.remote(sample_batch) + + # Update weights if needed + self.steps_since_update[ev] += counts[i] + if self.steps_since_update[ev] >= self.max_weight_sync_delay: + # Note that it's important to pull new weights once + # updated to avoid excessive correlation between actors + if weights is None or self.learner.weights_updated: + self.learner.weights_updated = False + with self.timers["put_weights"]: + weights = ray.put( + self.local_evaluator.get_weights()) + ev.set_weights.remote(weights) + self.num_weight_syncs += 1 + self.steps_since_update[ev] = 0 + + # Kick off another sample request + self.sample_tasks.add(ev, ev.sample_with_count.remote()) + + with self.timers["replay_processing"]: + for ra, replay in self.replay_tasks.completed(): + self.replay_tasks.add(ra, ra.replay.remote()) + with self.timers["get_samples"]: + samples = ray.get(replay) + with self.timers["enqueue"]: + self.learner.inqueue.put((ra, samples)) + + with self.timers["update_priorities"]: + while not self.learner.outqueue.empty(): + ra, replay, td_error, count = self.learner.outqueue.get() + ra.update_priorities.remote(replay["batch_indexes"], td_error) + train_timesteps += count + + return sample_timesteps, train_timesteps + + def stats(self): + replay_stats = ray.get(self.replay_actors[0].stats.remote()) + timing = { + "{}_time_ms".format(k): round(1000 * self.timers[k].mean, 3) + for k in self.timers + } + timing["learner_grad_time_ms"] = round( + 1000 * self.learner.grad_timer.mean, 3) + timing["learner_dequeue_time_ms"] = round( + 1000 * self.learner.queue_timer.mean, 3) + stats = { + "sample_throughput": round(self.timers["sample"].mean_throughput, + 3), + "train_throughput": round(self.timers["train"].mean_throughput, 3), + "num_weight_syncs": self.num_weight_syncs, + } + debug_stats = { + "replay_shard_0": replay_stats, + "timing_breakdown": timing, + "pending_sample_tasks": self.sample_tasks.count, + "pending_replay_tasks": self.replay_tasks.count, + "learner_queue": self.learner.learner_queue_size.stats(), + } + if self.debug: + stats.update(debug_stats) + return dict(PolicyOptimizer.stats(self), **stats) diff --git a/python/ray/rllib/optimizers/async_samples_optimizer.py b/python/ray/rllib/optimizers/async_samples_optimizer.py index ebc8676cd073..3b6bb861b482 100644 --- a/python/ray/rllib/optimizers/async_samples_optimizer.py +++ b/python/ray/rllib/optimizers/async_samples_optimizer.py @@ -1,108 +1,28 @@ -"""Implements Distributed Prioritized Experience Replay. +"""Implements the IMPALA architecture. -https://arxiv.org/abs/1803.00933""" +https://arxiv.org/abs/1802.01561""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os -import random import time import threading -import numpy as np from six.moves import queue import ray from ray.rllib.optimizers.policy_optimizer import PolicyOptimizer -from ray.rllib.optimizers.replay_buffer import PrioritizedReplayBuffer -from ray.rllib.evaluation.sample_batch import SampleBatch -from ray.rllib.utils.actors import TaskPool, create_colocated +from ray.rllib.utils.actors import TaskPool from ray.rllib.utils.timer import TimerStat from ray.rllib.utils.window_stat import WindowStat SAMPLE_QUEUE_DEPTH = 2 -REPLAY_QUEUE_DEPTH = 4 LEARNER_QUEUE_MAX_SIZE = 16 -@ray.remote -class ReplayActor(object): - """A replay buffer shard. - - Ray actors are single-threaded, so for scalability multiple replay actors - may be created to increase parallelism.""" - - def __init__(self, num_shards, learning_starts, buffer_size, - train_batch_size, prioritized_replay_alpha, - prioritized_replay_beta, prioritized_replay_eps, - clip_rewards): - self.replay_starts = learning_starts // num_shards - self.buffer_size = buffer_size // num_shards - self.train_batch_size = train_batch_size - self.prioritized_replay_beta = prioritized_replay_beta - self.prioritized_replay_eps = prioritized_replay_eps - - self.replay_buffer = PrioritizedReplayBuffer( - self.buffer_size, - alpha=prioritized_replay_alpha, - clip_rewards=clip_rewards) - - # Metrics - self.add_batch_timer = TimerStat() - self.replay_timer = TimerStat() - self.update_priorities_timer = TimerStat() - - def get_host(self): - return os.uname()[1] - - def add_batch(self, batch): - PolicyOptimizer._check_not_multiagent(batch) - with self.add_batch_timer: - for row in batch.rows(): - self.replay_buffer.add(row["obs"], row["actions"], - row["rewards"], row["new_obs"], - row["dones"], row["weights"]) - - def replay(self): - with self.replay_timer: - if len(self.replay_buffer) < self.replay_starts: - return None - - (obses_t, actions, rewards, obses_tp1, dones, weights, - batch_indexes) = self.replay_buffer.sample( - self.train_batch_size, beta=self.prioritized_replay_beta) - - batch = SampleBatch({ - "obs": obses_t, - "actions": actions, - "rewards": rewards, - "new_obs": obses_tp1, - "dones": dones, - "weights": weights, - "batch_indexes": batch_indexes - }) - return batch - - def update_priorities(self, batch_indexes, td_errors): - with self.update_priorities_timer: - new_priorities = (np.abs(td_errors) + self.prioritized_replay_eps) - self.replay_buffer.update_priorities(batch_indexes, new_priorities) - - def stats(self): - stat = { - "add_batch_time_ms": round(1000 * self.add_batch_timer.mean, 3), - "replay_time_ms": round(1000 * self.replay_timer.mean, 3), - "update_priorities_time_ms": round( - 1000 * self.update_priorities_timer.mean, 3), - } - stat.update(self.replay_buffer.stats()) - return stat - - class LearnerThread(threading.Thread): - """Background thread that updates the local model from replay data. + """Background thread that updates the local model from sample trajectories. The learner thread communicates with the main thread through Queues. This is needed since Ray operations can only be run on the main thread. In @@ -119,7 +39,8 @@ def __init__(self, local_evaluator): self.queue_timer = TimerStat() self.grad_timer = TimerStat() self.daemon = True - self.weights_updated = False + self.weights_updated = 0 + self.stats = {} def run(self): while True: @@ -127,86 +48,57 @@ def run(self): def step(self): with self.queue_timer: - ra, replay = self.inqueue.get() - if replay is not None: + ra, batch = self.inqueue.get() + + if batch is not None: with self.grad_timer: - td_error = self.local_evaluator.compute_apply(replay)[ - "td_error"] - self.outqueue.put((ra, replay, td_error, replay.count)) + fetches = self.local_evaluator.compute_apply(batch) + self.weights_updated += 1 + if "stats" in fetches: + self.stats = fetches["stats"] + self.outqueue.put(batch.count) self.learner_queue_size.push(self.inqueue.qsize()) - self.weights_updated = True class AsyncSamplesOptimizer(PolicyOptimizer): - """Main event loop of the Ape-X optimizer (async sampling with replay). - - This class coordinates the data transfers between the learner thread, - remote evaluators (Ape-X actors), and replay buffer actors. + """Main event loop of the IMPALA architecture. - This optimizer requires that policy evaluators return an additional - "td_error" array in the info return of compute_gradients(). This error - term will be used for sample prioritization.""" + This class coordinates the data transfers between the learner thread + and remote evaluators (IMPALA actors). + """ - def _init(self, - learning_starts=1000, - buffer_size=10000, - prioritized_replay=True, - prioritized_replay_alpha=0.6, - prioritized_replay_beta=0.4, - prioritized_replay_eps=1e-6, - train_batch_size=512, - sample_batch_size=50, - num_replay_buffer_shards=1, - max_weight_sync_delay=400, - clip_rewards=True, - debug=False): + def _init(self, train_batch_size=512, sample_batch_size=50, debug=False): self.debug = debug - self.replay_starts = learning_starts - self.prioritized_replay_beta = prioritized_replay_beta - self.prioritized_replay_eps = prioritized_replay_eps - self.max_weight_sync_delay = max_weight_sync_delay + self.learning_started = False + self.train_batch_size = train_batch_size self.learner = LearnerThread(self.local_evaluator) self.learner.start() - self.replay_actors = create_colocated(ReplayActor, [ - num_replay_buffer_shards, learning_starts, buffer_size, - train_batch_size, prioritized_replay_alpha, - prioritized_replay_beta, prioritized_replay_eps, clip_rewards - ], num_replay_buffer_shards) assert len(self.remote_evaluators) > 0 # Stats self.timers = { k: TimerStat() - for k in [ - "put_weights", "get_samples", "enqueue", "sample_processing", - "replay_processing", "update_priorities", "train", "sample" - ] + for k in + ["put_weights", "enqueue", "sample_processing", "train", "sample"] } self.num_weight_syncs = 0 self.learning_started = False - # Number of worker steps since the last weight update - self.steps_since_update = {} - - # Otherwise kick of replay tasks for local gradient updates - self.replay_tasks = TaskPool() - for ra in self.replay_actors: - for _ in range(REPLAY_QUEUE_DEPTH): - self.replay_tasks.add(ra, ra.replay.remote()) - # Kick off async background sampling self.sample_tasks = TaskPool() weights = self.local_evaluator.get_weights() for ev in self.remote_evaluators: ev.set_weights.remote(weights) - self.steps_since_update[ev] = 0 for _ in range(SAMPLE_QUEUE_DEPTH): - self.sample_tasks.add(ev, ev.sample_with_count.remote()) + self.sample_tasks.add(ev, ev.sample.remote()) + + self.batch_buffer = [] def step(self): + assert self.learner.is_alive() start = time.time() sample_timesteps, train_timesteps = self._step() time_delta = time.time() - start @@ -225,50 +117,37 @@ def _step(self): weights = None with self.timers["sample_processing"]: - completed = list(self.sample_tasks.completed()) - counts = ray.get([c[1][1] for c in completed]) - for i, (ev, (sample_batch, count)) in enumerate(completed): - sample_timesteps += counts[i] - - # Send the data to the replay buffer - random.choice( - self.replay_actors).add_batch.remote(sample_batch) - - # Update weights if needed - self.steps_since_update[ev] += counts[i] - if self.steps_since_update[ev] >= self.max_weight_sync_delay: - # Note that it's important to pull new weights once - # updated to avoid excessive correlation between actors - if weights is None or self.learner.weights_updated: - self.learner.weights_updated = False - with self.timers["put_weights"]: - weights = ray.put( - self.local_evaluator.get_weights()) - ev.set_weights.remote(weights) - self.num_weight_syncs += 1 - self.steps_since_update[ev] = 0 + for ev, sample_batch in self.sample_tasks.completed_prefetch(): + sample_batch = ray.get(sample_batch) + sample_timesteps += sample_batch.count + self.batch_buffer.append(sample_batch) + if sum(b.count + for b in self.batch_buffer) >= self.train_batch_size: + train_batch = self.batch_buffer[0].concat_samples( + self.batch_buffer) + with self.timers["enqueue"]: + self.learner.inqueue.put((ev, train_batch)) + self.batch_buffer = [] + + # Note that it's important to pull new weights once + # updated to avoid excessive correlation between actors + if weights is None or self.learner.weights_updated: + self.learner.weights_updated = False + with self.timers["put_weights"]: + weights = ray.put(self.local_evaluator.get_weights()) + ev.set_weights.remote(weights) + self.num_weight_syncs += 1 # Kick off another sample request - self.sample_tasks.add(ev, ev.sample_with_count.remote()) - - with self.timers["replay_processing"]: - for ra, replay in self.replay_tasks.completed(): - self.replay_tasks.add(ra, ra.replay.remote()) - with self.timers["get_samples"]: - samples = ray.get(replay) - with self.timers["enqueue"]: - self.learner.inqueue.put((ra, samples)) + self.sample_tasks.add(ev, ev.sample.remote()) - with self.timers["update_priorities"]: - while not self.learner.outqueue.empty(): - ra, replay, td_error, count = self.learner.outqueue.get() - ra.update_priorities.remote(replay["batch_indexes"], td_error) - train_timesteps += count + while not self.learner.outqueue.empty(): + count = self.learner.outqueue.get() + train_timesteps += count return sample_timesteps, train_timesteps def stats(self): - replay_stats = ray.get(self.replay_actors[0].stats.remote()) timing = { "{}_time_ms".format(k): round(1000 * self.timers[k].mean, 3) for k in self.timers @@ -284,12 +163,12 @@ def stats(self): "num_weight_syncs": self.num_weight_syncs, } debug_stats = { - "replay_shard_0": replay_stats, "timing_breakdown": timing, "pending_sample_tasks": self.sample_tasks.count, - "pending_replay_tasks": self.replay_tasks.count, "learner_queue": self.learner.learner_queue_size.stats(), } if self.debug: stats.update(debug_stats) + if self.learner.stats: + stats["learner"] = self.learner.stats return dict(PolicyOptimizer.stats(self), **stats) diff --git a/python/ray/rllib/tuned_examples/pong-impala-vectorized.yaml b/python/ray/rllib/tuned_examples/pong-impala-vectorized.yaml new file mode 100644 index 000000000000..9525f4115521 --- /dev/null +++ b/python/ray/rllib/tuned_examples/pong-impala-vectorized.yaml @@ -0,0 +1,11 @@ +# This can reach 18-19 reward within 10 minutes on a Tesla M60 GPU (e.g., G3 EC2 node) +# with 32 workers and 10 envs per worker. This is more efficient than the non-vectorized +# configuration which requires 128 workers to achieve the same performance. +pong-impala-vectorized: + env: PongNoFrameskip-v4 + run: IMPALA + config: + sample_batch_size: 500 # 50 * num_envs_per_worker + train_batch_size: 500 + num_workers: 32 + num_envs_per_worker: 10 diff --git a/python/ray/rllib/tuned_examples/pong-impala.yaml b/python/ray/rllib/tuned_examples/pong-impala.yaml new file mode 100644 index 000000000000..b54c79849c5a --- /dev/null +++ b/python/ray/rllib/tuned_examples/pong-impala.yaml @@ -0,0 +1,13 @@ +# This can reach 18-19 reward within 10 minutes on a Tesla M60 GPU (e.g., G3 EC2 node): +# 128 workers -> 8 minutes +# 32 workers -> 17 minutes +# 16 workers -> 40 min+ +# See also: pong-impala-vectorized.yaml +pong-impala: + env: PongNoFrameskip-v4 + run: IMPALA + config: + sample_batch_size: 50 + train_batch_size: 500 + num_workers: 128 + num_envs_per_worker: 1 diff --git a/python/ray/rllib/utils/actors.py b/python/ray/rllib/utils/actors.py index c663087eb04b..68788cbc99da 100644 --- a/python/ray/rllib/utils/actors.py +++ b/python/ray/rllib/utils/actors.py @@ -12,6 +12,7 @@ class TaskPool(object): def __init__(self): self._tasks = {} self._objects = {} + self._fetching = [] def add(self, worker, all_obj_ids): if isinstance(all_obj_ids, list): @@ -28,6 +29,25 @@ def completed(self): for obj_id in ready: yield (self._tasks.pop(obj_id), self._objects.pop(obj_id)) + def completed_prefetch(self): + """Similar to completed but only returns once the object is local. + + Assumes obj_id only is one id.""" + + for worker, obj_id in self.completed(): + plasma_id = ray.pyarrow.plasma.ObjectID(obj_id.id()) + ray.worker.global_worker.plasma_client.fetch([plasma_id]) + self._fetching.append((worker, obj_id)) + + remaining = [] + for worker, obj_id in self._fetching: + plasma_id = ray.pyarrow.plasma.ObjectID(obj_id.id()) + if ray.worker.global_worker.plasma_client.contains(plasma_id): + yield (worker, obj_id) + else: + remaining.append((worker, obj_id)) + self._fetching = remaining + @property def count(self): return len(self._tasks) diff --git a/test/jenkins_tests/run_multi_node_tests.sh b/test/jenkins_tests/run_multi_node_tests.sh index dea16b4d705f..51e27602d385 100755 --- a/test/jenkins_tests/run_multi_node_tests.sh +++ b/test/jenkins_tests/run_multi_node_tests.sh @@ -128,6 +128,13 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ --stop '{"training_iteration": 2}' \ --config '{"num_workers": 2, "use_pytorch": true}' +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/rllib/train.py \ + --env CartPole-v1 \ + --run A3C \ + --stop '{"training_iteration": 2}' \ + --config '{"num_workers": 2, "model": {"use_lstm": true}}' + docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/rllib/train.py \ --env CartPole-v0 \ @@ -177,6 +184,20 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ --stop '{"training_iteration": 2}' \ --config '{"num_workers": 1}' +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/rllib/train.py \ + --env CartPole-v0 \ + --run IMPALA \ + --stop '{"training_iteration": 2}' \ + --config '{"gpu": false, "num_workers": 2, "min_iter_time_s": 1}' + +docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ + python /ray/python/ray/rllib/train.py \ + --env CartPole-v0 \ + --run IMPALA \ + --stop '{"training_iteration": 2}' \ + --config '{"gpu": false, "num_workers": 2, "min_iter_time_s": 1, "model": {"use_lstm": true}}' + docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \ python /ray/python/ray/rllib/train.py \ --env MountainCarContinuous-v0 \