From 4284b332910c08a5b549d13e8faedce4b2137829 Mon Sep 17 00:00:00 2001 From: Anastasia Alexandrova Date: Wed, 1 Nov 2023 18:13:47 +0100 Subject: [PATCH] Updated HA doc (#468) modified: docs/_images/diagrams/ha-architecture-patroni.png modified: docs/solutions/ha-setup-apt.md modified: docs/solutions/high-availability.md --- .../diagrams/ha-architecture-patroni.png | Bin 32695 -> 110268 bytes docs/solutions/ha-setup-apt.md | 613 +++++++++++------- docs/solutions/ha-setup-yum.md | 610 +++++++++-------- docs/solutions/high-availability.md | 34 +- docs/solutions/pgbackrest.md | 341 ++++++++++ mkdocs-base.yml | 3 +- 6 files changed, 1039 insertions(+), 562 deletions(-) create mode 100644 docs/solutions/pgbackrest.md diff --git a/docs/_images/diagrams/ha-architecture-patroni.png b/docs/_images/diagrams/ha-architecture-patroni.png index 258aa1443a6debf8e164b11da30d5550256704ca..0f18b0d617df13933c8932e615da97afc98e2c9e 100644 GIT binary patch literal 110268 zcmeEP2_RJ4|3@k%QNl|L*;2C3SX0Q>WNQ#bXl!FJ#?BDZYDr3Hv6Zw66_G8y7K*H; zvR4waWzYUUcV@08^yYHeiOP06N&|HplRo(82_q4{@ThZ_%l*ym?CB$q9 zF0T9t6@Cc`jH9E7B@Tmi#^7B<9IRZyA@Dn%V2QKFSy__DNQg-wgvG^##ijMdWcU$E zUKWB4W1Bt^u)RDxO<3!H;1!NrCju?c)fIk;Nc zgTKLH@P8dW@W1WgkGPnbxTKl%8t_rc(b3+@(8^o`2UepdiI5gSNP)wG8k_aCH2Ecz z!DoA%ofY^?-O9p_0G*<2<4nMVBWmK}G9qHozu=%A#v0>{qnZNRnUyOBOWqZMXy_qL zRC9DDxNH_yBkE#N?nLr#+^w8la0Ca+=EX&%M8wHIxDp+$$VagR0@fZ}5KuyuUrY&1 z4E={trjQbRaln8Dkx0B1P)C?D1CBycbr-amzODT3kwci-S*SXof`OPLYl>`d_ZbeJ{4p0Yd27)P-2PiBV>=viO@;Yfs` z8fJ*IbOiu`5GgJ$B}-lmG<>ra4r@cXw2Xu_`3F1(9!%MZiw(w-;6Xm0bcmWOWIDps zbS4nMbW}{E{uv&G_f)pBhfoUBk#ZdXroVsg!P3O^msf{Z@ht&Xuf;%n7cp1U$qkdmI+hP8I-9R)9B~ z?4gmHFcx-LXM&r9B^Ar8N&kQ$@T5wVb6p7#`^hYZg%7Z6I@4vPq!Ab?=oeVVAOu)q zTx>|@0r~=dA?ZcDCom(9HX=BJi?j$1Xbl$;Ji!6y3gKsyHNnA^qJqSwq3^&-P%YF- z+)~mCI{qWDC5e;~k*4TM3cMsKNTNcQI8s7{GKd;qu&Dhy@PhUEU*XHe&dSvS(y$Q2 zsKE9KBS{*<qYg-;SP(vqq(0%a0vjs#%uNwO!oiC=OPFvenljLKpnQV8fTeo3I^ zl}SGUtpN@L(fH&j(Ec+VB~3APr0i!VmL|<9B1@Wz>VnX4(nYC`NdPMfuJ>^)Sl>r^ z`FbJzjD$^*4Ldh;D`((mS-FrT1YF6{83jZIa0W2Q@<*UwLL86*(nm1#rN9^HI~Dd} zai9i1lHzXsI>47kk{3>8AhnN+g*= zTPs&rB1KtY+*}Etij0dZ#@ST~@_)c^2Z94Ir$B9jV`@O9Q6$XD6X&{}>hlihGtef8 zP0%kYkd1=`k@N*(1L=zcFo2|!L?nU73x0(TKxYaA83IhB9Qk-Q^t%dt`gC1RI)AH` zGY*(m$WD;1GDV(&)|IhDf~lujdl?CHgcP)l&tlSr;O0znCcuap?Vbb$Xn;6a5L%Gj zyBP<mXZAIL&7bBOf(V@|BLA`y`dxI3q%=ezX{0RWA4(iUG1cPY62QBLf0LDktpW^0 zvX}d-T5hnnPxPiZ?09@v%MCjv(;SJZ%aMR{6EHRZu^;ra)i{EJn!i%xa3TzcCuP1T zlWw0)B77!sXK7`PakB>j)8q`FrtPMsYDpJVf)ltM%m9d9!QYuf{X_hT$Jr`PyqnA@Swm261=3e?okKT1At{8qk1U>l;45Af6N>~Um*JjC!wbL0@IU8pJcr( z>@hAbI14Z>b>a*1T|WgbzXZZk%3MlvGMhzCZy_W|-U@YkOKkGU$3=dU-ujwk7Dz!$ zSU{`#EMC8uvXZ7$vc8#N}`3uf2w%@ounNk_EUv>vgQL#1bx|xaRmuts0IPhFZNN4sWXOT zk(z?`8H&JvotpXRI8J7LB*eZtXZtmCf4st!92a5=8j)}X5Vjis*fst9CE=PiEOQ77 zZs1G)Ynl75S;1)(gO?9H#ILIClL_;A@_ddXe{X(LWbjL79cR=Xpr-OOTm#6MlB_R~ z`N_f(r6ff%w3B&)Pt7jSACsqn&pRNe;Y)?A8FUV{s|vgVYZ(hGlBJ&JFPO_nB0&rb z+wdo;ttu z|BI3Wj5MmS4|MRjVx6t@(4cn_qAyiC5zZG_5bbdemR8OP0Q)Z_f6bu=A1g~ubGU8G ze6sCF$ITT?164jr7M$GfM=Bdn4db#w?@8QAFl(B#03YDS!rS*AkCG;BbMPw#s-GE6NCPu0?JJT(n4 zlTqDQhK1zqeH&#?@}Yhe9X&1Rk%0446rWELPGZ3D``2#fcQp&smJjD!zL8n@M)Uu= zKHtp!Efl|>O5j1v#2WzhDjiHxxUZ7J#E9xQ(4kvBL>oYP}8%8Z1~p zRIks>=IKdaehDd3A|K)uF#Z;k-PBS}ZcrmSNL3-owI?ug@{kl3`pE%w{FCBw${auh z$+**7x-V#9=&8AcKIFEEQy6cYzu*u+RHbATy&YKnVt^6vH&OLQUO{ zFqa@ZQx{0 z_2oeAAE=}sljfI#?LG`ixK;y0?bpt^Pg_20ME_p=^*7_A&t~zaE@!eM1E%Ibj%I&g zKhKPYReZXyuBxvLxjVoHP-%f*$*KN=g8!%YOi*;W1azAgw0nx=NWpO#l`jDc7qu?` z^_*&G7tlsNaVDnl^$U^pv=*yrih0uAqg-q1t*_svHcH_Qa$3Mcar3EoGkN1HHFG8p z!UFq4qvRPSPhz^{O_RF+ZdZhEoCNatJKV1LebtgA>?=~#lH~7jyW-5ir-YwW*5wEI zn3H}ZhNJYM%l~@;>SrU)Y4gL$=D#=p-wc3%b~Fk-8e>X;@*`XWxG78=JfnluBn}-QI|vfu zlL!7yNA~yX^P1V(eyv&lf7IuNghO_Uz5UjGUQ_3nhSLFmPhLN(4-k@+?jp+i|Ka?< znLd!1UKjlT(?|B53ofSLeLqJY6?CJatePlDMrkVPKzuXM*g+4Ms zn*X_;hJRV|Lc+Z<6v+I(ePpoLO{E#8MU+3ik!)%;c{1BVRmFliLS4jyO(C2i`S~?0 z1SKf>D{}v0on&N${DZ!Y-;X&!(?NFN6Xtwbeg4-=MtU3og?4{k&Ysb1{bPjv&oz@F zrkA+?pEs9CPI>6;@7r7go~AKfyFluOs_yf9HJ8XtDT2)81x@ymP}}9ngTIhro~e&` z3U~f%F8Qyygko}J;HRKa0su)V$}>T!oF3Twq((m6)BFnopOP`A(n3EV_4i+M$tO=7 z`XmaQtk!R9tL|Glztc17GuBOWk+9 z|9m>l;Pro_dh+xOz-sF6UEnvX97z3~-qXnj6R11wV{-}U>7ji8TyF{JP}@#@<3bPrU#j5t(Fv?*hM>5`sNjs>JMnT`k~A571p_|FV)L zJH_lHV5>$I$O6qk1{rlE`|CX&2{BXkb4m*52ZXl2XIINF9uvyIu!I9NxEbsBFbva{ z4?lzE@2Tz2+J$d4|8Hg%U?NhRh2QAuNZ+@sMFvhk!mbFcu&ER~EJW0L9VRL)SUKD=MZox=f()FCqlOHet@wo?1I#{T`KdGm`2bax z`=@uc$ifl{ODJsPsK^Z;q9!*i*uN0DrjuF zl9B>1xPsq3V~DeKgHjesytL=jm*P;EEe)&5{{d$IlrD-X#&_~9htrHN)mu5H zzS-j|J?(GkFvDE`VjCk_roX4vhw3d~dSe;Mb;NrrgZ$PP-faecjb5-wpKJ)b|Dc^7xyP{IIF`BL2vC zrvDG(Z-!@xSUH3D1z><5q6OYP0$u}+bHIWx7zaz}5XJ$#wi*ur1ilD@hoP^Tmi+yD zFDLo*5!2SzR#G#+^376O)*Q4Zz~TD$t*1#&$<|NalS-Ac5tpUhMM9mfhx0hUkn1u1 zX#kGcP2hcCx>ha_58=v~xikEPdqL<|H%IWBo|QA8^Ec-3*LK%^Uk*#aZ7vjlLmHw4 zR7IlZFzlmKa~MAM3vu`>T4CUq$o)buowvleV!%7+$t#jjgI*vnvB?u}4<2W`b+fjF z5pfgJ+|bj_!b=Q?*{myOp+a!iL|7s$iINB{qNKY8-ojnWR>?z4S=P%EZ-GN?HgYuD zZmDdJz{;X*m9SdMN#Is?+G7(7iA0*lf<*8|F=b@v50>@Bp+WJaJEus=p3x`B#*k2v9%y-sX*7X zaMSieOKKC9yfkgG2yHJ53AC3gm`e*unxBXi*V0!-g5N#0m64L#wpwB+6)iVy+Z~?T zdP<&JM6f1(B?{h(SS>xImlpU}Tg(Fq)*_~(51j|DrGh2u=o@HhDIz&ZNc#B`KNGywa8)@uRISM}6Z(N+O8ansR9 zO9C1KUZ_fH>tl&16^Mp_2H+g97jYexZGi5oU@u_bUJ%^?U%c$Rw6=LbbOHNQ1$5m8 z_JKy|=v#s{k=Ey-jf2*N0<_*H4$ddh6_6c|cXrt;$rj4qXHsBY)sTjZl z!VrgY--_E|Yo)F#y_GBx0QEo2T2gEbMVCrRQ@&H_QVB{>M6F8UW513nh0XWR^2Pzf zs2bqE)c;Nn?Wju`pQK}LUe%Jo;smqswebO(7sbrGCgaPBp3;h~uQX*V+D*~R>FZ9>veEWDIBHLGs^$Or6(OnVX&EvIeo5w2zd{(|^)xX2 z2fjxt&~y^77V^{3aMPevl=MAK+hgXQT(a%i!8}^k2Q=}Q5(9S8Epj=s=Omkp zdvW2r(+_SX^1R)D(oj6R?e%Gs$ARK=M~lt3&oVdAXAz_8Rk+x~Y3G%7F*{M{xp8SD z@8eqU&JA6eKDW#owlcl;r=jPjp=IC;zoT$usU-5O$vNXSs~jJyW;1J`3|j?AKfVc9 zRMsBL$1^C>(6MZ$^%puc>U}t2d`-(*6HNjC{Jez?7K#$tYAMh}U`Q$hcj3}`EPKGr zN4L`@3!1q{#0GZouFw$hA2b_#A$L$v$J(=njWo3+(q2MsV+ zpiBOM7DwZs#-bR0kze=GyVc+V-Dk)bD3hW!C|tS8nv~xpKZR}yjVj{p<(xk|GOIOY!lCC z59Io<=CvIw;Cjlale0L$uEA#_W_<9~h>_KUZIT7cUFgkI4S0|HMPFrm-kKC#a$A4B zLw4#M`?^M`3jD6YJ%r36GAaRb&;(n#r~2)FsQgJ~VrJaj-|O?+ov| zPVMV{O1+8a_XYGA*`V1;2!AcGM=*3fT^ciqRA+;FJR1DeqwsmhhLfI~Jva3|F6Y`( zR$KqRsO0&1!;rGs*dt{kHC@j(h0k3P$rsea?W4KZFIgd=hnqK8m?RSnQGoXLcV8TW zwo2PTzjynZ4E$5+u@SEgu7iz64pke4hkF{sxvUaf+OB5C4!jsD6;>H4KIc%@M!0?9 z71r$a)1l6hK)J5hE`6iHrE)c;BZOlRDOs<0h)f1Q!lb=F(b zXLtcVy_;RG`($3*Wh|##{*ZEi`G?Rn-G&Ia$FB+!Cx&MkISqlST<-}#zueur8tL|K zNm5?P!D>uYjAc!GhiuFHu{3mEh-7D7Mqdjnvt4rL+X`R5ja$7t(o;A&Ib9zqh*Wev z@q1poqWM|wU|kYH!1o+Jy;i$m1u?!i^>su~qt1$WQ;orAwBB5AUgS-@D$i=a65L-h z(ztvgy?*yHtZH3s`tZR0cyIPX02S-8{j+C&yjSy=baqx;UlBu z=RIsg#m{`xPv^75=Lqava*za=lK?OuRRWPtDWE52uwI+Xcl4|&akx0>QZBYoN4A$w z{X_Ms9-M;KlRHV$WubZ(Th925#}nV4GUQFZwY83;EqSEVbGrbi^b32u)`xah^$!!o z3?J7TlXHE;#wo3P5h4|~iC53sIpl0P*Lh2|V5RTg^lJN~k?ooKiR1P+doK+4B#OGd z+_*1Bqs1YoNiJ8iQ!97)am_mkKMvvRqjAx40*|G+ZXJmA>O|EImif8mKf>B|)eSzO zJ=A%wfRn3SLPI#wXW)gInQ#Ar%}T0hhCV)qu~)4r8AYOvPZ*X!ff}yQm?pIB^r>uC!AEyEiJW=8}FYyVkLz z*l_*r%$lK^*BM0%vkec&WYBz;gty{%S27uFE#>!mSK>F8!Sl{Je)LLPa+xeg%;ju{ zp%epSB2z(0qt3=FV-r~!KpCn;TiA50Uk=H(33_0lkn)SO})+O{C}BG~`u zbcT{CB1my?A;*`k86ZCoJ}iWZ%I=|h8}hkZ=5JG1xoHvJh? zl%lU29_L=M)X1U#{PGEHSb%4>kI#QyYI!J-Qpx8-Ahc&-2YT44C*l%tV4s9 zBp(p;U}AdVc*bR*MunKyKhpMCuLSR45oMCGHRAz$%9ik`mKE13=L~lq>PmB((5m#V zs=r~F8ROS23)GnSp%oW-nPo3zGL7Wo(SD6X9&V3O>MaFpj=PN%4r6Oc--#%z~Sa3>DGJF8{N`} zrBiEcHwfQ&AJ6GF*1Qp;{-KFK$k$W>O4bM<6Icg8Y%thl~W=k(}4mw~$U6a|F=+1&2Y^~J9TA2S}5 zZO+AOWc#c^ZwZe>^z9bTG?#8ZI2@&Z#91?nr2*H~akGT8O)jx7|60=RoBV|-nXR@l z#Bp2}_UTxOcXh{_n?AW8G?+xcz({9=Lsw%uW4U_+P6wY^#qAcIcrMl}y*x9iw%%ks zE~?X6#akiOF8 z@;l-1AVAc z_M*v|n+9ga9NJ z-*5Qt+MnvSZN+n22fWA6wK?2Yd%n|spPa|(JoARwG&W+WAIDB-?Os)a+3x-W&z9ag z%{yEw-@mhXqxW2|SH5IkaIB(|N`;Z-9HDvE? zbt=%+J14%@pp1(+s)U_g9lO3O*Q8*yBjD|@`@Nyedm^qXxf^Ds8DdL7QS1284v%_Q(+KMnv($q9*ZN(`6@=gRVIH`7NoNpz zOzryJq8ldqdUfQSTAZ>5$FrloAYUifqrq@)hxfJK!7V;*Wtb!E<5k7IroIE!Y4?t- zPsnVr?(JbrsW>B?o)lyz%*6^InRNx|VWyej_7Q=?i`r4|c190$ujZ1? zxSU8V$DhwLhH?&5Kjx?Px>jU;oX7t3n>uJEP+KSJxU3*9{ zWgf?YRZtYcZ4Cl|ld}|iNfNy^R35xEce+B5%B0nFkRn+!z&>_^&t~bFboCng69mb&zHuYls4VJK?iKzJ>HwkHijnap`^XO8d` z&Uz3iPS#YXfVXIQ=Oo!%oB$)ntC}(e;^j-r3p}6^t-N6IW{+FM!Q%aEfhXTA@YG)} zNRWBS>U`3sxh=sekJ4JKB&}!^&|?I9qWN?{kGwip1GKP&P;dz!@pA(6Lcl z=e`q+m){!w$3BvKbriT)%5x4ANh^APSyI_e6 z*-Fa;A$qTQoq;5mw2Q>h_1t4Ck7jC~!s$NI)x8&{`$V0|_Jf#S$?P|p$C2xWnrQ#L zzknpR`_(}9fYWFJDQ2Xhzsu|R;Ce>%v$CgRX_wF9UYzCVBpP`NW0sPn0Emhh!1TR@ zU`i&3&;Q=RP$!l4CgnK^29E_y!sEn;UydUrQ~lQ8w3D|g+ZGbIoWXTI`Q)QCK%(gB z{YA)`4_Eqy%fv6_okprL+2)M4&|c1^U)LJH`$fy27bQ0J;h-lub0P_3`Z-PFk@$cfxtC}* z(h{6i0Vxw58`qJx&RuwJ>8yYr8<{8bN#7IVfIyFl-=9U2*GvAu&7HH|ZHa35bK1}= zBsw1z0z24wU>TOAm;m95{{U+8-YEmJ#ux)jY7JerlDt1A`Y^%538!|!=6;Oit+Vc0 z9mA5BFZ=D|z+4;&MwYk@| z5qJaj@Ixq$8uAAC!3Nf;vYL_gI*&gHwS!``F#taE_om+oVzN~OVFls<*#faC8v^sdUs)Sh$f{qz+6ZRXc6Cq z&p3Xul!OSr;_$vB332>mi_;MSJ=f-y-Ut;8J%~smb8;A9{t1_|JSiWkMmHD5xufO7 zb`%HO`0bks7w@_Yc!& zdYw+6+BgryEbHvLpxteqsY-C0pCLFjk9Hx1)MwQ^hKbC!|F;Bxww_*$ik;FU*!HQ+HA}i z0_C?RXWwYLlw7fQ;S-kf`93P)SnhyZWF>bDC^_}Lr_>-`yops@k7C?>dhO*@zU-&P zruz)uof5pfW}W%f@I$KU=S@B%!v1NCWDF{1hrK<96n<9%Fudx>=b$e|mmV>;0@ciR ze~5sV19E*3WTiPvD0pgJ4LmBp1>GbGu4b@b+Zq?f9)?t{C{SH!&)r&ZEqc?J7P4Kd zLL2Wo?21x2)U+gCe%*~@K|MqKD@@O?cGu@g>(;m85}ZTAE{mWB$icsa1d_&Ikl|#2 zoN2`$s8TkO-NEs726&)t{#@C!C3A)Ut(wM1oUsadqPVKN4FxD8p^|QPCOpMpARz@R z>(xhcDZLnOrKq@g*9tzWS9h4;GWPT!^3`(}^8R`CsA}f(K+~}L2-iIc#{6S^I3?7@ zE!CP8Tf%$;^XIn;$PBGxP#y9OOBN0236;;i=FRAEBF!2jt%f{8F=vWr)F9RBk6unv+jOgT zbqQt|Ka)O|mXyHHXwVg&=7C{!=4N7BrWAf?@6CvEFkgI_Y5(E4=Zt0p?EU z@wHhmj4P_b(g*tn%?=dJ&g*NN)lX9TG<0YPxJsv_NMG-Gm)G12b69uP+?Q376X5hd zFMCt6UI%}b)6RC>F~ z+ibO=!d&hbm{}wu_dpItif5xRw626j*E}?t!(A6=q^uT3_V=&eguab^74t@Di`M^qaMMe)MU;)^LK+AId)bRkF_B{MEV#uJ=VG^z&_KE|~j{1VW!V>ynKmMvXWgnKN zv#;?|E3XSGY_52?sS&8a_`3}qJsB6{yHCPU8V73`b2`UPa*61bcVSS*TdJ>fKUFxM z7M*bU&yxiFY_i$nUDA|}ue<$NR;ME4K;0iw7c>)Us(Q}zk8y@Phu3r|26&$C)=$YO zaAa@|vy{e#E_A68kC%^r$z~j~HJnQXmvYyx%zkcu+1#lyUo)DydcdLA&I z$iefpikoR~okzklHw!JWtMo^4bfm9ORV%K~UN}G~J$oQH(|&eu(%q4w4!T=&`8)1x zAr7l+0&&j|$&K2*KS5`{bP98%>)wPIer)^LQQvrqN4^8-wNhKlZbmSlqVv3~s~^>) zZ|5X9#X58fwH~~txSo0GgGw ztIszhP*)n&%uzyj=$WVr(ZSqc$qorGOOv~(f1S>I9aQk<8^+H%u1AgFmsd48xG#Ub1(m1TC<*Ir zTVR%}U7}0iUVxOCrIIAA%PmiL0wt$j!GV#4TtlExRt5J|@r{Y#xKPp@M!s7r3J)|g zv!3e?gHhfODa=w9M$-1TE~6dEz%D~6^SSltOP;M%ub6+}c|xW2&BBU+p0!GqJJwmE z9ZqEVh~OkqDNC}vA`?RRlJv08FE1op@PKd-ZRyq`$p$^lU=73N8vcHvXdyr23X!nn z2v^OM;lMwv7`x~GFuwMP{BDEoOEWyeYh0XPy&b15_uLFClU+de_Pq)q`+}EZ@2mxmRMt0(!h!&>SQJ_ox76ebJ^-2vXLTEHbyR z`P8WlEj)mDlgoEgG@yGMe=eti;i4wYF+5J12klG5JgDW&5Os@;uvF;Xc;fw*G#hhm z@+w|}Tu?2Sw5<~4{Os|gOP3grPuRy_C)%oq%I0t=^xa<36P7;&Bh&&cM&_PS`IWOO zOGy$N5CVuH)rTbIvoh%IRcj3pTH!T$%I@9+rn5Oi!ZvMSJq+Tg+5X;y}TA!XG5O*BrGa&ucH$v!7e$R!jE%62^3-vR-UO+1wr5-bIl^)p)Nm5 zrqv$BhX8M@X62K+3c~ryv@7O>Jv)xvQDG4#caRua+c7&{{`8{2&-;Tw9WaFrta06a zHGz?@&YH0*EScfSS&$Qu)nekjBEXcr`{AA^ckP8rfd@@Ivhg;#GA#@)d$TN_9EH;> z?czm+YE<8F6#1|OmG0wX^C~YnOkhs=)-|NiBRvfYbY=|2r07;}slMuXw#H!gDhKHs zrDnJM>wE_{e=$6oyCrLLMSR#WQDx{|n7vST48l)joiqU3F%XFNTBATPV4D%q4Ewk&4JuhK-7m7z-=sPrc!a@z{Kgh?sL#9rC#kA z_-p{8vWpg#xS3P5b>F6fj=dQN_=;EK=Ay8u)0~ldP~`R(e zv)=9iqQB3_q?&XB`4>^u6{$wPp$TsD={EGGz7b&u=6Tlsj<(N4a*De^q=+)wQPFl_ zVNK9bewftZ3Md(n_E};mo(UIJ*a3{eKy1&$m0;+l&kqH%e^7At3w0IZ9x-upLi-d* zJD0k8Izc@;islTUn0qy$i!s;8RChIqE&82Tn+8UnyfxIx5ML`Tx0al$C;{5Xh260h zN>#A1`p@PcyI7c>aU6NR@CKfJz5V@@R)=!{pnf2k*PtCMAtzYMC z59hf5VXKhKkoc?*9_7t8GCqB~n&*rO;7$hinxi*r9F}In>*`JD%?;U~s#Bs*&PTT| zddc;m{lSt@ve&JAX2rU40^91CXGBzL)g0SUe$Oz zTgSN3S+Z-R$n6cJqAGaJN|L5iT;zXGgj273L(A2$=L+_O^njE-WC!d`e;o+7X(<{f zHeC&yD{WtGB#e!amnqp!&I7iC-~qcj(vzIWz_SQu8keUM33d` zkhbN>fFw=xF2E5Q&&-C%Cnwrw0a`9);GyJ0@$~jLqqp@`pB!3)hxR?PN__>HEUbVm zvGE}(B)cnScXKyD)d{W*XBI7gwOGcY@6Rz3Ztgue>_!^DSr&{hoV9t7bQQ=VL18lF z3Fd+ewuY?OK(30A29viE4n@N9dtAw>MABgLc@-eej;()D?(%UknOFbrBAZvzdNVz> z@4(tg`{egAVQsB@383@>D(WXb>@Tu26{CoXcbKiJ4YT9 zM>NNAuP>)&bd_B;$sSn25laYM7*?$eF2) zT$|zWNc&JL>uo>3yGOM|x366-(g?Lg`fmk0QhpE&cEsIApX}Syw?6A}cFTYdyGgdm zGoJ?v&`w4CGZg$@ozS_xVzAsP$}CM@pNwOuem498y$ncrv;?BX-M= zS#G+9LgKLIZ;;ATS$`!?S@_nBvR>B?ZM=ke2z`W?>ThH>13FU ztV=f#87shn#Wc|0j_Em-@B87Qj6qTFhK27z{p%7)jOX|s+zznfyyazI`M#*_{H3_} zPGA9!47IWYx-oJ#e7&grS=rtGr?dkmZqU23iZJ1S2PkrhaCVfI_|H3yp#!o zB)O!ft_3byKwt8QkK;s%i>T+InvZ*1w8a9_Hlo2c9Ac2T>iZpwnq;{cZh>yHry#^O z<=D6uOiF~BBh}xe{Lx@~XyD{4SNHaW9UN~`%8Ax;ayp``j?w`-Eujw-0m2&yC8u@= zv~UCQRso0k)T|=k`{Hz!Q&|RsdR{A}RAtWdk9{mwk+=kSQWpicy8;!wc^wcWb7kcM zuq{3yA*)x*JrH_1;L&CK!MIW25+SZLNv=Ioh`r}g_u*JY?jf+&;q}6CI+V0GWAk?o}YOxuqEb#@3{`cw(8N|WerjNrS-RSPe)Hg z?V2E#uM{?Te!Ow%27i#4i^0C6z`kxYDtCc> z?FTldVA&sCk#4Ltp*e^hQ6kSTAE3#IaQ0BGWort(y-%(*$$DJ> z(m0!1!ivh&f=vNEp7%7`*6;w@qOY~A1+?7`XnX(Q!6U#tpnx?ET`gDN`?_<76Kf^& zcvEiV&203}?X4n| zl)9U_Tu9&UQT}21St|z1J%9_dlzGz6R3pLVb zo0)~*aAR9{?@C*<2tfthMUm68{l-|gIuk9?jd7- zfi#K(2~p?Rvn8>TkT-ZBva7Pk{#N!1pX^jKLU}o2aOahHLD8li=RM-{#!Xr~`mBo# z`ayS2UA^!4v(SUb#@YC?p5-|;ad~kYukiKV_F;cM-zO_W#zNuU5p3VH zTRO%O88-&nZ$BC4$5)+Ld9KqleaPcMQ*K^^&&7VDn6|-m?`M~@i^j66)>K+M4YjDT z3D1T~SW8gtK9>kBv<|QSVFY3x2 zEl$toAmB2hUT~H1WH~jp@{c&g*BCsn6YhF&V9sHO=24sbSz7#t13I^xl+{nxq}I$~ zJi*pf)9G#+)$U${%e8)GM^~TAlA_N~*fEkTlwMTX9Vxxar)@>4+}_G}wEEs@gR zohRl!z#HYh?vZ3(xb{9sBd_#kXo38YkKT)a3y}A7SO>Z;KZVls6Jzp+q)d{J^Ew8r z61&?y7Ji@1(1JZzi*?NKy}v`a>Gs=~Fn*C1uae z2q*9Qb+sx4o~o6sj|3u3&<>kk2)9LwinOs67(CCW>zwZ>)X4VAbfk0sE#vg%P`HAnw|1bqMABY>D|AR%>n-y#P*UoN0g_uR(`eM zeELC-l<*0;=gUhXdn+&a#WQBa-K?ugFN2gc>SEiXrp~p*2L88B>B$Yleb227i%ig+ zZ0Qb%b{^ZgC7gq>%%doar$*>>S8o4)gN*(M_o6jDX!?!0bRsFsA!M`YPlGK2OI zF59P&k69K58uu5`0uSZRTyT`xX5*EUf7~|To&7ecFHwjUSwiDqaa$XEUINI zn0es#N_1!P8)Jo}yv_!Elu!4QNI_B4^CCCHI;w?C&pIU9UvCl32&tR(k>XXuTo1el!cVN#vlBHzn_f2UKIQDumX*Pr%zQrm+6mUC^CkEbtkvfW)A$j+;Td}SZK)_k0ln=Ki_Fzx6Sa(5o5Bka3xZ0I7>n*@X)4*#d+;E8MrL*jMyUC+ z8M@QK#KZ7Wu#cdru*ZW`{QAL-^+i>+NsSRAUcIr*DXQTdj5xyj^CG@N)^CN)Y@@v8 z&jlK^t=bhodI;C>zR7X}%jIQq{sOocWlFp|qUXgRnLdD}3~C(ly3F&Etts-PUL{d3 zYCE0DTho@!D#Tl_K;Kw8KBqy4YduFuX3g{U6G(JT(hbGnd;tH;m2Yj+Q=-bRGu55W zco@}^>tn=$&wad`zcgZ`>UL-D%i`fypW2T82V;b`f&SNw)j8N(#p#B(IO0&E8$(Uk z#-A5)>UG%X(Q!hqtMg_-lfx@t)7RHF+Kminq!jBvIqsCE6UAZ^yr;v~PFK%QMYfTz z73fSol7{qm>db4j8nGC#D z5BE0P_lzCYWlOwV^SmzWL1Ody^b`)W^taEF_`-9I>+32zUN5UGj`Y^fNn(4y1+OZ7 z#b%8lUGy5}AU^lI+`H5{mL;4G=n-o%M{EW{%Z)P^x(=2e$fRYv{-9u}sB5~BgHE?l zx%P{A$Ms%%$0NIPU#vUkGtee1_s)TBaQ@(N!uX9!q4KLCp#_I}x1^;TJgO^+G!jm+ zj>s@Bi~7vN``p( zEXOEr(^Qw^yoJG{Cf?1d4X4+?4#JK2XD>F*(KUUy%_7bo#w+;){cQm{E=6vWLzpbxO*Kp#Up(}4tGsN8hP(J~%1EqLCusVTybc%3f*4XedEsRq>@*CSeW zr8Bbg@|r8Vo}L@oyR9nusZ`z}%H2UteZv8QI)wI~XD%wldY)K4Kj33C?G6C92`TS4 zzMD44KG4c2`pSN3M(~SgWzAvD3o^Ie>}YZCt^xNel%dwU;V{MCmyNr-uAcEeR?FV2 zQvpoT#q{(Nb&WmGxQ}WA%vt6iFf9ga#quRU5wY;G_&B#k+-sg62rcA?z_5bd)X6^a z&4icj-tipFpn*_z`h4Gd<&LQJ9=+(`w3bVQqqj!ZcMUlYM%W37%Czywsu0thF5K*c zd~{K3;ydx)JbBgib+>s6pR7>uGUKNWbB;J8XdnqVT6u|QO^XA{ZzNaLwSJ$Uy7!~F9f9wy^jLYela#Y>E{|7@XSf=NN%I#?y4O!abTe7b{ zy_tC{JF9N!VqmpH#B8IJ-9i0L-oo!jlRs!?;&)^ec%!0{jy4SDaY@>TM0O!lE`)~r zE0VJb9eqPVP>*3xwS@}Nls68Mhbok1^rY|PDdgR=^TIsV$MsP!O8wrdZ}^~$4cVS? z`3hrokm}H$@{Y8}JM(PbmWW+w3*xGB}j+PRmxW_2}jJ?3qmgD*vn?Y zP5WIhgSORD&Bug1qvzJELRkhiV!c?7vNRpuOA4!G<@$=DdcNnw*Ok@hZ12P@Z^X7h7ET#FfNH6r%MAMq;)&VwjZ0Cpak+t)V zh9}aiUcUl(oT$czQ(}Dq%y!w?4c0OpSlP6~LZHAINL1 zL`-PtrS;Sqst|8C#x3hY6@o5QOVEYNtE{CAVzj+r4Xn$TX9>LZTdRtH zgE&%tWADeH`w{(v{mG+Bdxp5qtrDi+9FGfJGIlkX29+tYTVF((6sx=MY-IQV+%=Ym z!eB2B%5n1xPEDNW(R9^a93PE2$_Uo8bHA+fYEjRYlNryq+dkQ7(%ivLtU*HEgxtX8 z+&vzDJ8F0+#;?`mown29fq_+9gJ^mc@>R+AF0L8g!2#H2^5N3XgLg8SS7uh#x;#eM zp4*sR%FG9)F^Igo65p(-E+vi!8k>4;s^4AE;L5UV{KOHNat&~wh~StrTuUBV|_>6hIPyqR`uQ8$i&2ixJQU!=YAOnXfn}$X(>eIv>cqU03htiJIzHGK=GX&=c!s0ltZWPAW`WcgXk=VTqoN4_drAPT{mHB1aF>7{ z%>%4h%H6ueRUmlONazNY9}f(2=xLdb2b=aJEN67*1nJ{VkgZ-Masj|j z6GC~Q*xS4FREQ#X4A>uzj7z!3Z$9?gefMAki^!NafZ^kROo%+SE+o?n%bg;I;w=gmzf? z(pg~5R4tZ!A$-`xzhDdoC#}2jLTd?_C;qMy7Q!|l|ExU$^8hiPK-iYcIpw?%9edgF zwWcf`0KlAY0f2j3WGx_t&Ir(N$`cX?YR+bK0nT}EE^$Dv&(r%jQFfqx$W6Q~Jm64( zqc|wCSnj3iWlLMX7VZVe)7&r4p$!UHi>#yh)*$A9^Cmyk zVvT&LQ_w>Y{=JS1=LIIy{+j^cD)4IojdB3`oB1m~%%f$1Qy1_JQ|7>#IWl{F2w>vfsQlsbiauVR1Cx&eIu4TGkG1vxjNJd6kqe<;PAk&dvualKM2((y zIG2>Urr*W(@>!gA_6J`>eyy&?owr`khR*I&lWKnC+?$6`<7$8NNNAFHj^WB$DKb`2YnDf5NyLy)P1Q4gw^P)F; zT|2%S+&&{v65Q;Udf)R&nSf-!f?L_<9?RnWms!|p=U9-Hc|Z?EnQvr320Lar?Bmqh zQ#7t%$Nui>=&krVbyj`S?F6=Tox8h2kyT|jbU8{)+OcU0-m2bLV$S!T0+Bt%W~<1O zY&M<$+sYoPXiS7W>v1zDX&4l<&udV5CLok0dnWHuXDV%p*JJKJbB}v_ zc@E@4^piRRE~P*jTbz0r4VaR)pxu?EpgPd}ap4Gb`!vq*6Zfj&}zs zu13Dv9g*&nmrso)X8NQ))rz_97JU$^e(EcoX}6V4e6y|W90O3LbLpB4is~3&_S~W2 zdIDuql0D07*3OF-9~QM}mk4#vAb1^@Y&dB!FTxy9-<%MSq#N=N!ya$?V& zvIAj2Juf({x%gUTk3jMAMmcPTLzc9I*NV5xo@gHAy2yQ$NNy-7-Tl~!1(Z`m6uWI@ zU2a!@K+72c`(s0VVzi$gYMwa#X4w;lS)260{AcKMNX-vH>=xjg8q{iVs@IC|(o(ls z=-Bc*uOszyj7%TzCulC^2fv@5M`|&MeJfT0?rsF8aA)5#;Hb^r0(>t)?Zf;bP%Fi5 zi_>W~<)AU?i~(Ib)G?9tR1g$Y-vsVSibOOQ>D_w%=L5GZ&ULPHp67Y4bC$8fF+=!3+SJ{F|Km-96#>A7 zYc=Oa1J5W<0gC(PB>?aPL;rV{cUged=Hj@u`_EJD|Mh>agp642=ebA!=NJChvi{dX z{{Hi`1h5}$5Mu~It^PX$L4A3MZt-+zo*f+KWL?T&0gh(>dP==gRGS+MoZn~eder^T z>{ndotJse;|JGn-B|e*{8FY9(Ht#XIp{lDBD?}LIf$fp|5Z6abyb8FGM~%>|!{I)a z0l4sc_-(=il}qnL2sVKk7`^0r69}#lyX5fo&qE?M4&X!KL(24o6Z&ipERlkQ4g-9A zglIMi&DL=*0{F%&qS+3J`B9)LVcl>-4UlouHr*95E-K7CK%(rt3poMM@G}!QzkI@@ zzm;|dz~Q3EM0)_2=AsU!Uic&h?DOFjmxZctXMdh|0Wj(&5E0!9ozwX40E|cAR)IOy zs6rw5kKDaYTaBdG*`o;q+TERR|L49sU^kd*nex#{^`%;O1RYj zl>5yDJ{#C!pTPJL#qk1jdXqNA#U7PEuCokIle@n{5Q(fCt0plJWdic&Tk!Y0egNkmcUeX#g2O|6MvNl(O0RW4 zUV#hQ1rS6iuLMkSOA9>r>JZ;42=d;wqhyw>Zb=%5<joJ_@Y@zvOK36eELydeWsy0{3{kK7=P4L)m1L#Nx96mzMVzotrFOc6S4WDI zX$`HY>yB3od(uTG94nUJC;o_UgOHNHh^(P80F%?ZMDJMj)e}@QG#)r}s}lL6onD#E zBK?cQxu~Qk_R->=>v>Sbm?AJ-RQ>TPpVa=Sallk_Ty|xEYZB1~n0!m$f?amB8 z;)@R=;oxD#wYsFZHCHoY-}On}%`vvkgU+i4c@-JV&?^GNMkdN)MeQ~wa9sV#h8sR@ z;iXTS&xTmyOPQ^oSLbfJ%1hlZ?H=cs2Z!7Jtm;=I$h7x&M~v(N4WOSu^*eBT7FA+p zj+Zl!fn70`O;$k+62EQqXNDXfY?d-Oqx-2>AfBE_;e)x8-aVT6vtjaesjMGE4LCcbl&HjC~8ACdK8)K8rQnYDuob@rh=(8-w(pSz89ebTSd*0p= z7i&=bcJkfSC$&1-^;L{I&4SBfm6`vM!cD2&&-F)(4AAm+&egFBTb!&ij1oUy^gwYV z;q!deJmqLL-l&iL7jPnbG31W}*$k7J1e%H>=(1I5UU;W*Z;mD@wDH;(Qyc4 z7Jze%ZB zXzJ2^O0%X0T$T-aRO*VS0m4)ZxEY0OVO(i_mq{4=e3`NO9Z-hj5ElQuFd8m#<1rk3 zAtBrlIz&jC971kC*VTeVkAWv>q!1&0~hK2_fWiq46_4@nguI~b314J z3Z^vOdvEp5lbWW{x#wL7xyGQc+mypuOYJ6Mc$MEjq^9yCi`NbPonWVH`l6o>%)ZAEzX^oRm?QOaEL0s@#Eo7L1v|@!11B_Q7pj z?qC1LjDx*^LYn!Mi#~<0bZ9RLQ!nnhMe*^3fX0f_U-luxq- z&YII{w>p&HL;Ygn-O-ZFg_xat*=V6+s9V1v-C2v!Q`IL)yOLqtnjJ5c-^hIADk+Lo zVknv&&i4Lrl$-9qy1x7)lNRm=cz?j|?a;S+pK+EBtNy4zmR+nm_MUCJcO@PPoIm^+1-a4K}`l`^RFIq>X3)$IcQ*bB%32^fjjX|icalx-`8Q0u@a5-D+VZVT8b!P)_|H}bJ#|JIeAj=8`iOgN6se1hWvvJO zlpa;o4z}f{#~0x_!W>4jv8}<K5Qgkp7J{|o%Ec5jsxDea=V3LEi+2mRsZY7`9Z^4QALj{nDVeR;7k<|Ud;n+ zl+^yKCB1k1$A#1n6rU!rj=S}UEj?XW&yb_}O=t4vo&Uv|m(uAbYALO!ka#K83!Lh` znqTuLm|?exM$;}cLd4Zy8e`&CwOoHT^n4T5mbq>1`%_Co4#@)+6b1Z1<=7tPO=jjE-u_09BO|dJ>FpKskH?lk$afF;zCov zKa-oO$DybyI88uzZme{6CLQU(~) zBvII>DYX~-z+JsBicxHU8hFBht|+Nl8sXqJ@y6<-|X+Xm4Q8N{=P`1WC_Pzdq z9i&pDIN%h$zEI}J0hD~2D@y>yt~aCO|$_Az~R=5(E8M-)H3cAHB+TJL6YA z*Tg_7nlB!W;5oufA6ga;BE-+fY}3UGF$re=sC@uJ(oSHY*<d*&WOVXg{Mtjo8oq@RZltLk!21X= zE|sOqat>YX%=PRI!@1K7+kD!&_CqS*WMVqyL=!+Q@zuZ5Iz0irGWEk2i1%Tn2uCKZ zjR*9ht}K1e3uaT%BjwiOSsU}?Mbm)FoFQ;C8wj!Fm*Zp~NZo6xaLsgPj?y|CljqPM z&>(D+&$eaZJkVd!6%m}gEK&yT{md$tq=4dE2ZBRS+u`5iRkjJHp8dd?Y=$mcXgSgK zqhd1sLi%e&&mhmzn&JHe(lu>lVlsqyu*(~5}&$u{tq8mwP>i%UX=)IJJ@=<9} z94?66kKMkV4CqgnCY|7}_T7?m2A6WQ8U)2nlE)aHRH?%-yhVry9W=Wu9Hkswcl~#a zOqkh0={+W%)7lwm%|E|Xep{3F)xX-6)?qiiXUt>vV@o0H449N@4xaOYI{@+d!aFWh z_Wsc>(16WTK)ymS5Z9^ivrcY!f?O@Bbxe@b)FrB>66PIPm{48=FjEX<@~S*4D(`-W zA3=FVS&eD-(2;yu0*k*K^U3uKOnH!Qb4=pyZ}ge5txg=9V->lM_#Hrev3C|cgC_*? z%WzBkqwsA2LUfLaI~jLDYpbCONZg;2QBQ*oTuM#VAA{Hl+7th?W012!r5;@3LH*V* zasZ(tat@aJe;e{31+P0o>caeho-JP4SVY}~nCDuppO{NmoS=QvcgPr7w7Mizp{>Vn zTL0^hda!KlMA}Il2Tu33y)7ROi-J;CRjhdW5Bgy!GbF#KYv7H9uIOlj!#e}B76*BO zI6FEmcAhc!%aedVpQ=rDEdbf0Tj}&WxKzdK-{|JTva!15liB7<@b#kOjA%~^q4e?o z?22CL2{E0Gjo9b0N;0!3p9kKH3r1=aL0{WhG38uCA@JJ_6_rp^^PS^8+QlHydz18J8TRjkx2LrG(ezP;7WCm zu9$T;BO8?;tAF^T)BPh5;;+)0ype5P^4lL%VsMgpb#XAtFREOlJrI746)IY6#<}wC z7ozJ%^g1IdRq$f8!fz5mG%9-A^C-EPZ7j&SlTN=K>I3u&KaP6Q%^!_%P^5(gdG8p_ zFcl2E*50wdlYtfRn8^NI>loylm6c9z`%696SfJwSgeNkXX|4=wi_D=5cRd+!u`uWQ zywLwANKoi%dv0e*&34%4*>q-4gCM(?hG&&_Tj12-5L~a_?13By_u{Igc zy*Yt&E+qGHw4P7(nsPz>o)ESlU|&*%)s{^%8>cqRMM|XXEOK^qlbi_sB&to-lsP|~ zvo7UxJ#mf^mk6{&a+m?kP-Gs`wm(~cfR3f!>G^oi`>6Ozy+(stbhp#1fTAA@gYpO7 z(eVOBc@q)1{i*5zI?fW2vMOcVePuzz7b>B65LV9vZ*({PR)7c)SXN)t;>$zR%A1{z z0%#<)tV_BX@C!>luF?CvhxQk*CL~%#c~2*&M9fLCJ;}-30SeKRX-@F&$dwc`*qtmK zcez1^p?^iK@H(;A34Q=*ZfVo%Py0ezYUYygZnq@(Rk65o71_7k=y&dLp)Vs5X_iGAk@gp+ao zq(LUCp~3($%4d$P_|pvb11OK4Be&X@-6tw^J~ykwyesSziVRT#rZstfC;RW-mKCpX z78NtqQcsQlP+d@qO}0yygqQCGJ=jjH~Cq&#&dIh|Y z9kKHNc?(d{ihzxOgl}6Ki$I65@-@&uMTeg;ylfwfF2VVQhWP2pfcRi8EJalyCRVdB z#c`a&iL(gjH|J>{Vmd!ke7{4z(u)`UtH0i{Xm`~5tNwXs#(R&2L7fY!^8=#`XyK5S z|7O%BKp*F56^kn;>!^tX?>p)zas}FTBv69G7RinMdG7=W8?2s%DgJIkg(;pywX(cJ z=zl&g?Nh#b?CB}MUL$dakT)N9B|Tl%(uT5f7wWuA$hB71)7y|{UKM+*a+CC zXd1% zN7c~9sic~38GdJFqu|TYrW75RanaR+QHvQ%!d8;iWhGpFcLWlK))#Cv+-rw4zM=S~ z0&tC?^*&GnfxS8n=F!?n?Jb*`O^0SD**8#CLN&gK_F>NoDYlqdqbtR*#M?g>m z)q&M4OvddNr79q;R(-X$_e#5IFo<8K%Ar1MBlJumV)kp9pI^+P!m}Hvx%KWziS0J7 ziuzLNnJTS-I4dIghog~eVM?xwdT6OIfH22_0KPoq9Cs%`0JV&f=~Mv$k;1L-ul>^r z6u?c+p2VtKLsspnZThtz!A=%y=h&$)8`2A$0mEu!ondyHy_0t~jpV&tE+*??g3ujd zyZ0YM(h(Kf0EG>RS?mFK9$~*D6C!Pp?I;|9-bjo2N5m+}c7=xXL|V-J^RAg!`j$L@ z*?RTcZYG?0?T~zVtEE1s@5T$B2bCWh`Fl*N-$hahW%d_-7RLO36f z@EC=c-LO#4nQvslN}W#wF}#@GC?Iy5AQ~1?-?}0y3(D%B?F}Tw=hH{oo-rD zrDbTiziR6jAlzk|jw||0r12>{T533uFvN!!!?f4boXBSPff}D%ZU^n)5(aBOT zE`tlEtD1hFZ`WDxdPLrzcB7&(NU;4Y=DX=lGLGXg8J9!zo>$YBYiGb=$MJTj3ibz&4Uch_gXA-hwHt1KGjZmg!?$y zwi~QhIqXEyf!}CRn)vMK-|z!%lS^&?k}LL48^&{iFUqHUx7*B{Zy+Id2mO8qGH!0$ zy-0DZGmd1NE^O@-Udo3>au=U@&>?m4qt!TH@0*3o`Bdz?ZW#wit$Vrj*%(fJyFnNl z=92j`Wn$fSVT1bczz>r^lu(7=uyX5cTuamM_(P&7ZGc}=<_nk^`<+Z&)Q5^AHw;f$ zdu=i|C@sg?)hUJe&0Qqf9`yCxRcyy40E?~yI^*2MtLNXeW7T6N{XjI5YbeURS?8IP zqegEK6Bp-QT2c{JpQV?Inf5abU~sW2Ot-B%?KhDAc3>t`7}r2KFn18;>ohlO=6Igq zL5zSnz={r+s^Zg~&lfhJjskeW2? z?A)F^y%}csaF#h;xiiKjbj)+nQX0J0KvCTJpu_Kx{9kN)C;Hio*M3RBQH{Zwbx_V@ z-Dq_ybm?>Btis}0=l%j~W0ddd5sC4cWZ{A)+PrAW_fMXDaqLo9z{@#ks@}1EyOS!C z9ywf8em>&Dr}TLukozF<)0LtThiuu+Bw*IEp z6Gy*UyPDV{lp(3LK(|tV_VwIPqH5WJb`A(wt6P>BHXCNqjo+}A>ACC+%77_8f8KRT z$8AxKwZrejZLN)~MlbaKLQQ*wuB0Qm504Aj$)nMBFfpIjK%28vLHI*Q@p*MAapgq} zCL>ItsyJ3kN5iR*mbFyAFzNQSh)ZxGzqz*o$kixAlbr?!pPod#gYrTpqw;;jcg}o( zvZJ6Y53AnCJ}k|^shcR@(_;2dQ7w-4X|A5QiZ?ne%gYW@*}e}D%7kPAwqji6GZ$oT zA@>P<;AYj~nr=tT_Q9hgslRrFD}bVTTrT=q(V;O2tS*VBd8OZJ+$#9jJfHSPd+p`M z-UT|PvRswn*Nwq+380E4C#&&E=`f^Th~&K|{UiHmczH17?nh3r9>Yya^ zZM4vpKWbXN=M@v2)+RHR{o>t`=G;v3#jP9$Y_);*p=s35jc?>U0iZ9{$IX z+&4vu{fjZjn-nO1F*A|l2sq07ADg_i*p6s=3KzoOJ*)yl(sm~_;@6r%7e02mxFnz- z4YMKjk>ah;>+W&ghhZV6o=vp-GM8Ej`Ez}-Hb2@5%*iOkx8UBIzqFEctwI&=^}`4f z6#qF|UjYW;eV$fCYS0~oBj@jo8J6^yNf;i@QNt01D&e|5LYhDqaV4^i66q8#md_V~IuPYl;ul=2X5 z8$^r7xmF5qbhdq69N5N#0NO+R01NVw129%#K|pfTZy5f<241LyAn;A{*kgvRmVQ_y;tD>_wnT6 zOzST?Jbsc_^D5te{mmok0kg-Ihm7q{{i2xP)!hKH^8soZ2tdLPvrwdxX^JJJ^7TO(_!k`*zg&=L2oOnROe`U`;lWuw!6sB~h8>rDSyr>N9tBkzk zN_ytE7sQq!u%X_^%V_<8s8hrHngry{FP&iia?d#ic4)ga@`+DVUqVwJ5b&Iu@qJQu zK6F#&IgIH4hs|B|PIJ&+sRiyufYD=0jQ(RGcyO7TeSY`XY6%h?p#zqF?I*h75J&%aGg&tngTym_!& zTmz+f=4ghz`TCO0Pq%sg2fvv2ahJ86Sme5>A}*z=I>U+9&8L~2wzC}Z#EGsaBKwmR z3&~45p8hiV9)UzzuT>1WMt@5W6(}3VJ8qZv*W+$=Ujn<)nsM2Zl#VGGfJ(>AzEs`& z88h<6&Ej^ND%Z;@X|8p1CJAVku_O-z*Yns3WZ9<8%#6p87foJX?^E7&DUm83Tf0NUZqxLH~_m0c?(+W+Bv6A9B6(fGiPbhb3> z`Ps-9?&cKM1Q)tB`qD@)%C|Cyl=z-~cb>79Dt$%3<*FVPo#6fgb-l{^%3!>4_Mj8a zm^WFpUq1b;Jg6T0BZ8jZbI;++)ThWX5D}Am)buotg{6A6?wCyWjIBssGS!#drwxTsk0}KhhY#SyhT^=L!37k~_ zjWrB8ZR4GZ~`6iJ^1{IZPw z?Zuk`mBAwMBQL=k;->x&5T7Z}bsUFjX^?@-sXD7KAs)HEd%UI{lt4v02DX*#jFwtV7F4OhylOnt-}6SsA4 z@MbXMj0Ug^*1J>dx4u|wYC;q-DSdHIJ8x)hOkK_RN0v4YjyJ%nk-{*3F-8Bi?@1VJ zO~6)`ahqJM4+%WfHP`VEt!76{7}bJ?;Z!4Bj3Emug>*|SkA{;j{pmWAs|ZmLc>HH* zLtP-3F4OYn!>o4_NH@2ZufnZZ4XM1){q<=6relb($UqJh)U;G0ESxm>)@jUr_s9Ft zXBQh$xF^sJ{erxbT-ufHd>wys`Y9yk-o{r27XOAsf0Ob+hc$ZnfS!1>H!!8gmRKWm2dvg<)v)vN*7~FQyD}d%w=oJ zi(~gBiL9WcI2TB^*iBCw56YgoHjiIyrs`GJWoTIa7G#|Knz&>@DLK857hfIw&av&+ zAmdX3qi)2d$c>ZBmVl->#V!3aLuFiFNaK(`5RMFAtJZ8?6BD(;9QDkV!Q8=6>Id3* zFA3?Tpt8R=oZ68SAq}HMw=iD;SF}VIz`MtJ*Q2gzj<-JQr{E4Gw7p7afh@B=?oBajjs9jA%)MsjHNu*EiWBZZX^nVu;A<$5mzYTGtTW6UqLxiZ-n7F0AkuKCD$iOs=i>WK zOlSfnYHXzmGAw$AI;r#O{Qmfaj+c6&`vMl(62+oP!>RVo*a@X@n((n9w`rRZ9qOPR z&tXiI92P+ahBT3&iNO`m0{{_p(CdD>tXc4^A9myRF(^uq%p$3^tMCDKR(b3xbq2OD zE#|eU(N;$~sFUaONu?orr9I_eAvC?1*gr-tVC? z9!52P#G56=^(Rk@&{_OI~?mU0d`q8z=JzWU>V)w$LM?Xm#sfFGlzXtS57L)76 zt;-Wf>FY1v-=umFQ7}*5Is=^;ky{!|wAgw8+D&nd0 z22yyT%y^z(B9(-=fYaKrT;WIJ=;2J2>9vQAkbG9MEhC3FB5rrKvhGzD?J3BriD6$- z>D=Iw+0$maXrHQeOu4Ql5S|xuoPQlW2fY))h3^iRm0xfGqH7UR_u z_l}G`V@Ojk%Ee5Yq&Lyczy0ULz=V~kznk9xCYjB6(^(poOf_%P6v@ya`P3dp4IMS& zA1MTWZHIr;fEjZ*`;2D6m)Ml=es1fh?J!LA*l!+D1;zHehJ-U;L35LozQc6Axex26 zFv?#cPgDC;QQuaY`o4jCH~e{nKH0V0;A<(hWb&wMjZjqH+^rM6+AXw7BkfXO_ij@Br1ku^g^YLf)jgvdrkXci?u#l zs#+^1eC0Xa#bK|{eb}^mj631Rejd=kQQltTUeQ>6a_a8%H*>hH^ow%66b}r2X_)Ym zWIO{ooOTxrK5`ADS~BW%=zmqD0rA~WS(?YKCvl6GS4yvyZpL&m(=sFbD7Dl(s1_Vd z+9(8@YcmqJ+tVIw(Zkdu404@lhtcshaY|;S+vz5!31lbV`UJ_&i0wtGWrPVxF(p(D z&ihq;8`MRVz8#2B9k`@i3jYfL5@M&z14lKqT~orK4WjX+Z@WU&uwmXBmKNGsNT3*Dw=62eP{EC z<4U^A((D^F-%sZZ#(NxdA+O>csU176CF?`19gT>#OA8WhqJ`#&ufbJH~}?ySR$2l8uj(l&e{*;Fm3 zhQjA)!}1ovD*$Suat1VTp)8`b9Wsuti#M{)rlu>J`{adg@e|`-EIte>7H&#GPa$pM z`&LNk?nSC16wm7%eA-N6PKZ{Z`3~B_+zESZz`^m?-+YT%myrXk@%hK1?ze}6AW8dO zQWe@c1-};Ri^O%5{P$^!n!`4Y56>G$(mX^JQGel_(1H^zs2eig9h;2u7197~)6EkqHs~ zKi^`84)tEyXmvxJ{sbTP9vx~5cFcvj@MSOt`D-m>_yId=`q%r&=U_j9qz333A--~3 zT$n=>xa^vHhjXedY3jrS#FJVIq`a7rN=tk+!SnZ!B$xNb825fzKi)n23DJKU!*KOK zO45VRqLtyiHCLb@Mt0`vVtM;mog?b5DR~C={henWJ<;X&Vh(v=AU89A+*-WUn;gar z?20Y1AVaF+XzflQW__Zi`1Rm9(3k4j>5}7VuAgFr0=zta9W}4nmO+pU=gtGcvnl?x zSoNv~%jL0q&pQ73Vt6c5L)mTcGmkA^RYcn*)~m74MGuCve?s=Vvy(9E zrETLBGs;_8pkC5*Za|32_fG0L7`QA>YW#|Hq^|;fLRk;GuE}=ntA7`V#Ss+dx(Lq4 z`H}4o`AD)j&~_QlQ~TJ_Oepy8?egXtB`VGSBpwaEj*h?0mVd+P zTC#Sya6hT!3RBvL>rehv^DM8%OMT`+I-C4OqtYWh>8x=XDc8hEeqdjHu6U&6Msis+ z(BQL>JZcbp6YNHB?%Z<|tYb+R>I!=5?H8|$&oXA~TN8N$@pBrx3T&A+gdj>=+ZP-> zYYJjlg zgA{TTe#ce!JH{#OnHw`r+SES-OLH=CvFSLmti|?zR?+Ga74_KAPeQm&Joy0c=vFU# z2Pr_7>@Wq!M&JZg5!XfPXdK^Tw0kYf42VketAJv3Bgr~(W8OI#r2IF7GA|#~A0d9^ z{|QO`7;=4sb)(BY-HwI7o(fNO)El3uS#>j2O55%nQlLC@(ghcS?=P)coZON}1(pcp z5EFDybdjV~%#sB8UVYdrFd<we*Im}sWM16Mz~0lG2JsQ@5mn*bXf7?PL5X;;ihbf} zq5U(UPk?lAS}JQC(E8x{y1pmw;9-xLPvq+Q$RsD2SUI`7m;$v|yS?#ZxUmo*Iz~UX$Cq1$zaErm%;{J830Zx56`gZKH8JsihxJQ* zK)P;A01(_!2FS{)d3e4F=dzxY4|VlGXpE`l14Znj&6gjF`cXgi3!sg`zAToaSHOLO zt;1dX1WQ9*pz5RbL|Pk98sA}36i3I78X-Ni$&Q}L_uW=&LMJ|G^zKNfDiAiUH9DMR z-p?_ToSan-b*=K;9dZPdf^S^rCkb;RNMUeJUEQzyj2#F#~B z0kC>80PM2=Fx(5MZB+j7y-6~W;u4G#19e)Mn z)0k5v{0>78r$BSAJM~{$hA8j$^E$s)?SMl(HfI zy|8@ksTW?Zg?ry72?WUsCdAmGJENa&U8DwyUySwj&Cm@#sdt2Vh)sGMMSZ~9E(Q}^ zNzOr_y0B*yh7N5*Cu`dph)HQSXkMRD4%o6PXOiRIWtfiS4$XhhQPle#2suo{{u^6e zlZ|C)yvHy^)?fd9KyF!7fTO&s-j5SEQu5e1>jJez>*y+LWMjgRL${Q4cWK&QMpUe> za8mCdK51`Tpq;PT-bAngGs&>$4ejYvx0}==wFBkJ2-sJKP<;mjMc}BO+}^`2x>QW|46}hats$F{xlqW(?9vAgszmz4OdoDWF2s zs6J*l*~D2mk4G80hPrOn2*A2)5u^hhKOs{HPT*%g6;S>Q?FeLREpWm17AwNuf8EDj z`Nv?rof)pkA;tc>WgU0=l_P}vpd|b~1`SF6R!o6jGz|xSIbJTdm1fyaJXDVe_C)1uJYXR_sdM-OQqBvx<3uq4Y4=O7eozRK~pn#?cDJ&xGMe zB>+FaDII_o^0#k)oEDn!a@ioP?0!ew&*g4qDzx{YfUKz=`ac8gY5yIdi7&}teUfA%yB||*(`wEipPL%@$^phcY|hz>1@cl*dlmoPxgrQ=#D=V?#LfXk+b6> zBKYJC$k~Wak~Hm8$R#q1oPBtNpS_`p$wTRPh0gwJJ%r;bW@F&W=u3W>A+X0cEty`v zbE04>^@TeMGo771SAO~VGtEAKc8fZme`#f`^|dXdw{4BB$SdHTwC$gO4Vl{M*ebq$ z?eXxq34uvA2(QJDJ%ue?RNvWW<~#qy)q(1535wy){l)_PA=j7b>m%&-N3xJ+ncZv8 zPD6o6@I?P=Da!e3(OaTiV8;ujRDda)!nbXe5CembF=v=C#h3I}p4O5+|4O$v#(^+= zOSgAVb?oRY=q~1i&vbt&svi9Cr_6O-eav7B6r@{T7mrxa`#yjd9|vnG0b!)BmZ_#5 zXiwL4;+e-D<(Fg?fQdHB63y)Tv=}>_A+}JdmN{Z*ADf)22sO5T3BX=nhZtFY<0l8~ zRKQ9ek;A$fJg?94{)C)Y@yd$<`cxFis^o-4YH=#?JnQ?yFy?-`Cu>(ne;eZRgU_%cJOup1uDK zsOpcPX@kq)2igEsKO!Q`V_YAyCZcc&kOJ(QJc^!CN;o`TPo?s{RlAg>K<~p3!czf7 z`QL&de0JO8Dg{OPNVJIbS~;JYlYdhDz3zO2*7%`ZOIq9%kE+$^q+LOv0U<;m8ycSh ziKbn;Frz__3?k&Vs|s`-ltug*ePE+y`d4& zm+C+yJ`M)!2_kBo3-dAu%8kCj1%b2$g9;=MAyD?g)?E6V@eRuLFLE_2Os$505$AQ0 zBBw{yUu5@_7`^MAF)_4Q!9ITfC)=9OnU#2le5mv6!7MM7UQ^#uQV9ya=|q z#c)>v!nYhX95JynS5@ad-*?#u@jVi1{1l<~-?;%RU|F)U7S;$^>;k-Nm#L|#^On_< zc)+;#ARp;GxxyWqC?55hFtd&=4Tr(U1ik0+CZv`1$yN|Ue)V(%UZJ_NVpVfa%GETv zt2MSd8aWcSNSez=`g%%k2?>Lxot<{WIR%`iVs6qI0RjsPc>!VH{)Ou;%$LDCOlMCK zueXvwc6vKWReGZ=*U~<2%iqn^?J-t}i9~Ky^dz>i8=m=arPx z-!?9a#?$*NV;^s0Ai@OeMnFzn zuWTiW0@u#=ie`%vGayI&ivSZkfC*mD0vvWN@0Wi)cDH}0h%O=Ev7kM__)$s7^*G@e zklJ)N+sHeid=0&t{^QT5PZ9h=f&m5Vp^Z?5Vf;4{X}*hXP2;9qOr_^c8gzN95el6w zClK}%_lc8dKwC+vA=K!R&@?*W392dX6D6DGc%vLc`1`NCFDa%NxkK%HatJQA@Lv}@ zzI?DAyZ8Optm4ZE#;@e&PK{Ahr=B`Pa7amx|LzZ8rH|hXU!`TrRWA>cRER3+L6+Qh zxR_ts9o78Zbs#Meo6wt5p>1$lM5XX#Op)R6dEU3}EI)bA?zh0C{ksPyz@**&gh}=O zUg+{rqxIGsZ)qcbe$c7CdP|uIoL=^2vG`%suKpLTFX+nLnr54!W3TRnLLy$l6EX9}`3T$3Fl~MzdEm;>YX$TG=xhX+^S?n+kpJMKQ-q`8 z(x2w5ZuUq|{xGC_NcQ&rNHznn{diCDiSM-U1rUQuKcsq9-|JP+_dVTRp%*qG67DJy zF%_9%K zID_H-(97^4PQK$o2#|@$9>5#XPl+as^BzizJ5Mj&*M|i9ATs6-_#RP){N36&B4BGR z4}c`XU{u=OOW^R9?n1Opcg?=iwID9s1L^?c+fD9dhMT>>8SX;~DrEW}4Qlr{xdd}N)_ygm7|6>BZ2QiC_8oUV7-6fh;oRn?p}5- zDD&7mqa)Q8(aSg%>Fv>$VVhHwyTHJk?*}HYdE?%iC${Mg;Q%%>2+&U}U(JtpH!%h; z)lIKbZeoIOOhPb*n@n*Y?!6O*O`{FBku+Dp3MW7|aJTl z(5}HLd{!ZKE(%b7%ScC5ndoVK&3k~*mCO3g8i-$>IY+5rm1q`hFHnB$8x6M(;NTx;>nj+YQ#UPF%bWyWAaJ(@n^0Sw?iylu(`Ns%s%tv~cAbk*viaV1U@ z-ZhfI&3INS*~0s8YB^2D{e-BOT`g@tqJi;0)&S3e#0m>G@5pOKzW)kn@{+QwY?tD$ zF5|Q=klGl(1JM0EA;&A>_gVR{*UO~~iGT{s?bTI=Wgzj^7td6c@q4PaD&z4<@^l7t zeq>u-?7(k66QpYdR>DN#fbxe$90u1JNjf$wpNzv9ECZ zv9jO3G=7cljq<7bO}z#7&>n#EjaM6xe5+4h4S9^Qn#3HO4#(AJGqWOS@WQ zJy$c=ui8&L8DL@?aswP z4{up<3bIg~wF5pGr3RKuMvB&>pXQkfM2xFCi2}gU!}-?^1B=8AK&qYV;^#O4Oyu)@@7KIsrhj@OoZFt4P(Lv(b`@OtC z+(=Jb-5_d(t>>3!dTrdfzr2#SJKjhM2(<&wJ6ye?EiG-uCU6rvhZrVPM*-%$R0|`f zWP}{iniQbxq3|F~SFw@l=C@GS+f4wxm(aXMW}%q5bnyNw@H2dA#MdmT(QGi#n&R<8 zF*?OLii%aBGT-um|655=qQxCxoMR^h1o zV>Yg~lQ*K)Rv44mKox6C6$%U*+sH);GGRZl>P)>x-pHa!#(uUtnQO6`QqkA&H?X{bNxQ?|S?rc7hF zUKvQ~xgED7$lOI;qkHJ41FPxay z#YKM9=2uwYs&!twAF2xbCcC1L-`IC^)4H2=rZ);!8?5#d4?j*(kfA|_uILE>P_#G+ zz+ja0TMO-wVC*U!^GWzEZZ zE_%X_ea5=eiDt$ubxRxU-0A&kI-8>BpRgz+V_tTCTK@MU9+Q2xXsgzZuSq2?$|HQF zB>3gA1(96zBmJgayTlV!qozt-D$T(J`Y9$#(02_quhWI}PX_5fiy)X^a zP%0*=vS<+BfHv*y#+|vE<-Rv2k3E!yS*zk5v!`^mB+G9z;?Fod=g*pBC7Q+PLZSs^n zaI@IF4|$jRu4=g5^FZ%=iy-G<$<`fx&MhUjow_CcUTSJm$;6GMRDgkEh z+zh`>p*;DhOE0shrS(645cu$MR(!&e=vRT92tzGz(sF~w6agPbMGGDzL?H*T?*a- z|FQbVH26}>0%IFx>90bqIS5l}?9?h($n*|hyOVHSSgH2h zRaF!Y|2zaa3jh}ulLpM6=6#SRp9<+SzO2Yi7CSS*%F5y;W|kiX>a{4OxfJ{O z`C{k;6_2;7pyh;$JYdL_S*MLIa+^+{M4Mf3z5exGM6)W}PtCe8dl9)h_1&Z;1hT&Y zAj+#O&Uasf(xpeJ`@c!1t`3rnt1BXT(mF<$lM#%&IP2SB`I22oOV@b_V;W54r@wl~ zQwcJRFIonSo}4fiUY@JR$t>dL#_neaD{N(UFAhde_Hi~PN=Ynp(&A!U>jSZHwwa~R z?e+3Wq(pQn`q`4&F26nwI*argXRVo6sCI-s9XVns&}3*tdNcJvhcZc@U&yA9Z&3zo z>ci#YS+UOAj0OXbw~U|;f5qM8JuVQb5PpAe zn{+49ajLqh_Eb~x^5_}YY`Z)2^78QXcQl7eADsjS( zBiS{E1sl;tPnyU%osYq}03hS_PzAZ0Q>Pp4!PH85I+mPsSdbmNP`gwLR-fo@hh|i< zb@WGBb?Q0v4DG1v&l&ygk)h5QpX%qf3R<@?tu7+hm+lsQ0!)b2+@;!nz$Bv2NcL9O zo72y`BhtrduxKab)rdkq{i0C+lg8s2_5)h^3Ii-YUO?yJ-WX_F!vz*6|8I)}#Ys`` zTR&d^ruhbw;XbKW#e0F+{R-B*%pQR!8GetcCM(h#?+!$bKdYOGml-O-{#aH}r(`@( zZ35%VF;`kmd=^uh-f;G|tzZ`f@PW@o{K41P8{kYAe6g*qiIst~Ui2_IhX&d8e$jPO zSQ>SpZeJ=lr+ZP@>zjOaug!Z-OY!5#p~L2Q;g7VkHVz-L3omY}3TNuel6+QhiA}Ql z)5fC0^nIKSnwImMrScHBOl3daUJ^?Po=&%$VLCbeym(VIPsU!etm4C9A_4YxoIL&6 zN(jl5EO_~%z3T;?#F0Eo$rrbUXi|XLD*$F^+-Pmux~a=;t;eyGc&eI5h71Y>_yJxc zCm-WrlG)QWj-cBXk#g|!+Xt?+0Wr7NMmM6-l~kQ79U8eVEn~^ma^1Mw!$?~r!0pv_e}+CoTl}cUns1<} zi?%Z>WkOo+QcZdG)^6OQ6QeBCb|=}!Y)Z-Mfq(c4?_@*o)!zOfRheGe=y3JJXt6V| z(x2(}PAR3^$pO}BagxpF%OOUD7bt6O(KW$zK1CkYzV8m+$lN^c3qP^n8|ggk*S%-E z?SBRz{8GR70q5mqHerT{*snxDa$#`nkZ?vSO5iUdap}3p9SOrQzX+6Sd~?wrSqfQO z-U9fa-=F2)eT*@}s$WAsKRUL|*decEiP&Jpbg*6KPGugG<~<&%C~{{1`FS6AgQm@T zTI~_8_jr}3&Qxc&;q|PuT5yK72*2JV;TJdnkJMBM?QI3R&*=~)_($G9*hsWASGv?E z&b+uhBfw$iNgjF}Gsk5sxH%xGMVT_+R?BdL-jgNgoI!?F5lkY=AF;vsRw(TSOtP&U zONhu=2)v2=4@#~7gHlss2|0rHKI4%&eJer50P$>_CJe18p)^C?rDr;Kdv9U8-X03erEYhKiw$l9xf zg;r8G9$GF{IP`vw6#n+ixTU&3NjC;iP%NtWQn@$l3S40`0xB_~eE{tU|DwE=Jp%i< zrj$@hBFb{3pNS*DYzO>snnx18VDFN+lh z-Ds`iGhIA5R4JG(suQ8dG}Nq&(1S3ER%P*rC{w?(95Ed^P|*G2sFpa6V=-KXN0G(^ z2vy@6JFiCu0K+tR&u&oqD7kcIx)-F(*}eCPGLpsZ+hG3XZFCo>UpMG~BzyOg9X+}M zzvbHu=!8>|hY5DXm#kOi-0K$|yke2v*dt?`wKyb`?yJqbhUfg1_0HxpMn@dq#EY%Q z&`8jKE&6CCae7mD9(57#3W|xzQL}WOt^k}bD4tdnObRI^*bSCPJdVWIwLCM8e=fJ!Qo*p-YeEy%B3>h~;jObbFV|jd-M9 z*m!`fb@R4(u+#N<-q(!WRHeHE5rE$VIl+*SP5!Wrr*}Q9Sv%d!te;fY)g;T_{+YFi z?7C?3VWy1(C;Os?m#fF@W@V@cOvltJ$c`-LIg1zofR>AKQJ{DnyGLGB4OcfHg75Cg ze$uO~dw4hoK-hl#`Bw;pxt>3uLf%c7OxOLoEAnt+SWrtw)xN+MZYOqaUAgNjGrO6t zy0voZ(%x!zlSrkDxCdKCaE$h34z@v8`|y?a{;A#7kpPL?xut6L2?`Q_?_O!pSZ73;^pMp=-&^HREUL$Xe&jJ|)pTxogGp&Z_dQ@!ilcY3~J3Ra=f z;&cen_43`7mmcqE^7!e(d25|2gv1aFDR5JG*anI(g2MmC{o&9;Fmv2I+hh|fOGLPc zLhXoG49r>E1YJfnuL~y}2790Gm=^mm^M{g3yK=RBKkmQ_xe|3Lz{lm-PZSKz-N06v zB?MhK)fGM6)LWqDnZNY!w8f7n$WFEQaJe#Rb%Z6ml_K?k8znHPh1I2!x%Cc5rEUr|EkmqdI60yy(Q zUYoVwLo7_gPK|*`bWYk%xW&dR4m2}7_=6M{zRVw5y^*BTu}5noPvIsaykh_+^tX#! ze#%X3E4pR#qWA?MB-A7{na}fX%nmu+1D(aX?7?2sWy~aIet&F0ahB{#^8_5v;+YFg zP-U2gui!Va_^FSupIl~89bEnX@Wl+`Tt=bpeF4#SB+lNlVT56X2T(dRt4!6Ml1c2l zzoC#%PF=}kfJ`DA{f|B_qCo3zFY%3VSMN>0jE%WITbRQYO!`D)>XCe*4OKIGkEu4$ zK^nIH*1lloxni{UxI!1dnv7zja-W)m{v6-SVaDd*u54qa!W~2%JhQNhO0UXYg)yA^ z14qc<*ViCp0wq~U`;ohkcM?5cf-?Ulkwy|S)LHZ*4e%EKRRG-lz^l`G$WZFGg$D_K z`OlBww@(VLg`rQq?&Q!4V#(N`NUNkCt>txO>2%oJd%B`qTpVJ`F-h%cmCEt`<0S4x z9vk+yIT-l!YFwZH7Z;%EZ#U}eu%!qu*@{FswvJv?`3r z1J|q;SdkHhcZtQbV@mRV(Ih*rp?tlTPf;eSjv%oRysW7Ut17XuSSsBEU`ObAw-RVa zbVbke1}1mF&6>>LU?b_4zRLlOr3%dROh3@>OxwLgs*3l9iV%Rjjo^Rm_XQ1O)HB}| zd>9vU^VgGbEW4b4m`cQ+2kubOwa!hQ@ncvNcZ_#!<1YSol?gxp(RsTneBNF5~sMNUdeV zzUC`Ljl-4#xo7B+66lX9v)r0RDq?8Y;4NZk%-|MQR9 zwQozVv_S&oxDZmSh2_ytkB4i?Y^(E$eVVJ$E+O zm92bq#<84c=N{C9CFsM&G@O+CLt6nozA1r)+irGFC(j!ayncM0@hnwq6C})6npd4UTz^K_lO(!sK{Y3d_9WivpnjyAU zX*NJnX1j3j!8&&YrhDXJM{co#IL&vhu@^1}Q=E?xWQH5G;||@-PZ)0$RR?%{D-yPx z?*{(eMBi-(>%7pkdYojJ;BO7!pVABPS2w?s1s zO$-+RC*&AOj%mo(s_A&aWUdI=JeV%HdoToxE`!r)ZJ{=w>IY-!hKJviuaLy#d$CqB zU;Z+Ks?c09{-G~)b?ENg$?lS;=RROd=1O9bLt@Pg1Qp+Prq&A&Q!GO}S_P{TnTLlg zQj*=T&k^mp<;%%vdc)EulXOvR$DgY`Dw)xW@ZeSgY%V?6!_D@JbruJy1^$ekWCn=M zZi|{p2V)xcUhbN^{d#A+*hL7N72quKAUrEK1q3(qs#B+zbrv6-xPQi;kLIUp2>7W7 z+3_di*Dc>zugboQFOfYRsrI7Ql(6MxC@dZZsteJa+DOsP_@DZVM6G?8D+AMtWGPId z-?JTmBr6^#q($_wr)W%ll6lBndY(9N^3(goh0pGz?tFGk0$X$Z{g8_i1LD4@L_KkYDqAc>=9Ua_uyE>RXSSKGaO7kj%7)}(*W83}n z6}m%b3>B-UCCSdc^tE6C;l?!5R}HRWyVv_^n#~h-t_lOC_far(L3938+h^=~u#-zs z09Zz;=%V;P-+JkR7Ed7;L3tnTbolEFR8#^m74gC^1}kp3RlU(k${x;UWySw>I^qWM@ELInd7uq+>y-C)YWbJJM?(&*&@s? zfV)24Kuw7I)aI@xqjuMO`>|Ilh1f7#rQtAO7}&^MRWdaCVLp)U0pgGypc+Ou->Ux& ziR_5;r6{V)phyPjYEg5CVKAY99<2r;hNBSc0kYI_$2ePA?& z0wj*YVLY-(KyWKm^H1y{!P&cfsuV#Hc)=d2T9OJ`sVmn#&m(uVA~EiG<=D`9a6^@% zI(kucu$N6yI3y)DP-ljeC0N(5g@X)!ulY#@iF7sVbL5S{d0zb+hd(!ITmqP#hlrZY ztui9b&w&IgOmf?9WN1^XK$BINqCK=e-0 zq(BYp_rpO_Wa!K~`!K;Rf$!UU^@#~jAMItkcgYtp7&+bm29uP}0L?4-lShRDTG7v# zT3|-J`~$dK<+iNt?iB&daizBayj*yzJBt53!ER%`H3C5lDl(PYS!c07H7$nD!Hk32 zPGZvP&ErXziB-`lD6_ofGAQIk3bff9lXnQwriC)* zpKOd97{5^I?TBAoG$a@I^3b2R4P~P3L|u|vwnF=Cfx~fta~B2C=)@eFGm%(d-`d zSkRcv#doS>$?RaNY-Kn5jZI769Xmorv9j!X8IBE-BWXYkFK+&sok973{?HFY9f?1r zbPT=%ihmt74vKo{sBQ2eD{Hpn&^P<~fN(&}3)~^B(**zUiTL{x<=gO56FL`}>}kcg ze-4_FAmX4oCiyTG7zIWtLdtX0MOG=qk=N@%dw%q>+N1Op!np|n;^K>lSrwwYH}Bx*0RV? z>(DGYK9Dg`$vaf!k=-=bH^8!bA?oFT7nmGmRZ|;=CF89Wf9^K)So-m##T$xKa@#0f zdqTi5g|mT}fWZf^rvy{hL~fdLz+m%@R)!ygH5!8*{#zil5TP@#dHQ35KpqIH6jA2( zjLGO#fSb$p3yjP>H`ul<%*Q7=6#wJy)>;uUX4ZVcB=gt;YM3*E!vpmB8qFqKyFd=0 z(AJ@$QZtP#=}_s|=^@60i$y$edU7V8fKfqN%CUX!KSzNKItqVRA|nV)WFHfdtzWr$ zWsm1S|LqsWa9nq*_&XupKcuDQu-m%afh2B8a!4E$Jx0q17+@=J|9oHp9GeTK&_vM4 z-rNAh(K@}1{5vJTsQFWIV*?S#eVP!8I1AN`&~YCH-8{%|2`2Y<2?4)lMO8Z%w4|lW)z$t{82idtoeoI7c)eQ1m{DJBvvv}|h zJk0s+C6kZu8F14-=~OX5hl;%_r3$2zY`<)E#A)Y80rQJgGNy|@1mS^l1knVRn-lEp z;~Cbmp#wSBO(@_0^TTiYr_|K^7I-x;ybJ-h38qL_5A0U6-!THn?cjvQU3*ggJ z>aJ2*A$-_t;KP99g?!lBEpiRJG}4}%UD7*mEbc*UK(mCqspaSQ|c09o%+*B{#jRM}~1@v5=5F4^q~;*9A!MbIY{ zF(r*6Mc6wn!roggN{Q76=+oi`%V*d>39E}pH@le*wgxVr8E(G%T!}9Mxq;X~@F^+k zu|tACy2l5V8ELQ&=wsS3OA6IO9KI0Oas}pgQu^HOSE*%`BhF7P1^ddQkc-QVM^h5X z?z17U^mFn45sf|ZjiQ8Rtoj~EG|Qa!5M7!42MA@@{9%|R2+wx_JfDRg2xb14hW?|* z@^1x{-{E;#zXie7XbwT7wd*5sQN&?7D2YUhfRDF7I|O4d=0Tu3!#n-36S2t^Dg@{h z>VCl$UGXvcx@?lBw}6gh))sR4DF}8?p95Gm1 zS5QJLz*{LyBqVVr`zNkVo4D*GcHttkbKrdkiV;L{8BBx}LvBS%#Ev) zEV|d>OueVDFXO3(M(6W(f59RKU+{NxF_-a2Wzv*gyQ#%Z(aRPfHZJc-fr~JJ8B^yO zi$clop7G=ClVdAr5%>8@a5;c2WlTp*`HlTf4=h%q1)^7?HTTpFvJ?Cbatlb3??ABA zYVLVW2>HwxTuB-q!Yi43?e>p5U>slRQu~ye6YPao(al*Q9GFLqty>yG*@leliZ(qN zd%sbtIu#GzCh*mzI=D)y7q@vf)JP13mKAi!I2Ae+oKV$?mlP_B!yEOiLO%8_1WA`qQ>M4t*5Fr%@#|c?rQ$Ika0Z0D?n)!% zMEX}wUz#s_-3ZI{P<6GjQ_POB4YGXe$Kq+X}b-h!NQiNVs73JI=4 z1yhDUpRz^=?xoUC^MdoQ?XmU6{GQGddtq5_^ZfVXfs|IZlyL7yEk0|S8xV}*-}pJt zTN?DqI~V_ZSKCYa)C)HQLb%$J(8Q#aT4gIeZS=IB|0F7i_q05U-`|E7)ruCt&JrRO&~|2!<+KnYOw< zb5$f*B4-Do{#&H|*!N<}m2c%jrE@ONYx@*K83)<|tb-c&*fW#;uAVQ7kg4x^FdvwY zBt7iKfuiZn`qkopRM!(@G4(<0k>$N_(I&+vZC~%eiw)~I(*cM6gT2dZJioD>U^_Og z!bW=yTWTKNP@<)IUfihY@T_^0Oza&>@^y8UUK^VVkqU;TZsUw;1r=S5==g*o3n^JA z?9~-Z1my4pzH)JG+@ixz{nB;M<%eJb%@{GYa(uh#SWi9!@p~6T?G1m64@O(jCBgW0 z@04TqsWwXCrY+TE?7Lh=tprQ+%QA)V-#%Z^$0)4%u_zK)Mrb+YnI^-i43E zc^~ha&v(#t`0Bk3BVfH9E^Iv2=eRWWJ0;ek!BsUd0u&XMM@(LDDQk4p{UkneAqk5^ z$3!V&xC5=JO7|%{*Txt&k9)5yIp4RSBdTF`hB1eAIzXTP>d2VHPo~Vp&eeaxZ&X(uvADM_E?hHvdX9qv6iT&b2jcJZ zFL(U**lrM6;NT)P={z*lZstp1J~jt!A&;A5s)3LWn?Jz(AmgMj?MP+d!nErlo`1|2 z`LW<7`9s;o<3ofbUjCT z=1vsRjJ;88{BY{jjj(xM?F{X%4_zU)*q~UpLaoEfj0{5Fs_5}fCNr$zSMH7`GC$eN zXj>C5l`YdxdI{%`?s`MO^v&i?2t2_{8M+>ulMDx`bL1xEyC!{U==+wMAoZ}FZ;Rjk z$wwC!^Wj&+T^7Rd-80PJrV5m<|U zMNY*E-QQT7?EID{2k}|C-5lu>F2awbKz`(_2cHIhH;L&$w}yqM{Jjz*8Ta3d;L-rm zJ=go2Q>6^W=_}qe#)+yb?A-ik;8k;)KN0dF#NvMG^04p|CWw$P13uTd{`ejp7He^t zYK-n?5~a)+dE2?`cV~55Q4$3)We%@4a3VO^llk z{iczbb~<7JV#8(t?37!4CZ}k1%h7@2-@94>2qELYEn+5%)J3@Zu{fh!q=W<0xG+%c zN)F0cC!QfjwF0ZN5{yyD$G25G>&(I{)5rN!O2+uzW2f(slK#>_gR-(DTO|!$h!;rY zsR|+KT>NrzJJYiE|^3I}3K{$$j z=5Ha)T0x6`AH~S|k@0lUb9txFP0X;P`8|k6ztSJn3%FmCx(s$>L34zuk?%}oASgcg zHZG@9W}=4NUK@dIIT%vy#=T-uFtAX~C( zUovq8?faW1Vc36jW!G2I)no<$BIefL-e-6ObR11J<>81#;Z!!>W$-o!7;-%J05}0ukU+T)?Qf366L8s$S5DBy z^#Et3rTk%zIGzN^A(=oY=*2fOT(+G=@zleI1$Uro>E^fH&t$rw&b@tV{$Bkp@bmdB ze@Y12YYHVR4?pV01ldGr^JX-~<3i9BQyI*|OW@mokpg+UD(zRR)NjFdvb~6v2~^8Z$9wKzsJQ3M2xPXJHH(lLDAwjg=uOG zTn4nSQs)e1t`MaI_pyIY%yV(>mGIxJ_yIG&(@8EsX)o%&`Wk$m_v33r_B@RFR}ree zk|{_`RZ<`5DwvA4_=DWPeH()RDYQ_^cBY_2z>8U1ZDrWUfdp;;4HJ6NS6RaS1R1Do zLAUI2{V>!{cFYX?Ysr8A>jLOsFRq=sv*EQe9%(APMhexFW#DxF?195`i*? zOV6#BoG?;+8*{DfH~G!%on(D4$7cF1f@-94>n9o!_`9;3Jp?gpUAF~o+#yr?<3d_2Bxr`gF9a~-9-{gr-9({AIEP#VU*<^JVKak>A6B`d z0;Gt&2!qPh|7R~rlW6Fi^(#5AqQrrxxbDcHp{h{SN$DZ_z?vVKqU;AGgNUhWTe4!LwLi$D_SHZ-$y`E^|0q)(s%-m1@kV#`7J}v z@BguK^>F58RFq}NzW3u9$Pt-;_YE{335kCy7Ck-!cvke!Fm~8!tiZck$A_NK&4S;N zlxqS+3bD}qKzM>Kid6_fZh!f>)$q5FjB((&Bw)ON6>OiohhuRl0AY zyS+eEs3Ho9iu=t!UWm&Y1f=Z-&2A;tqO}z z;Ax}n1Mt?M+!>YE$-+T5ci*!drIgjp%DHR;B>~#K+R*~7A>W(SkDaG&Z-R$H#~to2 z7XEWD028x8f)zc&KZfD@0fu`(gT!|+6tI~;@u2#RH0NEshmioCCAG5-AnN2od{f|M zDy2YbH)b}ClFuEy;Gr#rBtFYK;@9BalvVI}CZYG6gq;W~;X!b9-f`$Q`s$Q*V-|S$BveUcS6Z{tt3DnEu3J7h7*ZYruOTV#FZX1EXHNv3 ztm8>BT=4(pH^Cs=z6 z1maLVrQox`(;BW;OcP592@PGVob$>AkA%{5CAS_CD2vdRETnnt0qj(~f7j+a@^GWK zH*Nobj|3Om`}_Q|k7F&G3BkzD2T_%idJ>=*VnwLo<8+%qIkydG9~t>0v=$+rzXb-Z z#3xv~df+j|FT2U;u%?1?myCi#T1<}-S=1r8AgA0#h%YI{GL?Wy>GKO^(T`7`K4lgc7cXhEoT?azRu+4*lIqUB z>rZ5TenzCOP1~Edui?N$gU9&CpHXB(BW2l4opZ+_-0;~Fy?B>9zz4wHUn%Y&d;lG| zVFlb)!v|&CfNU~%7(TcJYlLR?J6UKez-c|U)Xr% zb_SmFTkB6Tn53cgQK&BF1%)34sPOYx+1qBD;jtDExN$?(U5?cC4itRwmtXio8HGf# zEe@hHOSK$pI_RfPJYHO&oF*3Y@gsv5<+g(0^~L9Cr8|x8;Egpi@OCGU`;mpv`j^aA z@CYA)pVm|ZxC5@r*69K~-T0~8d@RJ))zvk2G+c5)3SqoGE+WufYpWZ?kn<+qYa7GP zWS0A+w|#0bu1@%wB-Tx<=5P<(bHt?lGGkMoby(vy(;c|>FX&}Tf;54l$*eftr_Qsk zId})V4KKXFIT^y{pT<)yo?)^vNMNV^W0xCX5a&Pb@)0R?zf zjgA#CRc417ci&)wp7F5<|6>+R$g}AnJR9}r9MoeJG;L7I1JSCBJ*W#6ndUtbqM<5# z3n9u8-M6Q|p@h{5Ka3g|N<02{&utm+*W*)#X+@8Q@CRX(jHv+oSf5g>5&Wk^U_L|k z#rxy5X`Q^M-6Zh;vExE~MwLQLVbJ0ZxVJn2r8nyr$dK2LzlHIZ8b~*g8TaxL5)w)v zC|J7neFnD@l-^P!6hren`0?CYv{F}BDkgB7><1Mpay!6fpq?IYZ(vA+1Y*Rz+BY-z z1x^CcbpaRy8F?RCq-*z`a0pH@nWCad(9rx*4RMpH1SLi#J~QwH(^|{mB9|`eSJ-$M zcMv$x;%}Fb@X|*j2sGs>6ZY_G#jQlsG0@CHv%#fQZ^pnE5kYPg(yMt@D+M~#xadYh zlY(6LqnQ65JpbiZJUKd>k1^&We0`kpUp446vL%T}yMZSpYmR5;YOy|QPZZ=FKCY5~ zY)3wz7;7Oo&cn@dhJp+>*-exFVXRwQ-9#_Q4gz5``qZ|1xF>+PLN%U?~29W&N{YY`qe0=axe#SKOgFnYWL1~q4 zxmKJy)x+C*(jh7l!{Un{jRbNteWj$D=posfVkB*Da^xU87c{(`8;CQA13(fNNPpWE zBxrtO*btkE;49M58OJ!pLi5+cH@C1cP6(lHbNm8Lr>H3&D^m=l56x1K2B84-#Zi!t-kJcQC3#fM#kj#o2Md z&I9rFy?(WozwNi6K~k|o6}gR$B9VRz>^gzE)C^#)J`xy z3OM3`5s)2PaNb1xB{D1oOMazt5)h*RxyeH!mzoE$*SM3&FXAhtk-Lya#Efm)CBBzp z-0O@5cNJ)q8W?~dKtjHz1qf6s=De@s6(^nzDi9dESf9@JWYp_Uk9-34;~ zZjZ5A3jWx=SL78Q*cfo_+;B`^PzMGH42BS-3K|cLpC2qw-Zue9(np`(2QAyx-oW?2 zE*msAC3IpfW*(9k-U17KjwYZL!Rq=vz8@I_$@e|<@sQhyq7w@W6B}(mEnd^MS=ELOvrv3BY1J1qF1V>E0NMQyo5`YMPG36l@VG5!#GNO3~2g>@U8n=?pRLDa}3<^BiH2tl^)=0-r;Hl)j3AutrSD z_LpD-Z*h@9vU-rMA_$uJmAsj!&WN;y32xS@?NAd0A&4dc zo0v8}9upi*8vVCG=qOF|^xx+1zZu{EvhDAkS@nnp?fq^$4Asrq zX{-#9ai_K!23a4O(@DWf_ASrJ_1W*xX6K-gj;Y9mfPzx}2y0(>BmtmyphDFa=}Jel ziS={=CrYEI0O3mK0?ohoj0d+N1HWes9a_{F;|{eO#s|9`p#F7LPNh~z3_?P=iy8gAJY!Gj0T%Ez>82=Sc+j9sN}XFt9yvW&?21DUgHq^ruLi znIe&-s{ys(7Mcij>{xih5QF*u_u4Z=w2#_m$mx+(aJG?9z`}n^0j~+zrDH(HH2%<$ zO$YQ3E}G0pZ+su-v-;5)dd2Zm^;Ng+HSIJ(kfzL9TO?2`eOka9rv{P*UYSfdvxy>KXF0^t_nH5~cMPT}!&c{awLQd>{ z!tMWqaEefs5j_2j*Wexr>D$qda3Z}cwO#d zZ?Pn4JG|yy_h)(a{7<>-ZzfBfaTB)guc4}>-if!xhcYBc0p+wHQUybi3TXOYHuLO$ zr(8t6k1XXwY2(lyKE;@^f<&T;-ir;tfU+^#oI#Ekx>=H*=l%9Y^~%?tdqk5ex=OHQ zmwPh~6LW80HLjl3z8&{iSu%4Nt;$@BZ^p^MDyokbOXl)_w>gtmSiQVRWh@*{8IO&41VXORQe z`h#WT+=7CvOj+z4cdMGP!NH75?NVm%`L$r4>qv&E!@Y5P_B4q`HceS>?|6D?R~?=D z`0ZelxzFo~nnIUv=~uVZGw*v#P2#=t-D0iL%8-Yx+buQ~Fwy{d{~uiH{OTNpz}@xNB8`f$Z#!QpmPejF5V zvR+QXEGn9EL^Yi@YRUYG;>xvf!SN(@`Lf|Hd;J>qz;PIX+>o-pcaD3zo|2Amc_i0b zq`g<>)LoBv4QE?0@(O2h-(AUb>})1ok5ILuDVoB|wSxp6{H zDjF;*0S6AvawpMn(RDoI&K@z=)jyaOGC}_T!K!kl1z{Z;G4@NFQ=kn8)0B^u1Q#?M z1?XX_G!z?S%^d#RKUfab>rYG*BSzW&$<3a1^ZJrweG0}ev{It=Hof6AWPkO2^p&Ju zcf8_Cnk|aXVI=ozTP<~F7K<7dr{HO*^=kRCSeCeHogz%vjob6O%cEvxx$8@Qou5+e z&W*EQBY|co4ep_v3$@3;WpOW8-nd=X_pN#tHOLdmH+~9`SL^>0&GDIOyV`ehxgy(k zMNd9*&XFpdOq8orG9TpYJO;|SFLa6>kFclA=S!RFyd`(-)Q&4do5yv zJ=@pQeQ1qj<}obI~o-MV)9t`rTm`zX&uztwE^dOk%vx%%2(Pr=SUmWQkaIfhS~#`Bk|T3k zRJNZanwkVQP#sSiI#jAG`0k9FM=i*vH__fma{2*N(v|Q6CaqgJ=ah5Ii=-J`V#Bge zpL{D~ta2`I;Nm6B0F6D8qv8@-pBCtA%}6e7VHGGkJ@z-$VSUNbu)yxNxJA)2>Ol-Y zr*_T8>ty{=UHm@HW;?oUFEH}$tL5@yA9$PRsZ6t(Q3T$``O-$E^r+5M!`1Gv_3MMR zxRBCy57QKiqNJksiG!Qh{86J<7sW0eoLASWL3iN0h4w8!QNPXw6_=UK626IwqU|l* zb{^KS1vlUC(qCUNy&26Xt3U7HIqPEJ4~-+O!_r?Ln;VL=7H1f>zod%n-%~YT%L=s4 z#BrxmQofuE|B-4Dci~0f+I1`WzViuf`+)?FHY9mE_y5VO(CduL4VITgFAq5)MIv{j8U$ z{;wxxXE`0iUp-YmHy+s&C!<^1xAtj#x#C%IPCPl;jFNhj3|}72A9{9ny06YG&Jkx- zwU6(%8y-pbNYe1vy5LhFc{(`9W3aagV(Zq|HABGpD?4Wll5u6f&SrV_To-+6_cB<#= zGU>jzS(?F_yA+7qs=6|q+)pQv>G8qY#BAAiSwU@_2ekd;+eSP9spBK!5jRM#(Fk6E zmIQ!gvSJ_|+@YPL{Vv(TGohM?2WQpScw5T6l5Nssgq1{gncUtY{GzZi5H&(DQ1*AD z^1aZyEQch4V-e$LdQtbq4^Nr{(q~lElnTx%S>M9NP>=7ie>hs*x5n5Vv%faV>~&9b zXeu|nU|30ImEQi4usr~(RWiYQE4t@CrdI-ijABl;E>FH2oo&w!J&W0Qto9+Z4p=Gp z6?Q8vYgAg$C|z9ca(v&qzcx)FCsNP#anN34nrnst2M0Zvvp=P$@?174GilQU(+*GwcV?)3A1Dvy6l_cP|4ETE|MkH46b} zUDk@(!I3kPIzr_H;^GTGqYRbERIb_=!7Uy{l_4PqtUs{Mt~Cf#&o{od55V-i)4){@J z+iG1X)(h(pe9^m7s1a|sv^f}}VME-e-ISd7(H_5iAV-hRt2g|7|J3~IVJ(3#UN zmrrrT4-FSP_a;3(c79wA7d{mK%Dhr=IibO3w(}|Hg>I7C;`AAPUxO0KoK2;RO4IRS zYODjV%0Y8|@c zP7c%)Zfk7^@C?W^7x-qR*|6mK^X-`$K74qocPf)bYnmhIK-MWW=*=E^wcW~$y046X zf1XQ%25~a!9{o3ODYCU`Heq+ku@qu2oS{7xVdbDCqN}Ez*I#Jq$ZteR?9a-Ymuuc~ zWW&y7I(#B>ka$SZnp1ahSw$yg-MDPuGdZzmY_31hU^>INK2V++AIj3audzSgm%)%E zcP{EpY`NAsk`b@@`^0O-$k@PH zc+%IY{>(3?tJRj|3+)G6qGwpRyMzZoG}-DEe|w%zie{ouu>U`K#_n|d(D;4KK`nQVLrv4*c-7tFY&M@-FLNY%yT^}NPL}<4 zgs=Mn0V2K9{C6n>Bx5aB2JLgfA=}iP9v34+u!nIdk+Qd|1jw&R2t`hctyq1j8iTv5 zRaFW}#$;DoPD?I?t?I&c2=-(X4t@6Db68nJxy%(zZy+O!#can%RC++L@a^4qrcyEH zzT(%x-EO4GsK*#+Qb8cc!w8Six`Bd2myVlSwpm-OHTmfDnEPz=2}jR$IBMs!S#kV; zrn`57?RLjSy@>`A)IF@%@x$1*V!wP#IugFD>@w<0NU9MKLPZY5dyfY{^Hz2tdfrM% zrOBQRWJe>0GTxN&lbfcK2szjcq$7A3uQimfuG3AVEN5`J;J(KU<!eOf-!am%FzWIEj>OBQX$L}6`tw*ub);HSRnxT>uOIZm>xIxk2KTAr!RYseS{jN+c)b*4PZvC&qzB($(t&JBD z1VKVTQfcW%x?7~-&@m|8(p>@~tsh8thk$f<58d62zz{>{0QW_|bB^C#>#oaUvG`~9 z`{eKWJ_`gTPtH2wv?leF8b|(j`OGlxK!ZP@m`uAuEZBnL4r>_Cn5ZQ zX@S_#E4r=O6t2Hqs++)lT<(xb{IHa);fZyX$|9din=frt`J0ChP{T)Jzuk$X&Q}eG zo^IGjfo{@6w!bcvWnzqw(M`d=1fJ(svNZ;`c-~m9q)N|(9!miy+hhf=I|bD)=hjo@ z;frgqwnulXkz15c@A3IOqk(9Ws?RoS+XPr9Fd+p7V**%+&7uIj&AEZAnLV<&*80O* z=KAAAug3M9+ox*7mHk-m&X+##h+OV59WB||z2h(QohrEqAIT~=in4`URvvUl<)MID z#8U;U@Uq~mw-=c}E%-)4^4S;E6d{xHLF9pjuIue5(fi$o3R*65WK&J-yYv;VdqJ@8 zx2%p>9Oiuen%6Z+Cd;sIPwexrp-1DPz-h_q_psm%m3<6UYPT>)c=J);(?bjV+(T5i z+jcGZMtAqG180U+WYfsN_{$y#82_v+;t&*3;^@_}QF2x+%$PFe3)V-bN?$!vR?fNd zW6lKSHN6eIK7V%MIt;!GbiHxiukn~i zRRbQ0oj(c}jT@KijmWyOZ+;6u=;d!6+s77>1o>QWVJmQvTM0DcLDF)Wn% zZb&Odc0|cLy5;iP>-r@9(>T}tOJ$&EwOsmyd%ZPuPggVnn<~Rza@ojzAF14QNl;=~ z8xU8jEt#&AE+_eTUv5N{J-}@6I&t{g+!(W!^zKTlT*=34b8f~G95Ht~eSh#w)J%1l zNZ+lW|ErzxNTV+No>X~>rOWkR*fZ2VuabSM#$xU*)M=F{;ONRcBJ{IkqA#E!UZgOL zME7K(H?9?O(6jNv{Kk{Yq105;qma3+uA3mCV};ZKQREOC_&N=*1&zwguic49rml-m z0g6{B17M+gGpqgx4-zxYrD!276L`oQBxFd*XUh?a-hmOZLyLewfgmR({?{u#_VD9; zkVnY*-cq3F#Wtf9Yxs)mBiXS_reV5avP$~&N~33y0#_2A$7!SmwszJeZgA4<-YVyOuXc^}?Uh#kx_ka!-^*(`Z|67!>c;Ns|Q6xPdWC+4gC; zUC#ZIS0&Cp@5%{ZD$h?e&#SL~;rmO>eXxc#we^!p<~Wb=yOzKkb)~YkN&Abm_l>vN zBg(j0H%$}0XaU}Eey#DYUiPK3L{a>7Yuj9K}JSD_@4n9mzOlVISG z5{16>B8+VA{s{b1m1n&VHE8d$i=egY(#D}a|7axnY~^aD=~YI;%edgBVEbW@tycq! zf^NQUT;?s!ZU?We{TZWPYGjbFk3GBb;}2js(OgwkK`^`fL9)>lXSA1LS44(A0$V$Z ztSri2*(UxR;T^h@eZvqwn+!iRN*0X<8n~e2$JIpzh$_1a5S_Q$xGA^ zE9c!0V54ynT&;H{o}e|m5Zda!4toO8Jy##8@LR?)KQHOoG8LJgDKR^_xYyp^_gPE9 zXiijwl~dwp_?M$pEz@zvlIo?Msrn-oz7^Z;=F!N^>4=%a1+Vy`B#GqZkg&w7ux9tJ zoWUY8@qVtZXDTxCU633J6DYq7cnrGuD-+(4Yb50PB7pHZc&(>I@WX)se$?6Cq|-L8 z3h8ud5mK`ah{r*42CHPWeo0ikNp6YcO$;q&%^p^eHB8l=%p3Vuc0{Z{E0{e)&r2h( zP)}6bbmZ}q^Bv%|=XapxEtX`xbX7(pW#Tct;A5X%!i=<~cx_0nV4M`f}6!73}CC z;#EIG&)vbY&^0$Og@0|Cece&>u~U$Sb%?ah>!-zYFx$(y?=lsW$V^^@;I&rdWB3mzZVE6S|5 z-Hwa}194l~iw>HyGr^9;;_>82dG6;jn*#VZ^z&Ry!6$)`AT^JrQ7_SSQ${>b_-t~OcvpiQ1OfX6ae)Wb-=vujr z=bTnO6A$;)B@;er2l8I>;S*uid=t3|c0T%X_!%&X`gruMD}gnklEfO)l`+x}yPT#* zz1@Vbwg&X}`uPHH|FX|an~PJqoteVvlQ72x56lZnN;H2LW7UH_Q$J#7)>n*^Qe$D4yiOKz%@&hK_%gp(d}bF&jOraAg6zG`mE zBP2z{$PMI=BW{=E!me>~z{Vj?SLQ>};WW~XI&>)XVgWfgze5QB`g;r#{u^#2UACIS z$28y43R+a`l$gsU>m!5)&?=jjcdyGt*V@)%I5PF~;t5$SvJFvbvm4G!8nZz0u0*_b zjp&I-!HzmjifQ{;!p(T(KPx0)&n76!jJ!M{U(nag16hk1ma#j@B-6_5r!(S`WDHAm z=CF)yl(?q;V zeZQme*Bi`fgU!U~Yh%@o1t0uRqb;`ULnZ0)(Nf1B2|uH)^+EO5@^eUW_~2JG1NTaM zhvM#C>e}CmA}NKJCaoYh*r{Uj7vj)FTb)Av+J?HbkorldbxtTVMzqJ%dy!=TuvU+C ztJBY4s=Z{T^ga7&PyjlSS0JCl-Y{crr%I2sg1Yr*p{Q19Q*xK+=XnXbZikcB9uzrN zB%~liv{Y3+B|YlZx~D4lcWVyBlMm83Kvi;`!jo@zvh&$|>0$>OdriBYE%Spe3{&+d zqp{0*6Ajt+$5C8Ms3EvDF1v_}qOG7?K1x$lRly}MFeKy&kP;GJ=~LE$+oLhiD{uKZ zA}~vC{_I0C@FV}iU^-d*Ln8bPv9O0^P%K9f;&^2wmnlSPKyY>Z%5J$OnhNEl*vK8$ zPQ5v7nS1fvfb$l3z2u9&q*s-?bGOnaG{c~#A_99R)hYx|vp#?O>&O6PsJO7cModp`B@YF%%|0a{T7lfeYTYKC*#@b>=Kr>h$afBjH8Q1n&! z?0$@IX#AMwA`{O==+m*(Q8_yb{g;m4vgZGY+S*a^N9t$FID4PmRI)zyA3ciPPmaa{ z;G(_2-XXQ@t>e80p<}k@|F;bMRyuxE43Z0qitI zt6ZqM_Y>24ZDRaYga=yo%qy5eJj21UXXWN%seWU?=z^PSu~qk`(0A3SZWTLx8BGab zH5|l}i_oVKhMZ0h5}^87B3BvOC{75Fan~dBsio?@d zA3?Gn#Thp_ff?y3q`zY@5U?5nRmPdf*UUu;f!LuO1e8H_pz2Nw>5XU-%ll_8YX9eZ z?>Gt&9?ht8zUV4#3BR#2vT(Qoc+OwHgn$~U%{}3Y&9A`P+G){x5C#h!Sor?8z+1JS zfGiTDde9Ef0ga#<0r8U~#x@NPhYm6kI3_p|H4_1aS?X8L)dxG%C(9wCC*%VKtHh%w zt%NfU5V@R2Cps4}g`OvN#Vuedj{oum^;DM!o}l!=6Qf;jpZ&5;gzBZnT5e+bV2s7~yE;1pPyn$NYA!2^5D65b#angOM13S4L_I1jvh! zr&boSmorQ9x8fD*h)G-xO;F!4@3^+wSh=p&s{^nw`s9~4w%q=2d-?rHcDm~w%-Z1nE8HCetJ&*=lkZF;cLQ1O(F zq8eJC6C}8Z^|90kvXNg!x&JX;g|Z|seH=YF9YO@ zF^2Ccd|YR!xVYy`SY*jaax4@WjZ2tGFTe5_{4@yGOLdfkZj=K-<&P9c`#T25 zJ>&KP=>{K|-}HBqp&pw0wyEk@aZ)n+2PutpyGu6Ht{$8I-qS+;a~}RCIA17O7n(CX zl+!~}hmttFLUX(4^$-5@G~_4cgNwCOe!19@+Ie3opWs}lMD+UFWphB_>)KjwgvzVp zH|fO z`@rK==Cp6fNk`pmohO+>-n;BVjEIzxO&D1{Jz#0O$_I=5`cu}fjeiXqjpdh~y|*5t z>+g98(*G{>Z$Z0fprGEY45z2RYg*x6-@esQ#yWeMZ9|ZsX=zy-@ z5Wp*@E7N}x%eYkirexj{lYW>A-(7a~lrg;^UYUwu%5Z%tSt#xmfM?b^=H1f(fBi>t z&EK$qIF5K=C!U+^qrqtYjumQ#Kpx&jO_2>Dtg(4gw6U*#?ITtJ_9HQJHWs}Xka|rH zz_i|pC5Ik>vVRNn<;Cw@!`#MHGT=psls-jAR!Z{k@x6n=d9Uu-MexL~c-yp5MejL0 znKrQc0Y|=1w!vHv_^&G-L>iR%uQ*Eph$E1MZ$EbzjP1qRXy`Z!c5Qy{6hiZ>T8_*7 zaQx@FVu+P^zno|%Hp3B1>fb}TADew0SgqIotc$3ckxZ>IwQ*Z;e) zk&a3~g*?tE|7?i(1X6+IMM5D48(wVxj?5R>{=FeE5&+KsN@qsdrji!4^zf&yunwwC z4KW@^s79RaT}ShJRJF-8p>{+Vl&|!!IgC_+3LrOY0Q~u!99A5)m3Um+d6fv5X&_qTm`$!{KMuo9w%+(nk)(jGlV)^=1Pa{RG^lw`P9bM#RAF-6L9S6aUSC#d2v~9 zkW=FF@R+0e8BVnPzaoku?gKK;m*6&;zr}_SmKsW<&=IDrR1lN@91Nu!Nqe`~A+25Q zLP|!)Z=y2#lHZ&)%doX4_U?8a6B6%gBb)b>`0yv@hl69&*E?f)#=@TOw`&ic9HfUt zcfM24XZ`2~lk&CRMULmy2)GD#H1Kj2sQ#jriNYD1TomFC!12FfXhCR1KV$c8RhWe# zc0?CaEO7=SWLuvlHyV4KWW;L}1rO8!hay-3B^ z_}RtnJ1fG!f8cOdU>hHEKw76q(G=2x8;wtmRq*l_HC0%tZVW z)%L5CJACd?e36?v8rt)nz~l5-wHqZ+IWAMJO?xFF+i1Ou_0yZ_myr!Fy(6t@9r>^1 z-*081aBO{9jZ*rFbS>e*$r}=q{ZCXq#60dpWT6T9NCUcGq`fAVhx5Fj{y}+J_u-kl zLIPTJjqPZ$f!<$t4J_dJJ7h)cnU09(B76?#%)Tp|Apk#m4$j9UK`6GF`v6lVio3Rz zN@WBUeSsfOG2y#?==xlms#>qlACbgMZW{Jn>xONq?n$Tl(8rM+Ika#z+5E5BI90Yx zadwLZNR3_taRqD8usb`u!@Y6hhMnDbx4RsPtKtIs6h1F3)<>xCreftjSdAiZF9~!o zdbvK`n|+2zr(~3Lfqu5pmiq;#3d)N>YChanom{)h@eawAS^snIvu762F^pX^g}}hn zmz+7uWo7b1>#(N~voTB?hItNiQ|j z-ETjmz59V|&T1&tc5M2?$t6&7D2ZsSI$R(PPNCAEt-2@ROhj1vc%!S{>4V*@Smg8o z!3uDRO1)60uT3@EFsS|`bl7!c0@J(>P4-Zf$VC3p@UGEyc{ z?=qyT{n`p&m$!gCAJyc#L4iA7k-%YrZ-7bZp(zD=(D`Q{)@cMH;aDOubd(13RgCVf z%|mA|ur4%N#%%ZaF6|bg_H-)SN<5q*U`rTmR3Oe+y0wdRjX#PKAp12S_!%j2Q%G*G zV%LF?< z*=qQDT$|1iZ_Gbk8eEj_lb4qTT4_X`t1we_0e^O1gI`TKZ4z^ts8HXcMgixCVEOND zM#wBw%g7|_+gI!e0EYcXL5Tp8%kF9S#pFB7Oz-{SRK9B8m*akii1U~)KV{X_Sb2KP zmFd2}d!x+9ZG+}vI)_aPUk^GKgF*UiO@n)t)7so_T$v49#UsBDP+2j-YgacLxZO`L zIg&1*fu}nY8(ZI&6HYd)zF%3QPAjE{<&S_evMMo#uk6)sb~QodoShh?7yF%aL&+a# z>FJk)ms3!mN|hs_uy+A30p6X~DwB;dU%Ma|jEeKQS*yj|by&T0*c0(a2E0~ezC!A} zlP9yiz|SMzGVd}eVj4y$-(mmJq4*~kr02}U_Rk<=&jna~z_nni?(CxND|d zKUrXXy4j2I%?h?x4s`EeJ&OPo&+@mkMW zcoY%ImZGOiAFq|*;kutiU>Z}nSs}-cC3MAq^1oWs(8Eb7yaNJYtOzn-7&*Z!^bUgk>iX0v`6a^vWnoFkxv#@0;j2pl43H866)skf`_WHA(Umvc>j(^rEWD>Aly( zSm$q(e9=Xz67dCZsc&3;g8Ri*u=%GPn>+dB2{BViu;j$OQl|}vr-^#Qi7dx+IOg8l z9H*Vklx0tW_kaG_c>;lYSzzs(kp7Xwz`wRg_h*`H2AEGFN7_)C{vnUEv#qaR3znCk z>un9SFH_5oIdydLnX6?>pbV07yvPlI1s7hb_>bqZ?s?sFcM3VAlYk96c4Nrfvtft1 zqHp&v0tI~nX}PVfZp=t6%UVl=GfuWS_Ilh~*g*=NupSh@s?T*e>LrlLJ@Z6YAo)ylAMEZgD8 zenZZgaI-eVvtfHehu>hs!Dxxl^l| z>4bbs6@zO6&qp8f%G3Q&0vXLI|FQ9dG;PcPJY#MXsF$^ny?hzjRJ~vB){h%|pA(A? zIngZZwroA)Fz5O(@0=p!MkZQzs+q26a1aSxT}}mc!GE#0$6oIznj@b~u*C|Jfyu=< z9E`s@yV_0@K2$B$x609%@EG2kWA~o6IeXN$_fB$ccH$9mfgeR{HFuhj*QcJPhEBg> zo0MkfC%dluGSh>J<-QkWv8~*E!-Fz z!obT?CbeNv&*1gB_o0{mJ3{@B9?Zth?E;`j;y0`L7ueA+)J9_D>B)huDuC1l^V<9G z*4n*W$Ch6eekU}~=>)uLV|FP81-Jd&OL2`^{%PKL`>2sCV-7N0~k`Kry-$Vjh4Ko!jDbnr(|r z@%0(IaV~a8x={C>sXV*soMS{{D*|B6&J25jtd$G(sVhteSP%ee8=WNF$g^!S(|cvic>8pT9^PypZPgs)82uz>6rk;A zFkBmDRFuLW0;Tb&A3p{@BOnU^r^iWdrwUr|n$VLh%w;;(J2EywJDMXT1bnHg?(hx8&)y-F`YG6U90!*5Lk?98V!Vl_via1V8Y9r382~6Fv*` zYio(|N&#+x_A2X&t9P25>*In{`uKH~i~7oL2i#tIjvC>Ym$wgQN={ate==yua%*%Y zn!eJToh}Ip^|{WFtjr^Va;pnR3mYG1Bm^&~wYnod8FUg3DGr`Jo&FV-}I=WD1|Zsh?!XhTgWOhy5oQqdVfVSoNaW>znL3v(Du1 zI;Z*IX9!+TD<0ToBnvcO{u1Q`!Zp+|mw|wQZ2fQ+;QwnC{f|ziR@@Zo4FD60f+X{_ zo}@W?TwGKIJW?&RA@1YawheizYJROctZ!kXB^j)vv$!|ov}Ftix5)QI1t_KoIV`KQ zj@T(1w7ej!v1?z)^d)MwC@+%NLJ(T#+a9^n6EW3t!c6v<%$JS1K}}RjlWTO;$_) z-!fx~tdK0B;4~a6QdB5epCXptuYQ>M?VD$CVjK%VFBco!$b)TgY06P^Ve2@!N(x*k>z1f? ze@?sKc(O_vM#%VCftb_6QZvP8(`TR4Sj;e$HhUX}v3UGQvTGtgLEwYf$LM!^z`Uo2 zz!cc+__rXk`#nVMb>9n#tNoh`)y^r!68bc1Yt2W)9d**qnfkStDGt`kN%I})BEWpe z{Kd_fR%BnYC&1Ey+e{p10% zbN`+IesXM~1MaaWf~q zMd^@`_`q%C=bluxQ=^g0T`S3Gx@!67M1VpA*dwB=xH0X5_cu4DV$A&TN-KUdHkdM; z_Ga`i75(6&K*X6^$vVH2}`6u{)wg!h}fKRA>xUpP!Uh@h-U2gk6A1X)u z^{QAd6N%W#r1cA{aZjK!Yzefrx<#HKp&XMh{cm3Z_{;+5lHA~#&slGifjI*3BPXpS KRVrch@&5pthlUvd literal 32695 zcmeFZbyQVd6fa6jNeBo6Qj$t{x6&moNGsi42RKLwh=@o?Bhua7DcyZY0jWcCfWv{i z@%!|#{1)59KvSpwdY!U=5Nk558>*na=6&!*hol7xC-*mHIR@{Oo0D< znD>AYXJ*+izz;M_DHSOsr1EH-Yg2UKcN%kf4HYCLA4Vjkz+fb#D_|(_7ZQ>?Clb<* z2@;ZUA`%j*Q%b{2QQ(I9TRjB}6&0jsz%?ck3Nj%QDsY7i{6h+}Mne02jf4bH0=`f) zkpH`zf%4}rib)3Q-)p|#6OrNO0FhVLT6%7JD$2sNGNrE^lf0**V!cpNM1A(9npw zys;40crN?z%XrBydcNj z9S$ybPL98{fk#E}MupW}tlt7F-_aN668$~%e;NCC9Z`ZgrY=zBW-$y5`m&ok#WjM;Df1vHZCKKML&0YIn+4PSXp@hhr*#P zZ~I|>;*~tEn}bi6c}7H9y@$WP+QK9VL_(uMLdKLtLis}1}j`c7|@*@(; z2Nq-$E~YYg)eo*x+Nnqi@D{c2QwQyD;sk+`!AK~qD8Yu9T`_GA6O7&E`jby{&V)@mG3aPGmH_ z#Cr>WWto8oJmVgy{NV2hjWQB4NlrWeU$N(*0?$y=Xrlcc!6cFl46!c5{k!o0pIiAw zx@1a$LLkosW;3X%L&w=_oU+wzr1YE}vu}kYy77O}kplQ3kBKa5g0;xrBti>dV!-O^Y4j8L8>L(lbtAW@4Hm+$bwLA7(i*2D0zbM0i~ zI@TImB=9L9i0oVrYb>+|f{Ts>iS-ueZCyzF1fH`*%c+ZJH-3prZmLUYBMjWySjpj) z?h9JqH5`r!vug$6{~}zTc#lC{zuXK<;aU9rHqtlgMd?T^f%fct4WHgY^Ojca#77Z~ zg=Peiy>a#BS&e@RJ~C!kERxKpqdig^bg&VT*ZHO8V(IOHjR2z0!gB*ZVEf+vM+85; zJGhwY!$nlwmT!M}Led>t#+tWI1NOD1(YO3(4Ohe$pR(JFNL&r8*-n3FJuD;>WTj)R6+Je%okoy#)ujF4*x5f?#sX9s#W| z2hWKEM|VQZXwmbs#v8R6^bNnmy#Uu%L}xWL!uYytzhxH+rI9%Go^5w@KG_^`?Hv&u z+ZW9|EL%pW78_yHZ|Vd6LM27xz`X8=M(Ho6kxe1dKN^mU5en+$9>;;0{NhUqG3b8@c64Z|JzJrPg*O z#Qmjc$VcD-)GoD4&Ptrr4j25L@9H$%lj}~tv+p-{&U9YbFICJn!%!{jLP}ZPj4vjY z1-wR0JU6B!+?i6HQ{W55u*Ou+6ur;OzQckO1wY6E>m~zSRKM_-P02RQri<}NhvTjv zH49m1r6q_EgP$)3sc&G>&OtmE0YgP5kA~A#aP1v4M$DF{7EAn*fM)ZWY{7UY+dw;o%lvmn%JRqVmydbR{PlF4NVL2a;{ps*B)+!K&fs&2L`F6&76YtHAVG}LC>FbPkAT_~R z^oJ9Zm{1rkuM8Nhh`XJmgD*6e3P$!MJiYq*^OJ{Jh|~Pf^J(;rBykA85c(x$S!N1c zv*J$!QLh5?kmOCpRph*Mh8;?y+8o_sOOat?&VUf%%PZcHP3k!8LfulRi{42Qy`D#^ zC_{j<^Xh1=n)6Fi)m(e;Ho)apl@7dcn2XqOn7_ejh)xN3$@`UBl@jvQVZA4+y=BOJ z!Qu2gH5z`YgYDRIDVuw7IGn^M#7`%;*<`nBKF?n3Fm_@WaF(RqzFB74$@)ys(Z}4J z<_TNHe>+@3R?-Q0X@L1fCF3&BJcV)17`kN+>QM8?4_jV73}}ab$co05^PEeo7$CmI zE%k-8royNl2G7mktw$!XdL(7>`J8$fjvI8v7YiLE?I-9jkNNnngfv{kb?b?u#h%Jv zg-P7B3_KkXg}LPlsszrALI5QRy2x#+Rd6k_Tf165XcT$B44M%WP9$8YFO0t7W@B#xVt^;qL0NWCRk1@d%+n<1ytPPh7^^0>nzQd|;$*d9%koah*-@-w zcL3AFf#G`Z@jtBp@5Lk?pX}I+gB+x(DW|&Japw%LIvs^TtXj+We0!fBuzpS7)PL%C z^LeDMH_eN;Z=Ei^uFVhu#5b;m4YXH3?q5r(i0Y2^>dg=z_Si3o_u4|6wjiIgBO&&c zj_$^n(?u(fan0vBe-70mgb2QHwD8U#V4kfcUa%s6#U_7*@XT>FN1eAnG+Dfr_DE=` zHIL25lAt5eKGIZvPb269n(gns|r!9KKMekO!ej5aOAa2 zYLXg^?^gDE&}RPw8YJ)@VH;{Zq$PKQjOK8;vSq&~ifn7SsZlf$(@|wQIp|6`*gKNd zXg4S4;}*AfAUk3n2I-Z6rMDZr7cLRNExp<1yQFOgY~o&e&?pxUdC=q*Eirx2&-|-N zQt?gV+meYT`yWAPnHZEL%6_*wx04|xprthyew1Im96HxptU8wf7Un~pdn+_Ol@={(f&<< z=&a{kOF~YD!e#ajiM?^+GTyp}lMmNQhZ<;>6127|<$rb3>3G|}AYNLmzrF0S$d^c+?HbQ&xaRRl zqFp7a;T`Dy>)8?+ji2nZ3j4JYxvwHC9OzIa+of|H8if7AHYFYG2~k&*RfjfZrDAx9@=5x6wj)*(b0C%Ui1SQURPsqUT9(sMD-UT^TO|B8l+` z9Yb1dNmO~f*VtDf6}567A8l6=V?(Bj8_#jcdGmCyai+orF(ql{(1d%)KSS_r0KD`s zL?)C$!q+ovh=`D;X+)z+LFlyyJbm-L1u=C{Uu@udYqhYZcL%TE^8f)S#W$=0z?R71 z@|zby8m0>|3-Q8(Tz8bp>Dmb1>@@^q-$Bp24C<2ykONe~IQQTEn@-Vq0Jyt+m+%?d zze`KbW6`vYFLhSo|C?xmXCFugc9$sS5&Xg3)=Ys&O1}6L;(s=4CI>+EM)pkUKj8g) zDi8?v#gyTHQnvS$z&`)C+DL}+p3UyKd)bru`e2bP{W2bb-6I`sDEXn2{H;|rLIO(gu_{8tn_1=7x@zD?1eGNMklj7Gp=T5xYe`WNZ*A>3FO=47k8e3z!8N|A2LO7pz9ed zhzFD=!jW9t)+Q+?(NPGPvYL{;W>idqd&Y@k&i9MEQMN!}B@f4BEc03PrWON0Qj~V2@y<3M7XOr;;7u`c(w-P8`ZEG`3 zlbFEnq}rN{YS?4G$o%Ci63WyQg7YA47PaF$+g4WtX74B@R{$gn2o8mhdB{wsUE|wa%k{Q%G*q;-jc%UKD= z#+?VvBSGe9sabUHr8JlZi1&m|$vt-N67Wat8XLbVzW#xMg2`C|HhM7Yy_K2{I`CZ< zvH3NcxiFF;tpSCB5ByShaH(+w^HSn?Ku-B;Z@ zS88$fHr`ktAuUe;p3bHSny>K!KOiBCxQ@0x%`t=8cOM4BSAtOj@RGUt5B4iMDTz)D zIk}eC>=6yrp-sE_9|Lymk7}lyxY)>BzId{eyA^4Dy7-K#`3Z@Igx()JB!$@p`VBuN zjZ}!xIAlC*y(5S@p%>S1I6Q|mAbf@T`uHm2+1O2)*VGHC4t3{vwK`?W=p5b0w9aVy zeo<%e`JDukRstFV0B$Sx`Q}5yo3UT{Mdvt*o55r-_pvQ^s{-6 zh*fNt_^=89H$(Moz~Gyz5nDJjk(aQ;H^l{PQ`*FVgHu1M)vH%N+2x`^_b+0JQP^6( z5gL)4HYx^6rUB_!^pg{cE9cSZ7q0N}of%9nd|HKe@rJAZk1Wl#`Rm4*uP8d!6rD*^ zwY&t?dLzPl!$lDTPe2N-2cJX0d1IVbUQ|iLrLtw2a^9Oi5!}c+&koAobK!r%`X;^g zikxzyCVNUf>s$qMyADl9Q2tTAC>F1GdDXAS;k@?ApLk4jU9-#-;UWE#_1I(F@8oZ~ zz5-%?t>=|rQoJ8ogGGP|kGS;UKnRf2sH0n9lbJ#VfB5mj!AJDHbo@ zVI3Piit;yHK6XiMIXU}kytHw1_={zK$g59b@_Ogn$AiG_K*kNv_bFkEOY32c6ymNn zaH0A*v_!{^3*7r@l4Yct}g81pU8g>VT^Ap)BlAhaHONgH4A)GLA_$i>QapHdL3%E?9YqsjjUavNi5TWrt zp*qHiB+|-v$a0Qrs6`(_=GJUmB5ZAahC{oF7J*AW{cWDyh=oDbM?VKwq96VcH@nG% zT=;%#z7>k1_w%q@yBi(e0IY9T^HxdHaK?FTN}R|_>E5iMVx|q!{)DVn=_7d~`T$U=zv|x^->|2S;8-xe@Hu^+awm9DdT=UYst8+82W5hcS zR(JhxQk~f$O(v%8Z?}GmDj!rquIHOPS6z9x`q$#r7gpCVV;kp27r9a`w`>>ABU5eC zs+=yn#|f!%P_9%aH5A`hU#*{AQ>H@)Y|rnQKwxqNr6IFdN-db(8+gjtc8WT&MjEoY z!VzF5)fS4tqll%Ts#Lg0qAAVnNQbmeb3`{(?+Vn}vZp~?j=&$I3kr}|@yl&)MxA0;rc1}4FDe9w4#vM-S2uCO zZA9YSg)ytHEUc{0m1t>mF39}zb*+xj`DO|^`?rOFjCo`(dw)=RjfJ&oe0piJs`q&! z`Q*0JnZE=qy4qER?bJFvukLcnp_2 z#lC*ty6WR5tiGx`3)kLbQz`VMH+W93t&yq^D;RGO@EvAT)DnUep=-tThFx@)rMV^80imk|b0Q*Vg?=e=LcV z0HK9V#CR86TINH1*z?(Uu8nvq^GAv^gz|*4gz}6T%E1aLPvs>;#Ef)8hpVg3=OTX5 z`)o8Xe5ibP(up#1Y?4ZjQryA-?8{7AdH2qXe*%ht16h5iq28?oMp#Yvc`;b8qw6|V zMsGoW9!mqMH{~7|UTzD|=R*!egaVkaTj@?>GNWcZ6Mwyi&s@liT@INAjS>b&hb{ZD z@?I_t$8ho-WxECmFf53v#OF-v8x=`QNh@|JP;s32JRK$>(p|+mBeWDQp>fVJpBIoK zVV8TNEc11>sk3*Z+4u0M8HK|@Fm3-iq19RS^!OerkqMiBG7>mYlIXFSLPqzD;ynX= zsmZ;8vGKJ87wHoK=ei*}@g+nnQo6od-$z#I!N*B9tG*cI(1)Ylmv)fZ0d3y zBoaO~4skigBRs}HdCK5)*p5-^^D{>7R3ck{z|PDS-FjfK!lZ(P+MpCJ*N#o_iV8U2 z-*8rREznsOWD7KTB7hWGihPZy$7u}v^g<`_nkAUPAq(tr(yduDbr;U<&v0JZr-=*?c8EceZFXMfIP7f-A)PvrF4-Z|3Pm^g(+l~ zl0e05RohZ?eW$~Jffm@UNzP0pC9i34xIp(T49Ea3e zYDG6LpPg^M9enUo>@Ke?rvVaZ5{M)Njk*{LKdaZqu{e}7ecv}n#f_+^_~3NYY?8$y zJGpjOJ@Rmu$5WlEzWz<=0q4_;HIQl`E?%}7(Lb*b{~d2E#0mE2wV4@yiU0*V2y9w}JhSwc>Z#AlCgG+IIl z&SKQDDqilMwLTVUNh;mnRr~s=YP;;*3*_b28morAcIbot=;C*M;~7vF&_<{GDoI0z zW{3H*8nH4~Us@Yxo8~*mhO|jc-VF>|KuZl8euD(c|)-eBzuX!?*3@@Y#P7 zwF02Pp!4v<8x-&xLeBQ=`LNp9IdYpj$B(fDr2TqFZm%|U z+6PF8Ta)>0b!2uWSMO$$zsRVro>EEVq=mJ%kZWW0tB3+_cdfDP2G?r^jFIMOZJi&` z{sr@HQ}@usFMp_&kc;`8TAHapEcj#sa0JcMD662wv@?|BalrM_Ou$j3@#1WrusZwQ z?qmrruD2vw9gfzzBh$Rr!X@~>`SmN~Be$T&r?nK>DmgUn*YFtm9d*~MsHhNe)c+(b z$!ZwSNO_G*M{uP3%B*AR*x9Bwb=F0Op}&PGPNyz*?!#lQ_|et21d25<3TI@%ns4s%&N06*; zI#67_QszlmXRBRwDR!)i--LK9*#8n}Ig08Oa9FHkaHX+w62lri4LN$|AQo;}R3!E8 z-q~6D6@anlTDFT_w}yuvAtxEw0=&~F8rHcf=4d($(!bua<4c~N{wi6?p@yQI$#5}1 zjj8oDFyMWW%f~ku;9;~x)akDnzCab>8Y}Ul!u{G-8TamF;yIufPb6nUkkHIjGUjGf zBST0QOn$470w8mR9kv}ng`PJXje#)l$~f|9fOMZUge%HoRlSC5oFv@9vg55p;rWcC zdj`1-&IUp=HB-AuA*U~DC*c%GpbKpHthNHGtPr2hawa#KoKcZ99u z5`1bp{v>z>ePHn6*q%W_ww~2f4;rPTim0j@fgyV(+w`%Lf6_T8-cU4@gM=v@j>H&w zc!LOsu4nJWFq75tbWP5jwY^o+Lj1INmf^o{&1w)cxY<~TugOIAN%gM9GB+C}Bpyom z{jzQjTs$$UV;c*6=b!&<2(WVR9Q4Fs=?DBRTN{oA{dfbOZ|&)sFC3+1n&*0ahcv{g z;k9!7O%mm-u!?;x{~Yi=i|s1|5Gd_tm`YbenvAEh(>_E!-?m{{=*P-Zbf{|0^6XZhk`R*QA6-V;(hE zm_(*y=G)8BMz_FRqW3in4e38?Uc!{`(!mCJ>;%yFz+sE7)+75wdv_Y!K-2mPyE5AN z*lY$=esNKq;T088ovG({lp=@%zPLN#D^<&TiHyR~Y#g7)WKP#Etb`U}2SYaqk=Eikr2MYIhfp=*No|#$( z>;LmK079&Dzvlx>FO}xxTw7vbPp2}i2kE_6CIh>ap14eOW2x1>Y( zVkfY=TS((Q8@LHpX!yqP_~r*&_p?HCLQBRjFK%|(hznkVywj@^UJxCbw*HLbgg}M; z7u-KBisf`PFSQlDUa+K)nvJ3E$y2Xs=IEDwvT*z*YEPK8bMrpnZnSlb#b z&S|zb0L8VbhXNtm(it0rKZ@hND4r$BLL`UJbEii};!q+6qm-yalq9sf{9 zFpy5Ves6O(v7@H@k%+b}0oWh+S%TT`7R!uWv?j@?TeMjS+8n+|!}z}OKKc$~!%D6T zJMGcjz$r0ud}7JBZ+A{Mg+kaCZD(f zQ~2O6I0X2Ef9gy}fNY`;P;LHM18^TbfCg|+ukw=q4!eMl?g7-2?|<)&@M>o@8#h?^ z8}J>|Mn48OmcY{{xDke=7&xyBfx;BVW-7+1hFxAo>If(bI~>OejA+!bZ1Q)x(@L{qXd?Ija{aof$NBV z<%2|TF4m!*t?*R%zGJ{H00^*Po5_wn&H0n~C=c%@EczZaLj_tdqZ(I3DNx{fbtbBT zlJ6y_T7s$jd1{ggU#p2{Q;1`Fz+45A(T?;PD;m1t1$HO8AdmqKl^=5L_EHToLQv@81a!+Q#l8czU z{o-#Z2#uK1{agay+%=%#GCBC=snuWsH%QoD|2hYN$4c6FDzDWL(na7{=7Gsv%VXcE zt6}VH-B)n|#|`egi`SPYkiHl?2yNKoHGaEU0#Ow^ps5z(HlYG``W#3u;osD_{!s*_ zyvAjte`VVbz*(;Fy~=3c%Z-E+bn%lx9x&%bY~M>c$_)}$y%lKF$?&!F!XqxTU|(3{ z^1j7jqUp;bT7YR$718hT#b=w(iXDeeyw0X{A&0A-CvZDS8ay=ex6}vTBfcm{)o$D4 zZ6ZZ4iCyu!z&P&mRNzprwqS*l1z-#8G>oMQqNYAcpD=bm@C-Y>yN8R-4rM8+d*U_z zNpN?$wN>P7QuCcFzy18rG@qS(6N9vJF7qC=HKhkYQy*TFX}i>oBn=pi+?4v*|7qWe z`v6<5pxEgh*u?rS7dN4r@!;H)?pXkQVReWR_W=cn40E4xy^1&Cv9N)l&HO~9SPYfUuL;Kf^^|xY=vq7Pp%Kbvyga8J zsp!&e^+zmP=cZ1-&66;Fl<%N;C!*jJ!vmj$SW;*Yh2wo${bAwVqAwabuc{eGIgu^Y zh*t0QN4`C&+p7~bBs9?HFm4e|f2~GC;#N6p8b-Ou)%%#}xxy#wk7URw3JfMQ0d+Uk zHTWuZt(Tk0?}XUMBUrV)IAUo6Kvi;Uw>9X3ebO+DOc(n)1tI83dEmvR=zd6W>exX#4B=iS|$QSoH~_ zrX_V+Ngznt*i%=Cc&Edbd=FMk0UIbcMdH1mZF5_#GIB&T8w zqn!A9RUJN+<&MiO?P$KBXVPh9H8uR6ZO9mCYHK3;cYH5|1`aGoi(Pj`<$zz5e6EhrwJvr@vM-iG`U_ zCk&Vy6s-EVSlf7MHP;V4P-9pcv_===NMAci<lYhh@lZ_f7Fxw7o{!HM*I@oY&R3xBMxeW+Hx?_11P%t=9pSX8 zR&5EIjWxB;#Q_w#wInLL>^af{E?J4-tp@xM&BysWEo!7B2vKzY%Z! zz7V#46O(8wsEwi=I&t-Lr>jlP^#y}}b)Lkqm`7184IOX1r2d}y_)?hfC(_2nhH_y`p8rePc zb7@VeqYk=x=j7I{b!zWf*F~LD(G!-bh?e4IU=Y=EL?%ST zj)M8Vb9AJc?qxv@N3$jOvT#0!zHqv=dUW*--LhL;V-Pf zPvnU)*hyl8ZZ1y<%;~vHx|g*%t3+ag2n2oU(n@$YD`&{5t0k|qM|XsxtoSMtJ)A?@ z4>Q>Dbk#pX4R^wjXpe3Q5eee02`G#1IX);kty+*glDVUifgds*+$4G7*7wuygxK3mEJ?!qba z>G;O_Co0pZIL{~>x=EN3I(TdF{^57gxP}U z3EISMI^$kXNYP-ObJSRiv-Uzj7~fpN40reEmO5oeio?UXFV3xlOfBp zoQJ5X=_sKSBWtR=aEA@dDfO4k$elf8pQ5i2 zi(Ya!*7ABP5QW)gua%+40)Hq%5ay*>sY42}s=_RBkaP}gf(N%d7;|(G_k8$u;BdOc z8p2Sxr=hOa|-r0`n-sc?TK7cQDPFaiDmb4*DQSI z#FC{L3jU3*Fa7(=r9XxYPaEJ9GD1?DlwL-5)%EB~0`~@bm$+yTzsyEw2oV^HElYJ9j^)J3ZWBPk)s$^i+e(isa}0`ypc$7 zbeUOfo&+7IirjOZh%fTtNkpx2fh1q@+VQ3ZBJxga7-!`^(wvViu!NuNDg?siU{wK^ z>$e`B(Ud>2u{cEFK!kPCe})@H;NVVV!Y^*{^oz#!zU&E@RD|Y!tHHtc%)2Kq2B1C<2=;cNnvEtYeuM)53|jcS%Piq%O_`+^W@% z;pAtXldpbU2zW!2Q&e&6L|M5kUpw&&89IVrdcP~-Slsq1(WK%w%Kv6IC1f`hEk6FR z>(;Or*Ta%8+8HKR+k-eOp9x}(vp7f%)jKaTm(7@rR#Yo^2DduFXPQ`^s>2}O%VHAi z6?#Pr__E~*oV5C7#BqhAo3FH5fSe{vryKA6e|}9bZ5j}2f8Kb@M~Pe#aubHiSKlN z)%dJ{1ZQqtc1uscd`GcpI0G7q8%KvA7-f};ctNMvOM5dzs1|Wr<^$SEL9g#uZA=JT z#w8b{F&hhrT>pI2LM&O|QYI0ko0Ra-m;fD>Js{jCg+H}9l5H8SR zbd**Wb<=2y9X)J7T6AW)wSRB3cCPOp5Dv1^%cJo~>=?q3V9{x3aiZ`_%Mm;h`!E?g zbnQB-4ZB{QK!v%`Df`7>T_LA4lJ6bT60jXl7^`b*tAIehk+Z?VL%n4W$GJ||nFa%@ zp`oeqBP&7Y`5LJ&k4={^Iwc?ih+ZwW<|ad5uLE*g5x_YOgQ7rgo9#x&SWdUUTJR&W z%BAshAzM1P#N`i7?s2Ad6q@U)#jZt$5aB%{^XlS-DW0^+`f|jBSoXrv>ofc4u3?=& z`GBL0k!Etrw7tI9OVvC}6^1JhV?8FAn234zyl=49U(5BIRnv;Apq1gDS@dd0gT8O? zsQMZRml!~)r?z+V2$Uj7S+~CBKm^d~YgMbwAFnJ_rEj_1Iq{{79 zc_Yn{mg=Vr@&l$VTiQ2(&wJUlTpqnBb ze}6#lADjoC1qv(OEs=RNf5NOaP~lNBh#sf<6VT0o3J(dK{aerSpfuEVew!g!<^fX)nte6b$112&!kuof9YIXqya=l{Si- zbJdAkDE(fD3ND1|edc(9AchuRBlFuR(hNR^=RMtt;OP~ACaZHsR2E%8SF9guT^p^1 zSw;r`*ymZ-WU^DmTe#_GZr+qh0ji?zKoGAN4N?k@1Nn@X@HZ zHxsZ%1z&0?Bh=k%?7Zg^>HIt=pULep8% ztIF3(N=jveDMCEnxEVd=jGx@#b?{>;qd+0gVJ>q+*OM;2${d#;bijDx*|j`J8*i=s z_vCF#!r%)+`5}Tgeh)Nx{%8fzpbumWJox8-lps)Y2q^T}d`v|7Z^RN%gJIFxKOYtW z{ssh4Ni7sI!v>TD)D>xsge2yE_qPE4eIgYgAbgSsH?p-YoX}7mU5Mea7-VL8YEAPA z8zpXDuD<3Awjl456Tm0DF$PHAgBiQJy3!QT7ezos>6?<0k}gR8LiSD(tA-*J%_RZ3?hf}V?Y`0;SC=%Vf4Ct|js z9)n|?Kgc18wih+w`W04Y+Gkb8JS#kV6wtBF`jHQ9T$RQTGQc>bRJb!IJsA>0z-+G)xm{cPZTDtIUEYZPJ|N&9C;JhEgbJ`Gy{^O9K1+(s0?8 zxipE4az+yfntbQtJ^$w_f+h}-y_i#|FSie*9#;fc`120jVo(>cT7BNGnoEo#uh zKyhavsKKC8-HAx+Ae2Oc%S$qa6ud{mX@5VEBi8NoE?g;EJzDSqCmeIxZcR&K2L&t( zqzU8-trZoQ1h#LR`c=gOwtHyymG*5#oNqp>)BuMk+Yh~GVhe-J?X&QmcuEaFsR>vQ zW?Xx?nUBwcr6zx>CuwYFZB^ofeF)2To0-m6JtTTn=HVm)zGH6$DByXPwJ&vWu}=FL zJI=7@dX+p`v8cBzhCo10)gTMBdx8z_nF|mQ=~z5(?wekIjR(s$UTlH(IxuR7#F9BgM8Km!*FxsWz9_hi>#Z_ zC^nJJxh`p-+xlABG?YkSAsg(TjkaS?9XDOtcseTwG>{u%fxj=v)D>G$=*ztzhkfMC zPqUv<2|O$Ei4w9sPgx`|aqNmWOtESIdA=%TAGTiYn5ox9IWzx(_Oa!H$6od*L2mb} zBxzeEA`**&NU@_Ynx)d0gb79{vl!1j)*=-z86rRldgd%CuH@2?N4#pm2HOCM>F1(HRQy(^APok+gW@Lnk2%r{ix zlo^}9KeX16GfWo$ae*lMxJO3ky_UyXS@g%2czY9SO3{j%?3S6ySVij5GZ7mY4%b8r z!eJ)D)bn!TNE(JarC?v}u695vYz)I^Vd?uMfecp0;e=@Q^L;BrQkV~Ub2cSxV`X7d zxe$`OAGR;!eEua%lrx7j_0++fno@vfkFI&fTZpaj6q)`9ERr+Y42IXJW@V_D_(~We z!O8)prVQp`Y74^YG3Lz{HEdAdQ^@w->BG~1y2za+2%BA`h_~SEb`TW)_(*9rWORvs zFUxh=j5^i(jlEh&;bgP4=$?OOwIPS~)|>swd8(a(de8Tf;an=NUt^Bpww^Om7HE2& zYMkd?pFP^AAn2x{+VY>if?rWF_PF7x@yRlLwcX2V^6wPyU;lEd1;OHgk{bF7+E6HF z`UX>N;y^@*nVge1;Bt&;dR*ojy#o_*?qgF^Uxdv%WZ%jX^D%cXXyQs2y))FPDT_{n z5ucB_dN_+#CbH#~oG(~_sCut@i&f1Y`X@#&iD(%&2JphY&0d|n&(;ro$isL)-_pWH zLX?}COl2~$!cJa{Zr(W_l=I>ikzs@k&ttdV%W4qa{c6Bvpq_sG$tX4Lm+FTf6gfh3 z9swn)`Pr`4T?Y5O-K0hwjo)n>`DJX*uK9TLKRu)? zV4K_7{0P!Yu&l2)rQY~lVBC;*v1asA;(YA$8|u5zxJBqGs8#`hK_c{+pCBxbUEXTc zcLf2CJ3n47vIWCC*YpHJkvVZ*IEqePqbV0QZXS5#=`~ap zX)?5Rq&|5OMIog8-G5;$`9m5n*3pwd5O0YDP&g^C{oE*$1BaK^9fB>Ro~9CC}-U)tWwuqC8&l&0Ds*;r?s#?D`$VH z^I4=u9jo~2`%nbvyH4QL7I)?sGqeu&8#C+oVUjw4&wjP^0L1VLzDf`Qg=- z3gkeuatJsk9Oyo9QS2h=&(wnRzN(C#1ik0;3GXZ0LEL74bMNc$Kb%~DG{meW{>M`{rc!!8{9V?PU9907Z zD8z&RMIlaW6=4yFq;nL8t-J{w3qTmD<(X$%HMgEuEyhq*<5~@f%u=mndJOTKY+p@Q zPZF-CvX7Y^$2jidCGXNHePH?a$=fP~!fda&&4SWzmhpHZbheE$mRWq?PhJ&>@ZURH zWGPS(Nl18}Ae-~Svo4qwBB8<3Hklc6U+L&k0`n>B7jr?efv=T2DnR8%{K#i6ERK#g zb;Cs9a7kQw_Zm=^$&E}1^r2Z1@B8vJV|%k#dzpsyNryY$SU*H~rcjh2rlHxx_!6tX z^k~;BT&oy=V*cti%KTKTbj1A_9Wsjb1JI_^mew6>>SqIcoV#mEVwSDDQ-pHeREkBc z-rd9n+D4yVMclesQrP&Age16o@1Q60RZT|8T7NTBc76q#=gNAz`GkZ5eVwi*_zh8; zc>h7G?5rHn(VMfwu~)L5sfIr@Mb6D@KwwGar5`49>d$4pCn@k%9Lwv+z!aoehP9Hl zX-~iZ15)L^1l!k&)c*5h{^U{067=UIs{23m?b%4W94l_i+ZVJys8_6)M-L-rtl;ml zISNx9jDkzhlpk#i*I;y&;lv}Tg=~&{2$nRGQjXsZD-hE z(T{%&VwD}D6Z5lo%Y`ws6_dTDrPCzmsWNGVT{k!x{eVmTkWjeV6$E`OzO0EA3?H3^T!E4H=>^v!qHZ&eZdOjjXcEt^SR>WQ!Pi3Eq9C_HAQP+!?W`n*E8hLbo zU8w|33ynk|EW*5h36v#CD|^GbsNB*JRJMjY{tv#q%=VS7dky5h7cH4y;e{sqyE7B& zN(;ZNQ`h{cP>LiEem$v3j>1b`O$0CagEdcHuJW$#=dEt2S&jaW_P#PKs_1)H5Cue} zL%IZ{RYJO?krXKz8l(k@fq|hrC6!K5q>&n6kVZl}hmah)bB4GF{rUgj=eghR{c@k@ z-f#1qGjnF|wbovH?X}kX9{EX=?6S(d3}uBqP8fQZbdfS9zE3x=%6R$w4qDdCEO`TXq7Lyym;hK6=;> zDz@x@CI%F&gEf|ySoC1jsY!d4^NDrgMpYPgaog5YtWk2Q6FBAkr;akyYp%U*Gaq#riNL58-}mHF7_FTK3=b(ADSmqGI$>N z)SUC&J$Cj@PhYA9;g>Ken(e%qyFtMp>0VXREnmujb&wojhgqvyAhTWSM33xr68(Ws zJ6ij;)6RH)5oB~8vWZMkr4$&%r~0kV^h%XM8&j2HBKGRQmO}9oEm)sXOTVlp>HVEr zuMu>pH2=Dd5~#1G7n$A`a()Z% z1bNgWuJ%(KcI$uHpuGy*52O*Wz{2~zg-u(@a*q3UPFQ9;lIj_8vDVB+fVD4Ah6Yi# zoD*2ZmI#Mr&BG_T^^(!bjb!7?Sz4hFwl-sCo$^KBO*nADChWusua^prr6tf?V(s`F z#t_pg`~I|@i4v9BqT@8tF=Hn)xG##b5FL6aycEbBSMu0+Ogm;n1~d^33)`my!I={S zVnV@Hsa$6sy+TUjJ%HTVl z=Y`<}S^RL1Pf5-vBzWTu&5})@ReEl!4U#jpGttL$+@AHMS4V^QPS+wgVfkYOKeFfy z)>#H0%yP4}k`|U*+NCU8Dy#_{xPU5jLz4P1swS#?OL1@`Z_Y;6^m3xXxAED^>woNK zIzHMIbaWPysB3jF?7-fNcCvT#J@Vv;qMe%GehyojpB%f_KIXyWY}cs3NyXm?*`7~^ z5%hJ3&m5=K)Nx+H*z1H|j9iU~QbV^gsIT2R)mq;6jt@4u+hIV54y#~Uk1@!1EIb3< z=b+j2UWhHa%!YBLAnwCCj(Sr|jRl5FO?w>P^wJ>&fkd<5mw*kg8lhC$OYNT->cuXn z5^0Anv)3~Q9@D0IfeT6`#s}cb%^?K#yjkkfof-NjzJ%==)TG^x-%o_A7i!%B-P*=B z6kfH^_~;be#;KZrg_u;>S;)^k(YywIZdb^3t%VH6xqqjUpL2a|7yN;KKZJd|QY(1CS zBLxxzS#z-NOGNlmU}I^sZ!lyI(kTcTs-1`l^nt$gk2EaZTK&pAqka)LAM(<0b_ha1 zFf-&r0!b9&Kh>m37olUmWd~GQCr)-*VcsbZND}Xjc!*+}G5?Y#vZz7Eh0I_7b4G z_iclGUX5$#mr`I{cG3?^aJ)OCYn$=CUdU?M{Z2Q{-3N6^8u5`n#S+JfJvxo|XtHc* zWK%Iio`T978yE3;t)>GY{iREH8)|9m9QO8n`)5_SIZtn@8O__+>Qk(BDQk4q(*z_B z=mv!dg-hPQ#(`ec_;qjqLoou`FcY-PwwXyYT*>*O>|qEP|wf4Z)@MR;4aPR zLL2x9tqI`ezCcrtAhhM0|%#+AIJ)GK#d?Y@;K z3r4F?vr0ytZzzi|=otIyi!0+kX}S}R^Gqw+2soi$Y}(M%7F3c6VWKR3MQBkocbKtCwZ4Sh$Z2 zLuWwy)xdz0@nQyc;7le-NZqVE(_(9Q?^mTUG{MEP18H+oYFf4~mfQ%NpspdOQ++ z13_jg%yN+z%Md&-07-gOUWU#Mm@Uk;ncP)osKQju?F~vPUkhK}24%06B&6sr^W<+g ziR!k6-(&MK|8PqiJQN#g*Z79w8;?bk{@21=o(XEtO)LrvBYp>+NV@ymo-W^3Seefq zqTznF`pzqywaGH?U7?4AVVucGZXy01ag6n7m44Z?z6N)fSaj`MmcHedc+NLnS^H>| z^q|V!z)Q|H5uvRq4T*>mjc<1#c^xNqO)>YLG@4Yig2a5cn4v&rq4v2J0j3vwEhI3h za6t@TMPdxFH*F@80%G6Q^;d*B(S%Q)!JQl60&myntDE zi=Lt(DcHwztVeFQ!e8RM8I+^rwqt3g2e?o2_=0wDEnfT4E+a+P19p~@zSIuA)89Ppdl2?RC0|`)&8p!2G75-DCIT6i9~$!q$t5Qh@Y4!=NXR;I z8{il(7ge>IIQ1Ej@=L#( z(@iwf39wR@O?+%(o!bn)<%3Y+B`DrMY$ZLye;TfW*Q4bm#L->4GB^I_X}?h1!qf#{ z0iG|O>iY337P=w1uH2RbO>fsseSU`uAKWK0$3QA&y)t&g=;2iFS7Jjxk<`>2NCWudgMe3?+&}>KuYnff0mh`;-=uGLm@&TcbCh;1> zG`s~P7N;_d-UEaCjL=qifS4Z4`t8szZ0DiZh(4z3X;d1$(K_2s8U79~v8J{+dh3K) z`q6I2>V3{N#0M(Vq06>a5yabJ2`Kwhs%5dTxn~ixhg&nYGm>0o&&!@|VsMTT;!FYc1jtZZzI5;LpHsn(i+XJ@HNU-F3V=ayK|WOvmw9!7V2$wdq$&X z>L9Nso2ItfU>q_FU2-6h95tt=xV-xVdb|H_ZtfF`e4Z)g(5v?&CprNhF7NvyE7HN*!FE%pKz`x z-Go1dwF3zymrVREX6F>gevaaGj?}hC(s{KE0t$M$Z9BLq{y;dF(})I7(>_^k0=?R3 zlN6(n*wjSRkVMG~=hP)g-V{hJ=OE*LQ(3*ULP>RK-H5(if^vf&rx zXRfq1$kx|g3+R@rA+?2D0O(&YlFZ+7vu zvo|*0NMYP~sWA4^Z8*sH&ck}pBf-4Eg!ph?9~tJ0jXlVvZlpl$kDlQ-PwqTSYV=Pn z+{3n{DnR z`HYV9thC2Du;TsAkJ2iZ2;QKKj+~S95(yyg7m7hQEhIV(q?CS||IC|_au2ATy(=jx zaXi_YDy^@te>DuEjf6ThdCn7wU#0u=bk1DAn%Tm)%ckC$ca6GhH0l*GbcUjoyia+l zD-Tn28`?B9zKs?-|Cw4ppthDfEM#`n&di&}1McT7-|c(Ri@Upt-!*gXxRoUovge%i zRm6t4;LX56_ow}k5gS9Bh#xd7&*rG1cmPjnlIwt1dtJyFK++;Krfk@?^Vo^s zXC2RnZSky0znK$GV*Wsx>0*xD@Gy4I@oXWE^unvui|Za%0rjZ!GOfW2eFNE?t*)gD zChH3pH+`1~<3`!!ePjru~#e#h8=>G7W^+k3AE+e@F~y(Ig31R`>b4x#ec zU9A;6Yoeu<_;y$WO%d?bT>u&#)bVW$%@kzQ1d+C(L$rqww?@4wW$FSlHIa-{K z;hu+2ycLn>pswBdCz{QZo*j#Mtt#t_M|+^+xu$3*`lAg8!F>_q5HdlAaLoP|EjFR2 zYhRa++G_D;;4ZU8AYx+|+)WZn)=?_`uFOP-P7-qCrtYC zX+HmyjB8BM4w*q~vvtz4{`Pki=;KSnAy*3E7@6LL4vXO()@_*aT4uXR`lh7Z*)KTx z{Oh(=#+NkT(T?ImYHnidjo;;pRfo1sbMpgt26KfmU1WO`1bPTA-Q2V$V608O#*%8h z=W=3sjGUb$4j%C*0Q}$tP?I5itZ$Z1ahXhTsNh(~!`^U* zT6arzE06LMxNFn&V#XZX>(7B-nPo8`~qPEc9sa6;a2d$6=Ngq3+epDy_XJZXh)r z&EKYl49$AyS{&y*C2TyXc6qn*PVX~53ooinJ)PxmHPu*xF&tkOc;v``!a57ZATM=8T8c$<> zVsd(~TJC>I|JJrA%Lt!7C`ee-`h7)Y(w%v$S3vYs$o@@4_ybJ?gJ*Rz@}K#w-nO{k z&+;q&n*i2BRCm5@<*%%Y_DP(Wm{jmgk5BKhmv$(hA5q?+5aH$MBP}%9uWtfh>KJar zb5eD@b~O-3KT6%w5mW3NpAP-VFdZmDSWP+Baub}V+{=m1&E?t{}bT&e6ec$u?K|a*KHaIjP`YdMU%b9WKsJ zj}{+?jb6&wd~+MXN%^+@4rgA7l-HT87|~?Vq89919^K=|_~YK*upCf`rWI^kS4#}8 zs(jyyv=g-jFTAI!pcSNauw#_UCIuRJNNHm%qou01r7v1fW9w$075k1kLNq37$YpM_ zkd}Luc;DjKN7kPvrxfiSX~^q%CAM$GRvh8bv$@IsC zjMD#Zy4D)s--`&x7&6@{u_2e~vZ(ySK35c?ak&j0L;#Hil`hq^4`zf~`KhX+Z+#;= z-IH?y3Sk=25v+zd?U{a4K^(R~CTFW5`rde$-;|%F6M8a;NC_#DbdC2k!$|Jizg`bT zJRo0c;1%!w5BCEQ4&juyyONRp19>rU15>wISDOCU1uSI-zh0zpMxYl-RaaB?jiZY_Eo+TlS4->f8H4uS6VvMm@&?@b_!KfaaenmbXb;`3>K3?ImE_*6c-zNrcSd{)lSa?3XF&$Sz! z@&jZx#rr-|`2Q5#O$jKtQzU)$Uys)y7*KHMDTcsJ!`BUPkHH*J@Wah_sd)dvY)LHv z{XA6j*rWgH3cKYW1t174i`#5&VAp@X6b(skFP3I zaU!AJei!0G#!qS-;4J|PfAr6M3<}u(TFwCOca-+C!v;=g%Sb8*1b2=@2N6S@96)3= zWJOX*z&%}CT~q!2Ecma2ngEKMs`B~Qm*ciW@RI71f)3vB2i)+J@sDP91prRx8&<@W z+;fR<_HTs%o%$OP;#|YY5v6kVZUa?%OI+|BvRo;gc7Uv@EOJUFr#gn@kKzH6H3RS0 z!B8h|wJvQU18>e^&7tZTU>I(gnhiL+)*1NQp*OHVJj{jpsWo@*9y`6vq0T928^?rXRUsA39$BO-@iQew(TQ7PD&1g6PLj@4+t&&(7 zvE_gHZmSvMKFu51z2#Gp4~z5A$;CQ;1xi+LddQvC5iQ) z;n8D2JvO^i;~5A6)IdeaL$aHGxHvV;{&joulK-Yr{pZsDE#tom_+PI28xX%Ol3yhd z+xuZT)*bpk0VZg0X#k=D`ckVVln!UrSM991ir#YHpJ3SWt z8&W|)atFZ52f))7{(eCzo|O zT@k6?8PMf%EX*zDuIO&OiW0+Gn9#iUv#b#Ozpm`R7;DWGvJ?02>7KMu2-Ra7zoB~J zvwXI&)4HU9mH9NBFG%kr>0k1Te^`{*Ng{S#Fj-Muqq#w@j4J3!KLzuDr0S&tz?BUH z(&;%dn}|R|As-1xF3R`zSV&It@+WU>?v}qY;I%j&$)og$zkJ?{5F89)BQn!``iJ}1 zPf8uQ>7^WVNTnu~t0KKA#nNv`d!Kf^$PA~#;KL*1n}JW=K%_xgZ14`0dkTEv(Sl`G zJWlhN-sWAVcH5U9FK(Q2K0wgGb?K8{M6T?OPxHLomW~&nQ(5Dfg_PK=QDBNBKK%yx z$^Zf6I<0#Hp^QEN8lN1t$;R^Cd}@gIKHe^YNqVm9FYzR*RTk@k;D zEz)lRyYmu-)Nm>nEQ&Q&K<4Y)d8_BBraqzL4XlmO@JBfN-{d*(~nF;wNet9$^Jrg6kv-Srw zIAuxUF|5Q^50F*33F{J33pu(1-RC^uJ|=7eI-2D?v5q1l>1_yrNDau16+2KA@&-Hbe zqK;WUCtod{XID!$%|<@Y^fKJOdmpFq%WUQgOuz_+03!(Bf!pgy2wmIB{ghWx{Q2He znyD8nY0(}ro~(wX1(w?B@5X3yvivdl0bUG*k-VQb>-qPgw47lLLATk)>npU8ib^D+ z+*G#ErFI?)5%M|$sxAwy7f-?~c)b_!c4X;MaiRe9R zEw2L3#MjO}Y+M^$*D{--t1FF^E5B~13BR|>vA@^(^I2~Bh?;2qt~XNj&d8cPbSYZs zZr1~`J*m^H)8=Lq*v|wu7X#aAEIdt*{=S527tO}rIvqI zC3j@!*x)TVM4AH}B>AzxQuOgj-5@6;R_dUl0e^4w+i=j3Am@Lh&!Kmz+cUz&5#*3aeL(m9pgmacl5+>KOK5;um{u(!?u=HMLFsUuiIJ zT+nxtw|(~~ziSux=&+fP?5S9$Q?qUnDyD80c)lqJh}mp3YgzT5+Sjt-jt=KT#2%sm zP|yZHQiZNwX0dmg!Swumz<16)BZS%5YrWtC`iMI9?5f1cr7%j zZsP9yv7LHk=I=WJ0(LHk7aV<`Cw{JU7{I;j)1Fz!05d>i?q*(vL#hJ~QUPT69+N*P z{R{uvBckK95Wkfq7Qh0IZ}>c~`W{zcWAWz$R_I2S^)PdUOhTBeSlP#22Du4-zBVJ% z_Ayx=Ndwyq`@Jn48Mz6N;*|Z;-OkwcwoVB@KM%}_1`yD&XTP4e&ARk*OBc^2-zTsM ziFa@K3Kichu1fc8VM%AX&(W_gn+h=Q>IUTWzs|KSYo<)&0 z$8#~9&0ScIX*W?@4R`Taehn#cTCZlq0)rmq0mPn}UPaoC?njFOAsz}*-Qtvr={~5D ztewQ2VHI9Cg9qYAgFLczUYt6flIP0_8NWX0?k^{(yR)UglPLsmua%7B@iy{>QRHdn z$=@mNGxXZ7zHV+h#I4BdS$ucr?wu!;jEaP;{FHGxG9e7a_(6V-WM&stB$`T{CW$ZZ z>gEQ-^~r05hGWRK|7H(3^T<(iccjrI4B84Yam^8J`OVd@n{l}6di<>jEmAfQxkQQX zY3bDNG`VeZhF>=r%pv`tov(3`t-3 zPRx^NrXN7NMb)pQ0(##IXSsChCwBQi)+)DN_8{wD#tcOyMLK77+!xN|=~+^YqFY=~ z(gl4T@%kuL9Hvx;GH^z)J4}fVKGb#a_gd>a-EavNeONO~cRmDi@KG&~xFccWTq&_U%zG z+W`6ijlvup4D!rR@f8lNAbd*U@VJegAcw5=8~nr7 z&jFGSOGht;aY^?U@wIWPedX&ClRTC^dY;G^|gBXjH@ylLQE=E4C0mr5} z!RtX!P{95;s-s9qt?fKym9B75C`U!GIQ@X;Ip11leyym-PVo+)3=zkeqdwrQ}TyF0j|hT>}ktSAWhwk#%98=qt-hGnQI(~_Z`4Vdm6zwX2*<@_x;26 z%wT&r^k0h)?WjEx6Y2z`uo@jO1)j0o>Ett)RDX#{{1G4XUisA7gP7dlTbU1E=uEg8 zJNA$+@%3)YTC8}574eu>KM(1E4_sajRBPBp67^m_z2d=0-c&=0A<;5@e803>dLtUb z^qFebm1~JmsLe9$GGh_(Wb?^M7H_Tbq=8L<+2a`VR=^@NU6k{aJ3X-<1wKQjdTjLK zr%iX`?oiG@#v3p_er(sfE%58{R*khG4+d@`k%nKAbLDs<4wqW=aaT_ec=2P$YdUuu zk#M;R0kJ@+7~3hrJw|GC`eQ@MY<`daj2}f5SpX&HK9h>iJmnm~+a8gKH<{#^-+#mq zz*FpcUte|kaLu|prulL_GdAWLMDMkh*1>y3zh8AAL+sDQDvGps@% zb_9F5%w3{G21!iaW_5cxqlLpa%#s+El?B8h3~)*|jNiIuERy(bqV1$c;U8?7xOmya zDxWj?DUwHiDCRHp2oq_GagaWkIutCXwZxG*NZU7;WBO@GRGc5rfPZJM0&qx0-R~9E zzUvhO-1Q!>5VcNDkm41M5IIw9DtkNyGu*?8gT`!C? zV7@}}@rO;4H?ewoj$z!t&Nb;|c&3*nK0=ojGKdNEq9Cu$e}EMP;lJv;C&`h3YN95L zU6#c`fIoSZ<|GO1N6dE;6+*!&-Mw-KAz?{oBCgk0mzo8ql%DV}VHB}G+-dU3-ePK$ z;R=Y?%JL0dtT!|zmirL!$TSd$?cW+$oM5$hzj{%lCv*^K$^1@otE6_WC}i8o6JY%F zip&K|>=r3WVOG8T@wj{HN@CN?^`eC>L5zvLU56qLCqm(@yojixT_Of?T|>F_0zs(e zytNSv0gsxMDVECCInP-~HK=znL75naOs!Lver=u{$XG|o@_NG1wT~52B9$#lw zrZ3h8xO_fCU=>syi_^RelxkmK2vU}#Y-LlIbD>=<9`ZpxV&b{~B=jxkHrup&$;Z51 z-sX#^begUfk)NFL%DjJ#LTin59s!sk3#h zpb+CL2q}zYxPX^M)JP4&!{5amowgC_B>h}nr0^T zK{79f+Nfc@tm*qIyb8RLqVa~>)#bQFSY~Ik!n`^qs&$57LacDzCi3y?9rIX#XXzLS zCV4<_t|*eh)s(w@<_otBHMfD~KDiCMuTE=7E>Ng{h+a7yk7;DYz*PsFT{WYVYoof{(X16&_O1m`M48 zX+Mpd?{*S1J7C$c@97N@?3?C?Fszo^k+JR@gX=yT|W)-ThI`VOGVVPx>y5&@4S zus_B5VXX%Ku*KmkWI!%8N4xElhfy`sIAoTv25C9ze~a6J7?^)-w8#hWmm*J`UDc{gIHTh8Ht=^XDZ$-AT(p_%mszj);WrnS`GRjo^hx6VF7V<^-C#? znwsW$7>!T6ZnR-g36|9Zm;#OM&4Ym3#F(9aNg9>E`iR1$JYML%-*~4LxKb}B+4=M< z5=HZ8lzM1jA=zD1mm-8H1=xLOvcs4o9}hX7I{5y12H!kz`|#2Sd9`ik)MOQRY1<6O z+CQ~~8E-pJHUs~Gfm{3%*u1OHXKCD+{Me$t{!j|%3)~^`R(}NCQ#6soy5DL{BuL#) zjDbZdsH}0N+~qWQsr++F?ayqVVu+i33PIW>OJTK^S!qDYjTuqnTzuHT{tL)Z`%yNw zebJfE*>hZmh)J4HXI$}Lu8_^w)Yhlfq5awD3a093H{aV#_QSNFg`&>AodXW0p1B}f zvv>*$^;E}i-YUq8s6$oJ|D9|zFq_R&#K3l%sQB~RtZ|A2Ku&x@SWy^hY3l2df7AwC zDSr(F#^XIHF6n;-iGkaP?_cp!8hU+s%C;I3!OQe#c>iwx``OKe|0?lDF8}}IjSc=E f$w99f>+05L4#w;4UheUm7uzezsl6zbF$wq|c*1md diff --git a/docs/solutions/ha-setup-apt.md b/docs/solutions/ha-setup-apt.md index af5c88947..9c9638e79 100644 --- a/docs/solutions/ha-setup-apt.md +++ b/docs/solutions/ha-setup-apt.md @@ -2,273 +2,369 @@ This guide provides instructions on how to set up a highly available PostgreSQL cluster with Patroni on Debian or Ubuntu. +## Considerations -## Preconditions +1. This is the example deployment suitable to be used for testing purposes in non-production environments. +2. In this setup ETCD resides on the same hosts as Patroni. In production, consider deploying ETCD cluster on dedicated hosts or at least have separate disks for ETCD and PostgreSQL. This is because ETCD writes every request from the cluster to disk which can be CPU intensive and affects disk performance. See [hardware recommendations](https://etcd.io/docs/v3.6/op-guide/hardware/) for details. +3. For this setup, we will use the nodes running on Ubuntu 22.04 as the base operating system: -For this setup, we will use the nodes running on Ubuntu 20.04 as the base operating system and having the following IP addresses: - -| Node name | Public IP address | Internal IP address -|---------------|-------------------|-------------------- -| node1 | 157.230.42.174 | 10.104.0.7 -| node2 | 68.183.177.183 | 10.104.0.2 -| node3 | 165.22.62.167 | 10.104.0.8 -| HAProxy-demo | 134.209.111.138 | 10.104.0.6 + | Node name | Application | IP address + |---------------|-------------------|-------------------- + | node1 | Patroni, PostgreSQL, ETCD | 10.104.0.1 + | node2 | Patroni, PostgreSQL, ETCD | 10.104.0.2 + | node3 | Patroni, PostgreSQL, ETCD | 10.104.0.3 + | HAProxy-demo | HAProxy | 10.104.0.6 !!! note - In a production (or even non-production) setup, the PostgreSQL nodes will be within a private subnet without any public connectivity to the Internet, and the HAProxy will be in a different subnet that allows client traffic coming only from a selected IP range. To keep things simple, we have implemented this architecture in a DigitalOcean VPS environment, and each node can access the other by its internal, private IP. + Ideally, in a production (or even non-production) setup, the PostgreSQL nodes will be within a private subnet without any public connectivity to the Internet, and the HAProxy will be in a different subnet that allows client traffic coming only from a selected IP range. To keep things simple, we have implemented this architecture in a private environment, and each node can access the other by its internal, private IP. -### Setting up hostnames in the `/etc/hosts` file +## Initial setup -To make the nodes aware of each other and allow their seamless communication, resolve their hostnames to their public IP addresses. Modify the `/etc/hosts` file of each node as follows: +### Set up hostnames in the `/etc/hosts` file -| node 1 | node 2 | node 3 -|---------------------------| --------------------------|----------------------- -| 127.0.0.1 localhost node1
10.104.0.7 node1
**10.104.0.2 node2**
**10.104.0.8 node3**
| 127.0.0.1 localhost node2
**10.104.0.7 node1**
10.104.0.2 node2
**10.104.0.8 node3**
| 127.0.0.1 localhost node3
**10.104.0.7 node1**
**10.104.0.2 node2**
10.104.0.8 node3
+It's not necessary to have name resolution, but it makes the whole setup more readable and less error prone. Here, instead of configuring a DNS, we use a local name resolution by updating the file `/etc/hosts`. By resolving their hostnames to their IP addresses, we make the nodes aware of each other's names and allow their seamless communication. +1. Run the following command on each node. Change the node name to `node1`, `node2` and `node3` respectively: -The `/etc/hosts` file of the HAProxy-demo node looks like the following: + ```{.bash data-prompt="$"} + $ sudo hostnamectl set-hostname node-1 + ``` -``` -127.0.1.1 HAProxy-demo HAProxy-demo -127.0.0.1 localhost -10.104.0.6 HAProxy-demo -10.104.0.7 node1 -10.104.0.2 node2 -10.104.0.8 node3 -``` +2. Modify the `/etc/hosts` file of each PostgreSQL node to include the hostnames and IP addresses of the remaining nodes. Add the following at the end of the `/etc/hosts` file on all nodes: + + === "node1" + + ```text hl_lines="3 4" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + + === "node2" + + ```text hl_lines="2 4" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + + === "node3" + + ```text hl_lines="2 3" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + + === "HAproxy-demo" + + The HAProxy instance should have the name resolution for all the three nodes in its `/etc/hosts` file. Add the following lines at the end of the file: + + ```text hl_lines="4 5 6" + # Cluster IP and names + 10.104.0.6 HAProxy-demo + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + + +### Install the software + +Run the following commands on node1`, `node2` and `node3`: + +1. Install Percona Distribution for PostgreSQL + + * [Install `percona-release`](https://www.percona.com/doc/percona-repo-config/installing.html). -### Install Percona Distribution for PostgreSQL + * Enable the repository: -1. Follow the [installation instructions](../installing.md#on-debian-and-ubuntu-using-apt) to install Percona Distribution for PostgreSQL on `node1`, `node2` and `node3`. + ```{.bash data-prompt="$"} + $ sudo percona-release setup ppg15 + ``` + * [Install Percona Distribution for PostgreSQL packages](../apt.md). -2. Remove the data directory. Patroni requires a clean environment to initialize a new cluster. Use the following commands to stop the PostgreSQL service and then remove the data directory: +2. Install some Python and auxiliary packages to help with Patroni and ETCD + + ``` + {.bash data-prompt="$"} + $ sudo apt install python3-pip python3-dev binutils + ``` - ```{.bash data-promp="$"} - $ sudo systemctl stop postgresql - $ sudo rm -rf /var/lib/postgresql/14/main +3. Install ETCD, Patroni, pgBackRest packages: + + ```{.bash data-prompt="$"} + $ sudo apt install percona-patroni \ + etcd etcd-server etcd-client \ + percona-pgbackrest + ``` + +4. Stop and disable all installed services: + + ```{.bash data-prompt="$"} + $ sudo systemctl stop {etcd,patroni,postgresql} + $ systemctl disable {etcd,patroni,postgresql} + ``` + +5. Even though Patroni can use an existing Postgres installation, remove the data directory to force it to initialize a new Postgres cluster instance. + + ```{.bash data-prompt="$"} + $ sudo rm -rf /var/lib/postgresql/15/main ``` ## Configure ETCD distributed store -The distributed configuration store helps establish a consensus among nodes during a failover and will manage the configuration for the three PostgreSQL instances. Although Patroni can work with other distributed consensus stores (i.e., Zookeeper, Consul, etc.), the most commonly used one is `etcd`. +The distributed configuration store provides a reliable way to store data that needs to be accessed by large scale distributed systems. The most popular implementation of the distributed configuration store is ETCD. ETCD is deployed as a cluster for fault-tolerance and requires an odd number of members (n/2+1) to agree on updates to the cluster state. An ETCD cluster helps establish a consensus among nodes during a failover and manages the configuration for the three PostgreSQL instances. The `etcd` cluster is first started in one node and then the subsequent nodes are added to the first node using the `add `command. The configuration is stored in the `/etc/default/etcd` file. -1. Install `etcd` on every PostgreSQL node using the following command: +### Configure `node1` + +1. Back up the configuration file ```{.bash data-promp="$"} - $ sudo apt install etcd + $ sudo mv /etc/default/etcd /etc/default/etcd.orig ``` -2. Modify the `/etc/default/etcd` configuration file on each node. +2. Export environment variables to simplify the config file creation - * On `node1`, add the IP address of `node1` to the `ETCD_INITIAL_CLUSTER` parameter. The configuration file looks as follows: + * Node name: - ```text - ETCD_NAME=node1 - ETCD_INITIAL_CLUSTER="node1=http://10.104.0.7:2380" - ETCD_INITIAL_CLUSTER_TOKEN="devops_token" - ETCD_INITIAL_CLUSTER_STATE="new" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.7:2380" - ETCD_DATA_DIR="/var/lib/etcd/postgresql" - ETCD_LISTEN_PEER_URLS="http://10.104.0.7:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.7:2379,http://localhost:2379" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.7:2379" - … - ``` + ```{.bash data-prompt="$"} + $ export NODE_NAME=`hostname -f` + ``` - * On `node2`, add the IP addresses of both `node1` and `node2` to the `ETCD_INITIAL_CLUSTER` parameter: + * Node IP: - ```text - ETCD_NAME=node2 - ETCD_INITIAL_CLUSTER="node1=http://10.104.0.7:2380,node2=http://10.104.0.2:2380" - ETCD_INITIAL_CLUSTER_TOKEN="devops_token" - ETCD_INITIAL_CLUSTER_STATE="existing" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.2:2380" - ETCD_DATA_DIR="/var/lib/etcd/postgresql" - ETCD_LISTEN_PEER_URLS="http://10.104.0.2:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.2:2379,http://localhost:2379" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.2:2379" - … - ``` + ```{.bash data-prompt="$"} + $ export NODE_IP=`hostname -i | awk '{print $1}'` + ``` + + * Initial cluster token for the ETCD cluster during bootstrap: - * On `node3`, the `ETCD_INITIAL_CLUSTER` parameter includes the IP addresses of all three nodes: + ```{.bash data-prompt="$"} + $ export ETCD_TOKEN='PostgreSQL_HA_Cluster_1' + ``` - ```text - ETCD_NAME=node3 - ETCD_INITIAL_CLUSTER="node1=http://10.104.0.7:2380,node2=http://10.104.0.2:2380,node3=http://10.104.0.8:2380" - ETCD_INITIAL_CLUSTER_TOKEN="devops_token" - ETCD_INITIAL_CLUSTER_STATE="existing" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.8:2380" - ETCD_DATA_DIR="/var/lib/etcd/postgresql" - ETCD_LISTEN_PEER_URLS="http://10.104.0.8:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.8:2379,http://localhost:2379" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.8:2379" - … - ``` + * ETCD data directory: -3. On `node1`, add `node2` and `node3` to the cluster using the `add` command: + ```{.bash data-prompt="$"} + $ export ETCD_DATA_DIR='/var/lib/etcd/postgresql' + ``` - ```{.bash data-promp="$"} - $ sudo etcdctl member add node2 http://10.104.0.2:2380 - $ sudo etcdctl member add node3 http://10.104.0.8:2380 +3. Modify the `/etc/default/etcd` configuration file as follows:. + + ```text + ETCD_NAME=${NODE_NAME} + ETCD_INITIAL_CLUSTER="${NODE_NAME}=http://${NODE_IP}:2380" + ETCD_INITIAL_CLUSTER_STATE="new" + ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" + … ``` -4. Restart the `etcd` service on `node2` and `node3`: +3. Start the `etcd` service to apply the changes on `node1`. - ```{.bash data-promp="$"} - $ sudo systemctl restart etcd + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now etcd + $ sudo systemctl start etcd + $ sudo systemctl status etcd ``` -5. Check the etcd cluster members. - - ```{.bash data-promp="$"} +4. Check the etcd cluster members on `node1`: + + ```{.bash data-prompt="$"} $ sudo etcdctl member list ``` + + Sample output: - The output resembles the following: + ```{.text .no-copy} + 21d50d7f768f153a: name=default peerURLs=http://10.104.0.1:2380 clientURLs=http://10.104.0.1:2379 isLeader=true + ``` +5. Add the `node2` to the cluster. Run the following command on `node1`: + + ```{.bash data-prompt="$"} + $ sudo etcdctl member add node2 http://10.104.0.2:2380 ``` - 21d50d7f768f153a: name=node1 peerURLs=http://10.104.0.7:2380 clientURLs=http://10.104.0.7:2379 isLeader=true - af4661d829a39112: name=node2 peerURLs=http://10.104.0.2:2380 clientURLs=http://10.104.0.2:2379 isLeader=false - e3f3c0c1d12e9097: name=node3 peerURLs=http://10.104.0.8:2380 clientURLs=http://10.104.0.8:2379 isLeader=false + + The output resembles the following one: + + ```{.text .no-copy} + Added member named node2 with ID 10042578c504d052 to cluster + + ETCD_NAME="node2" + ETCD_INITIAL_CLUSTER="node2=http://10.104.0.2:2380,node1=http://10.104.0.1:2380" + ETCD_INITIAL_CLUSTER_STATE="existing" ``` -## Set up the watchdog service +### Configure `node2` -The Linux kernel uses the utility called a _watchdog_ to protect against an unresponsive system. The watchdog monitors a system for unrecoverable application errors, depleted system resources, etc., and initiates a reboot to safely return the system to a working state. The watchdog functionality is useful for servers that are intended to run without human intervention for a long time. Instead of users finding a hung server, the watchdog functionality can help maintain the service. +1. Back up the configuration file and export environment variables as described in steps 1-2 of the [`node1` configuration](#configure-node1) +2. Edit the `/etc/default/etcd` configuration file on `node2`. Use the result of the `add` command on `node1` to change the configuration file as follows: -In this example, we will configure _Softdog_ - a standard software implementation for watchdog that is shipped with Ubuntu 20.04. + ```text + ETCD_NAME=${NODE_NAME} + ETCD_INITIAL_CLUSTER="node-1=http://10.0.100.1:2380,node-2=http://10.0.100.2:2380" + ETCD_INITIAL_CLUSTER_STATE="existing" -Complete the following steps on all three PostgreSQL nodes to load and configure Softdog. + ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" + ``` -1. Load Softdog: +3. Start the `etcd` service to apply the changes on `node2`: - ```{.bash data-promp="$"} - $ sudo sh -c 'echo "softdog" >> /etc/modules' + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now etcd + $ sudo systemctl start etcd + $ sudo systemctl status etcd ``` -2. Patroni will be interacting with the watchdog service. Since Patroni is run by the `postgres` user, this user must have access to Softdog. To make this happen, change the ownership of the `watchdog.rules` file to the `postgres` user: +### Configure `node3` + +1. Add `node3` to the cluster. **Run the following command on `node1`** - ``` {.bash data-promp="$"} - $ sudo sh -c 'echo "KERNEL==\"watchdog\", OWNER=\"postgres\", GROUP=\"postgres\"" >> /etc/udev/rules.d/61-watchdog.rules' + ```{.bash data-prompt="$"} + $ sudo etcdctl member add node3 http://10.104.0.3:2380 ``` -3. Remove Softdog from the blacklist. +2. On `node3`, back up the configuration file and export environment variables as described in steps 1-2 of the [`node1` configuration](#configure-node1) +3. Modify the `/etc/default/etcd` configuration file and add the output of the `add` command: - * Find out the files where Softdog is blacklisted: + ```text + ETCD_NAME=${NODE_NAME} + ETCD_INITIAL_CLUSTER="node1=http://10.104.0.1:2380,node2=http://10.104.0.2:2380,node3=http://10.104.0.3:2380" + ETCD_INITIAL_CLUSTER_STATE="existing" - ```{.bash data-promp="$"} - $ grep blacklist /lib/modprobe.d/* /etc/modprobe.d/* |grep softdog - ``` - - In our case, `modprobe `is blacklisting the Softdog: + ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" + … + ``` - ``` - /lib/modprobe.d/blacklist_linux_5.4.0-73-generic.conf:blacklist softdog - ``` - - * Remove the `blacklist softdog` line from the `/lib/modprobe.d/blacklist_linux_5.4.0-73-generic.conf` file. - * Restart the service +4. Start the `etcd` service on `node3`: - ```{.bash data-promp="$"} - $ sudo modprobe softdog - ``` + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now etcd + $ sudo systemctl start etcd + $ sudo systemctl status etcd + ``` + +5. Check the etcd cluster members. - * Verify the `modprobe` is working correctly by running the `lsmod `command: - - ```{.bash data-promp="$"} - $ sudo lsmod | grep softdog - ``` - - The output will show a process identifier if it’s running. + ```{.bash data-prompt="$"} + $ sudo etcdctl member list + ``` - ``` - softdog 16384 0 - ``` + The output resembles the following: + + ``` + 2d346bd3ae7f07c4: name=node2 peerURLs=http://10.104.0.2:2380 clientURLs=http://10.104.0.2:2379 isLeader=false + 8bacb519ebdee8db: name=node3 peerURLs=http://10.104.0.3:2380 clientURLs=http://10.104.0.3:2379 isLeader=false + c5f52ea2ade25e1b: name=node1 peerURLs=http://10.104.0.1:2380 clientURLs=http://10.104.0.1:2379 isLeader=true + ``` -4. Check that the Softdog files under the `/dev/ `folder are owned by the `postgres `user: +## Configure Patroni -```{.bash data-promp="$"} -$ ls -l /dev/watchdog* +Run the following commands on all nodes. You can do this in parallel: -crw-rw---- 1 postgres postgres 10, 130 Sep 11 12:53 /dev/watchdog -crw------- 1 root root 245, 0 Sep 11 12:53 /dev/watchdog0 -``` +1. Export and create environment variables to simplify the config file creation: + * Node name: -!!! tip + ```{.bash data-prompt="$"} + $ export NODE_NAME=`hostname -f` + ``` - If the ownership has not been changed for any reason, run the following command to manually change it: - - ```{.bash data-promp="$"} - $ sudo chown postgres:postgres /dev/watchdog* - ``` + * Node IP: -## Configure Patroni + ```{.bash data-prompt="$"} + $ export NODE_IP=`hostname -i | awk '{print $1}'` + ``` + + * Create variables to store the PATH: -1. Install Patroni on every PostgreSQL node: + ```bash + DATA_DIR="/var/lib/postgresql/15/main" + PG_BIN_DIR="/usr/lib/postgresql/15/bin" + ``` - ```{.bash data-promp="$"} - $ sudo apt install percona-patroni - ``` + **NOTE**: Check the path to the data and bin folders on your operating system and change it for the variables accordingly. + + * Patroni information: -2. Create the `patroni.yml` configuration file under the `/etc/patroni` directory. The file holds the default configuration values for a PostgreSQL cluster and will reflect the current cluster setup. + ```bash + NAMESPACE="percona_lab" + SCOPE="cluster_1 + ``` -3. Add the following configuration for `node1`: +2. Create the `/etc/patroni/patroni.yml` configuration file and add the following configuration for `node1`: - ```yaml - scope: cluster_1 - namespace: percona_lab - name: node1 + ```yaml title="/etc/patroni/patroni.yml" + namespace: ${NAMESPACE} + scope: ${SCOPE} + name: ${NODE_NAME} restapi: - listen: 0.0.0.0:8008 - connect_address: 10.104.0.1:8008 + listen: 0.0.0.0:8008 + connect_address: ${NODE_IP}:8008 etcd: - host: 10.104.0.1:2379 + host: ${NODE_IP}:2379 bootstrap: # this section will be written into Etcd:///config after initializing new cluster dcs: - ttl: 30 - loop_wait: 10 - retry_timeout: 10 - maximum_lag_on_failover: 1048576 - slots: + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + slots: percona_cluster_1: type: physical - postgresql: - use_pg_rewind: true - use_slots: true - parameters: - wal_level: replica - hot_standby: "on" - wal_keep_segments: 10 - max_wal_senders: 5 - max_replication_slots: 10 - wal_log_hints: "on" - logging_collector: 'on' + postgresql: + use_pg_rewind: true + use_slots: true + parameters: + wal_level: replica + hot_standby: "on" + wal_keep_segments: 10 + max_wal_senders: 5 + max_replication_slots: 10 + wal_log_hints: "on" + logging_collector: 'on' # some desired options for 'initdb' - initdb: # Note: It needs to be a list (some options need values, others are switches) - - encoding: UTF8 - - data-checksums - - pg_hba: # Add following lines to pg_hba.conf after running 'initdb' - - host replication replicator 127.0.0.1/32 trust - - host replication replicator 0.0.0.0/0 md5 - - host all all 0.0.0.0/0 md5 - - host all all ::0/0 md5 - - # Additional script to be launched after initial cluster creation (will be passed the connection URL as parameter) - # post_init: /usr/local/bin/setup_cluster.sh - # Some additional users users which needs to be created after initializing new cluster + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 trust + - host replication replicator 0.0.0.0/0 md5 + - host all all 0.0.0.0/0 md5 + - host all all ::0/0 md5 + + # Some additional users which needs to be created after initializing new cluster users: admin: password: qaz123 @@ -280,13 +376,13 @@ crw------- 1 root root 245, 0 Sep 11 12:53 /dev/watchdog0 options: - createrole - createdb - + postgresql: cluster_name: cluster_1 listen: 0.0.0.0:5432 - connect_address: 10.104.0.1:5432 - data_dir: /data/pgsql - bin_dir: /usr/pgsql-15/bin + connect_address: ${NODE_IP}:5432 + data_dir: ${DATADIR} + bin_dir: ${PG_BIN_DIR} pgpass: /tmp/pgpass authentication: replication: @@ -301,11 +397,6 @@ crw------- 1 root root 245, 0 Sep 11 12:53 /dev/watchdog0 - basebackup basebackup: checkpoint: 'fast' - - watchdog: - mode: required # Allowed values: off, automatic, required - device: /dev/watchdog - safety_margin: 5 tags: nofailover: false @@ -314,92 +405,111 @@ crw------- 1 root root 245, 0 Sep 11 12:53 /dev/watchdog0 nosync: false ``` - !!! admonition "Patroni configuration file" + ??? admonition "Patroni configuration file" Let’s take a moment to understand the contents of the `patroni.yml` file. - The first section provides the details of the first node (`node1`) and its connection ports. After that, we have the `etcd` service and its port details. + The first section provides the details of the node and its connection ports. After that, we have the `etcd` service and its port details. Following these, there is a `bootstrap` section that contains the PostgreSQL configurations and the steps to run once the database is initialized. The `pg_hba.conf` entries specify all the other nodes that can connect to this node and their authentication mechanism. -4. Create the configuration files for `node2` and `node3`. Replace the reference to `node1` with `node2` and `node3`, respectively. -5. Enable and restart the patroni service on every node. Use the following commands: +3. Check that the systemd unit file `patroni.service` is created in `/etc/systemd/system`. If it is created, skip this step. - ```{.bash data-promp="$"} - $ sudo systemctl enable patroni + If it's **not** created, create it manually and specify the following contents within: + + ```ini title="/etc/systemd/system/patroni.service" + [Unit] + Description=Runners to orchestrate a high-availability PostgreSQL + After=syslog.target network.target + + [Service] + Type=simple + + User=postgres + Group=postgres + + # Start the patroni process + ExecStart=/bin/patroni /etc/patroni/patroni.yml + + # Send HUP to reload from patroni.yml + ExecReload=/bin/kill -s HUP $MAINPID + + # only kill the patroni process, not its children, so it will gracefully stop postgres + KillMode=process + + # Give a reasonable amount of time for the server to start up/shut down + TimeoutSec=30 + + # Do not restart the service if it crashes, we want to manually inspect database on failure + Restart=no + + [Install] + WantedBy=multi-user.target + ``` + +4. Make systemd aware of the new service: + + ```{.bash data-prompt="$"} + $ sudo systemctl daemon-reload + ``` + +5. Now it's time to start Patroni. You need the following commands on all nodes but not in parallel. Start with the `node1` first, wait for the service to come to live, and then proceed with the other nodes one-by-one, always waiting for them to sync with the primary node: + + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now patroni $ sudo systemctl restart patroni ``` When Patroni starts, it initializes PostgreSQL (because the service is not currently running and the data directory is empty) following the directives in the bootstrap section of the configuration file. -!!! admonition "Troubleshooting Patroni" +6. Check the service to see if there are errors: - To ensure that Patroni has started properly, check the logs using the following command: - - ```{.bash data-promp="$"} - $ sudo journalctl -u patroni.service -n 100 -f + ```{.bash data-prompt="$"} + $ sudo journalctl -fu patroni ``` - The output shouldn't show any errors: + A common error is Patroni complaining about the lack of proper entries in the pg_hba.conf file. If you see such errors, you must manually add or fix the entries in that file and then restart the service. - ``` - … + Changing the patroni.yml file and restarting the service will not have any effect here because the bootstrap section specifies the configuration to apply when PostgreSQL is first started in the node. It will not repeat the process even if the Patroni configuration file is modified and the service is restarted. - Sep 23 12:50:21 node01 systemd[1]: Started PostgreSQL high-availability manager. - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,022 INFO: Selected new etcd server http://10.104.0.2:2379 - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,029 INFO: No PostgreSQL configuration items changed, nothing to reload. - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,168 INFO: Lock owner: None; I am node1 - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,177 INFO: trying to bootstrap a new cluster - Sep 23 12:50:22 node01 patroni[10140]: The files belonging to this database system will be owned by user "postgres". - Sep 23 12:50:22 node01 patroni[10140]: This user must also own the server process. - Sep 23 12:50:22 node01 patroni[10140]: The database cluster will be initialized with locale "C.UTF-8". - Sep 23 12:50:22 node01 patroni[10140]: The default text search configuration will be set to "english". - Sep 23 12:50:22 node01 patroni[10140]: Data page checksums are enabled. - Sep 23 12:50:22 node01 patroni[10140]: creating directory /var/lib/postgresql/12/main ... ok - Sep 23 12:50:22 node01 patroni[10140]: creating subdirectories ... ok - Sep 23 12:50:22 node01 patroni[10140]: selecting dynamic shared memory implementation ... posix - Sep 23 12:50:22 node01 patroni[10140]: selecting default max_connections ... 100 - Sep 23 12:50:22 node01 patroni[10140]: selecting default shared_buffers ... 128MB - Sep 23 12:50:22 node01 patroni[10140]: selecting default time zone ... Etc/UTC - Sep 23 12:50:22 node01 patroni[10140]: creating configuration files ... ok - Sep 23 12:50:22 node01 patroni[10140]: running bootstrap script ... ok - Sep 23 12:50:23 node01 patroni[10140]: performing post-bootstrap initialization ... ok - Sep 23 12:50:23 node01 patroni[10140]: syncing data to disk ... ok - Sep 23 12:50:23 node01 patroni[10140]: initdb: warning: enabling "trust" authentication for local connections - Sep 23 12:50:23 node01 patroni[10140]: You can change this by editing pg_hba.conf or using the option -A, or - Sep 23 12:50:23 node01 patroni[10140]: --auth-local and --auth-host, the next time you run initdb. - Sep 23 12:50:23 node01 patroni[10140]: Success. You can now start the database server using: - Sep 23 12:50:23 node01 patroni[10140]: /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/main -l logfile start - Sep 23 12:50:23 node01 patroni[10156]: 2021-09-23 12:50:23.672 UTC [10156] LOG: redirecting log output to logging collector process - Sep 23 12:50:23 node01 patroni[10156]: 2021-09-23 12:50:23.672 UTC [10156] HINT: Future log output will appear in directory "log". - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,694 INFO: postprimary pid=10156 - Sep 23 12:50:23 node01 patroni[10165]: localhost:5432 - accepting connections - Sep 23 12:50:23 node01 patroni[10167]: localhost:5432 - accepting connections - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,743 INFO: establishing a new patroni connection to the postgres cluster - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,757 INFO: running post_bootstrap - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,767 INFO: Software Watchdog activated with 25 second timeout, timing slack 15 seconds - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,793 INFO: initialized a new cluster - Sep 23 12:50:33 node01 patroni[10119]: 2021-09-23 12:50:33,810 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:33 node01 patroni[10119]: 2021-09-23 12:50:33,899 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:43 node01 patroni[10119]: 2021-09-23 12:50:43,898 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:53 node01 patroni[10119]: 2021-09-23 12:50:53,894 INFO: no action. I am (node1) the leader with the +7. Check the cluster: + + ```{.bash data-prompt="$"} + $ patronictl -c /etc/patroni/patroni.yml list $SCOPE ``` - A common error is Patroni complaining about the lack of proper entries in the pg_hba.conf file. If you see such errors, you must manually add or fix the entries in that file and then restart the service. + The output on `node1` resembles the following: - Changing the patroni.yml file and restarting the service will not have any effect here because the bootstrap section specifies the configuration to apply when PostgreSQL is first started in the node. It will not repeat the process even if the Patroni configuration file is modified and the service is restarted. + ```{.text .no-copy} + + Cluster: cluster_1 --+---------+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +--------+-------------+---------+---------+----+-----------+ + | node-1 | 10.0.100.1 | Leader | running | 1 | | + +--------+-------------+---------+---------+----+-----------+ + ``` + + On the remaining nodes: + + ```{.text .no-copy} + + Cluster: cluster_1 --+---------+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +--------+-------------+---------+---------+----+-----------+ + | node-1 | 10.0.100.1 | Leader | running | 1 | | + | node-2 | 10.0.100.2 | Replica | running | 1 | 0 | + +--------+-------------+---------+---------+----+-----------+ + ``` If Patroni has started properly, you should be able to locally connect to a PostgreSQL node using the following command: -```{.bash data-promp="$"} +```{.bash data-prompt="$"} $ sudo psql -U postgres ``` -The command output looks like the following: +The command output is the following: ``` -psql (14.1) +psql (15.4) Type "help" for help. postgres=# @@ -407,13 +517,13 @@ postgres=# ## Configure HAProxy -HAProxy node will accept client connection requests and route those to the active node of the PostgreSQL cluster. This way, a client application doesn’t have to know what node in the underlying cluster is the current primary. All it needs to do is to access a single HAProxy URL and send its read/write requests there. Behind-the-scene, HAProxy routes the connection to a healthy node (as long as there is at least one healthy node available) and ensures that client application requests are never rejected. +HAproxy is the load balancer and the single point of entry to your PostgreSQL cluster for client applications. A client application accesses the HAPpoxy URL and sends its read/write requests there. Behind-the-scene, HAProxy routes write requests to the primary node and read requests - to the secondaries in a round-robin fashion so that no secondary instance is unnecessarily loaded. To make this happen, provide different ports in the HAProxy configuration file. In this deployment, writes are routed to port 5000 and reads - to port 5001 -HAProxy is capable of routing write requests to the primary node and read requests - to the secondaries in a round-robin fashion so that no secondary instance is unnecessarily loaded. To make this happen, provide different ports in the HAProxy configuration file. In this deployment, writes are routed to port 5000 and reads - to port 5001. +This way, a client application doesn’t know what node in the underlying cluster is the current primary. HAProxy sends connections to a healthy node (as long as there is at least one healthy node available) and ensures that client application requests are never rejected. 1. Install HAProxy on the `HAProxy-demo` node: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo apt install percona-haproxy ``` @@ -463,17 +573,16 @@ HAProxy is capable of routing write requests to the primary node and read reques 3. Restart HAProxy: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo systemctl restart haproxy ``` - 4. Check the HAProxy logs to see if there are any errors: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo journalctl -u haproxy.service -n 100 -f ``` -## Testing +## Next steps -See the [Testing PostgreSQL cluster](ha-test.md) for the guidelines on how to test your PostgreSQL cluster for replication, failure, switchover. \ No newline at end of file +[Configure pgBackRest](pgbackrest.md){.md-button} \ No newline at end of file diff --git a/docs/solutions/ha-setup-yum.md b/docs/solutions/ha-setup-yum.md index 1410e836b..9fb221645 100644 --- a/docs/solutions/ha-setup-yum.md +++ b/docs/solutions/ha-setup-yum.md @@ -3,7 +3,7 @@ This guide provides instructions on how to set up a highly available PostgreSQL cluster with Patroni on Red Hat Enterprise Linux or CentOS. -## Preconditions +## Considerations 1. This is the example deployment suitable to be used for testing purposes in non-production environments. 2. In this setup ETCD resides on the same hosts as Patroni. In production, consider deploying ETCD cluster on dedicated hosts because ETCD writes every request from the cluster to disk which requires significant amount of disk space. See [hardware recommendations](https://etcd.io/docs/v3.6/op-guide/hardware/) for details. @@ -16,73 +16,101 @@ This guide provides instructions on how to set up a highly available PostgreSQL | node3 | Patroni, PostgreSQL, ETCD | 10.104.0.3 | HAProxy-demo | HAProxy | 10.104.0.6 + !!! note Ideally, in a production (or even non-production) setup, the PostgreSQL and ETCD nodes will be within a private subnet without any public connectivity to the Internet, and the HAProxy will be in a different subnet that allows client traffic coming only from a selected IP range. To keep things simple, we have implemented this architecture in a private environment, and each node can access the other by its internal, private IP. -## Preparation +## Initial setup ### Set up hostnames in the `/etc/hosts` file It's not necessary to have name resolution, but it makes the whole setup more readable and less error prone. Here, instead of configuring a DNS, we use a local name resolution by updating the file `/etc/hosts`. By resolving their hostnames to their IP addresses, we make the nodes aware of each other's names and allow their seamless communication. -Modify the `/etc/hosts` file of each PostgreSQL node to include the hostnames and IP addresses of the remaining nodes. Add the following at the end of the `/etc/hosts` file on all nodes: - -=== "node1" +1. Run the following command on each node. Change the node name to `node1`, `node2` and `node3` respectively: - ```text hl_lines="3 4" - # Cluster IP and names - 10.104.0.1 node1 - 10.104.0.2 node2 - 10.104.0.3 node3 + ```{.bash data-prompt="$"} + $ sudo hostnamectl set-hostname node-1 ``` -=== "node2" +2. Modify the `/etc/hosts` file of each PostgreSQL node to include the hostnames and IP addresses of the remaining nodes. Add the following at the end of the `/etc/hosts` file on all nodes: - ```text hl_lines="2 4" - # Cluster IP and names - 10.104.0.1 node1 - 10.104.0.2 node2 - 10.104.0.3 node3 - ``` + === "node1" -=== "node3" + ```text hl_lines="3 4" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` - ```text hl_lines="2 3" - # Cluster IP and names - 10.104.0.1 node1 - 10.104.0.2 node2 - 10.104.0.3 node3 - ``` + === "node2" -=== "HAproxy-demo" + ```text hl_lines="2 4" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` - The HAProxy instance should have the name resolution for all the three nodes in its `/etc/hosts` file. Add the following lines at the end of the file: + === "node3" - ```text hl_lines="4 5 6" - # Cluster IP and names - 10.104.0.6 HAProxy-demo - 10.104.0.1 node1 - 10.104.0.2 node2 - 10.104.0.3 node3 - ``` + ```text hl_lines="2 3" + # Cluster IP and names + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + + === "HAproxy-demo" + + The HAProxy instance should have the name resolution for all the three nodes in its `/etc/hosts` file. Add the following lines at the end of the file: + + ```text hl_lines="4 5 6" + # Cluster IP and names + 10.104.0.6 HAProxy-demo + 10.104.0.1 node1 + 10.104.0.2 node2 + 10.104.0.3 node3 + ``` + +### Install the software + +1. Install Percona Distribution for PostgreSQL on `node1`, `node2` and `node3` from Percona repository: -## Install Percona Distribution for PostgreSQL + * [Install `percona-release`](https://www.percona.com/doc/percona-repo-config/installing.html). + * Enable the repository: -Install Percona Distribution for PostgreSQL on `node1`, `node2` and `node3` from Percona repository: + ```{.bash data-prompt="$"} + $ sudo percona-release setup ppg15 + ``` -1. [Install `percona-release`](https://www.percona.com/doc/percona-repo-config/installing.html). -2. Enable the repository: + * [Install Percona Distribution for PostgreSQL packages](../installing.md#on-red-hat-enterprise-linux-and-centos-using-yum). + !!! important + + **Don't** initialize the cluster and start the `postgresql` service. The cluster initialization and setup are handled by Patroni during the bootsrapping stage. + +2. Install some Python and auxiliary packages to help with Patroni and ETCD + ```{.bash data-prompt="$"} - $ sudo percona-release setup ppg14 + $ sudo yum install python3-pip python3-dev binutils ``` -3. [Install Percona Distribution for PostgreSQL packages](../yum.md). +3. Install ETCD, Patroni, pgBackRest packages: -!!! important + ```{.bash data-prompt="$"} + $ sudo yum install percona-patroni \ + etcd python3-python-etcd\ + percona-pgbackrest + ``` - **Don't** initialize the cluster and start the `postgresql` service. The cluster initialization and setup are handled by Patroni during the bootsrapping stage. +4. Stop and disable all installed services: + + ```{.bash data-prompt="$"} + $ sudo systemctl stop {etcd,patroni,postgresql} + $ systemctl disable {etcd,patroni,postgresql} + ``` ## Configure ETCD distributed store @@ -90,56 +118,66 @@ The distributed configuration store provides a reliable way to store data that n The `etcd` cluster is first started in one node and then the subsequent nodes are added to the first node using the `add `command. The configuration is stored in the `/etc/etcd/etcd.conf` configuration file. -1. Install `etcd` on every PostgreSQL node. For CentOS 8, the `etcd` packages are available from Percona repository: +### Configure `node1` - - [Install `percona-release`](https://www.percona.com/doc/percona-repo-config/installing.html). - - Enable the repository: +1. Backup the `etcd.conf` file: + + ```{.bash data-promp="$"} + $ sudo mv /etc/etcd/etcd.conf /etc/etcd/etcd.conf.orig + ``` - ```{.bash data-promp="$"} - $ sudo percona-release setup ppg14 - ``` - - - Install the etcd packages using the following command: +2. Export environment variables to simplify the config file creation + + * Node name: - ```{.bash data-promp="$"} - $ sudo yum install etcd python3-python-etcd - ``` + ```{.bash data-prompt="$"} + $ export NODE_NAME=`hostname -f` + ``` -2. Configure ETCD on `node1`. + * Node IP: - Backup the `etcd.conf` file: + ```{.bash data-prompt="$"} + $ export NODE_IP=`hostname -i | awk '{print $1}'` + ``` - ```{.bash data-promp="$"} - sudo mv /etc/etcd/etcd.conf /etc/etcd/etcd.conf.orig - ``` + * Initial cluster token for the ETCD cluster during bootstrap: + + ```{.bash data-prompt="$"} + $ export ETCD_TOKEN='PostgreSQL_HA_Cluster_1' + ``` + + * ETCD data directory: + + ```{.bash data-prompt="$"} + $ export ETCD_DATA_DIR='/var/lib/etcd/postgresql' + ``` - Modify the `/etc/etcd/etcd.conf` configuration file: +3. Modify the `/etc/etcd/etcd.conf` configuration file: ```text - [Member] - ETCD_DATA_DIR="/var/lib/etcd/default.etcd" - ETCD_LISTEN_PEER_URLS="http://10.104.0.1:2380,http://localhost:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.1:2379,http://localhost:2379" - - ETCD_NAME="node1" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.1:2380" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.1:2379" - ETCD_INITIAL_CLUSTER="node1=http://10.104.0.1:2380" - ETCD_INITIAL_CLUSTER_TOKEN="percona-etcd-cluster" + ETCD_NAME=${NODE_NAME} + ETCD_INITIAL_CLUSTER="${NODE_NAME}=http://${NODE_IP}:2380" ETCD_INITIAL_CLUSTER_STATE="new" + ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" + … ``` -3. Start the `etcd` to apply the changes on `node1`: +4. Start the `etcd` to apply the changes on `node1`: - ```{.bash data-promp="$"} - $ sudo systemctl enable etcd + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now etcd $ sudo systemctl start etcd $ sudo systemctl status etcd ``` 5. Check the etcd cluster members on `node1`. - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo etcdctl member list ``` @@ -155,7 +193,7 @@ The `etcd` cluster is first started in one node and then the subsequent nodes ar $ sudo etcdctl member add node2 http://10.104.0.2:2380 ``` - The output will be something similar to below one: + The output resembles the following one: ```{.text .no-copy} Added member named node2 with ID 10042578c504d052 to cluster @@ -165,60 +203,65 @@ The `etcd` cluster is first started in one node and then the subsequent nodes ar ETCD_INITIAL_CLUSTER_STATE="existing" ``` -7. Edit the `/etc/etcd/etcd.conf` configuration file on `node2` and add the output from step 6: +### Configure `node2` + +1. Back up the configuration file and export environment variables as described in steps 1-2 of the [`node1` configuration](#configure-node1) +2. Edit the `/etc/etcd/etcd.conf` configuration file on `node2` and add the output from the `add` command: ```text [Member] - ETCD_NAME="node2" - ETCD_INITIAL_CLUSTER="node2=http://10.104.0.2:2380,node1=http://10.104.0.1:2380" - ETCD_INITIAL_CLUSTER_STATE="existing" - ETCD_DATA_DIR="/var/lib/etcd/default.etcd" - ETCD_INITIAL_CLUSTER_TOKEN="percona-etcd-cluster" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.2:2380" - ETCD_LISTEN_PEER_URLS="http://10.104.0.2:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.2:2379,http://localhost:2379" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.2:2379" + ETCD_NAME=${NODE_NAME} + ETCD_INITIAL_CLUSTER="node-1=http://10.0.100.1:2380,node-2=http://10.0.100.2:2380" + ETCD_INITIAL_CLUSTER_STATE="existing" ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" ``` -8. Start the `etcd` to apply the changes on `node2`: +3. Start the `etcd` to apply the changes on `node2`: ```{.bash data-promp="$"} - $ sudo systemctl enable etcd + $ sudo systemctl enable --now etcd $ sudo systemctl start etcd $ sudo systemctl status etcd ``` -9. Add `node3` to the cluster. Run the following command on `node1`: +### Configure `node3` + +1. Add `node3` to the cluster. **Run the following command on `node1`**: ```{.bash data-prompt="$"} $ sudo etcdctl member add node3 http://10.104.0.3:2380 ``` -10. Configure `etcd` on `node3`. Edit the `/etc/etcd/etcd.conf` configuration file on `node3` and add the IP addresses of all three nodes to the `ETCD_INITIAL_CLUSTER` parameter: +2. On `node3`, back up the configuration file and export environment variables as described in steps 1-2 of the [`node1` configuration](#configure-node1) +3. Modify the `/etc/etcd/etcd.conf` configuration file on `node3` and add the output from the `add` command as follows: ```text - ETCD_NAME=node3 + ETCD_NAME=${NODE_NAME} ETCD_INITIAL_CLUSTER="node1=http://10.104.0.1:2380,node2=http://10.104.0.2:2380,node3=http://10.104.0.3:2380" ETCD_INITIAL_CLUSTER_STATE="existing" - ETCD_INITIAL_CLUSTER_TOKEN="percona-etcd-cluster" - ETCD_INITIAL_ADVERTISE_PEER_URLS="http://10.104.0.3:2380" - ETCD_DATA_DIR="/var/lib/etcd/postgresql" - ETCD_LISTEN_PEER_URLS="http://10.104.0.3:2380" - ETCD_LISTEN_CLIENT_URLS="http://10.104.0.3:2379,http://localhost:2379" - ETCD_ADVERTISE_CLIENT_URLS="http://10.104.0.3:2379" + ETCD_INITIAL_CLUSTER_TOKEN="${ETCD_TOKEN}" + ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${NODE_IP}:2380" + ETCD_DATA_DIR="${ETCD_DATA_DIR}" + ETCD_LISTEN_PEER_URLS="http://${NODE_IP}:2380" + ETCD_LISTEN_CLIENT_URLS="http://${NODE_IP}:2379,http://localhost:2379" + ETCD_ADVERTISE_CLIENT_URLS="http://${NODE_IP}:2379" … ``` -11. Start the `etcd` service on `node3`: +3. Start the `etcd` service on `node3`: ```{.bash data-prompt="$"} - $ sudo systemctl enable etcd + $ sudo systemctl enable --now etcd $ sudo systemctl start etcd $ sudo systemctl status etcd ``` -12. Check the etcd cluster members. +4. Check the etcd cluster members. ```{.bash data-prompt="$"} $ sudo etcdctl member list @@ -232,25 +275,46 @@ The `etcd` cluster is first started in one node and then the subsequent nodes ar c5f52ea2ade25e1b: name=node1 peerURLs=http://10.104.0.1:2380 clientURLs=http://10.104.0.1:2379 isLeader=true ``` + ## Configure Patroni -1. Install Patroni on every PostgreSQL node: +Run the following commands on all nodes. You can do this in parallel: - ```{.bash data-promp="$"} - $ sudo yum install percona-patroni - ``` +1. Export and create environment variables to simplify the config file creation: -2. Install the Python module that enables Patroni to communicate with ETCD. + * Node name: - ```{.bash data-promp="$"} - $ sudo python3 -m pip install patroni[etcd] - ``` + ```{.bash data-prompt="$"} + $ export NODE_NAME=`hostname -f` + ``` + + * Node IP: + + ```{.bash data-prompt="$"} + $ export NODE_IP=`hostname -i | awk '{print $1}'` + ``` -3. Create the directories required by Patroni + * Create variables to store the PATH: + + ```bash + DATA_DIR="/var/lib/pgsql/data/" + PG_BIN_DIR="/usr/pgsql-15/bin" + ``` + + **NOTE**: Check the path to the data and bin folders on your operating system and change it for the variables accordingly. + + * Patroni information: + + ```bash + NAMESPACE="percona_lab" + SCOPE="cluster_1 + ``` + +2. Create the directories required by Patroni * Create the directory to store the configuration file and make it owned by the `postgres` user. - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo mkdir -p /etc/patroni/ $ sudo chown -R postgres:postgres /etc/patroni/ ``` @@ -263,83 +327,87 @@ The `etcd` cluster is first started in one node and then the subsequent nodes ar $ sudo chmod 700 /data/pgsql ``` -4. Create the `/etc/patroni/patroni.yml` with the following configuration: +3. Create the `/etc/patroni/patroni.yml` with the following configuration: - ```yaml - namespace: percona_lab - scope: cluster_1 - name: node1 + ```yaml title="/etc/patroni/patroni.yml" + namespace: ${NAMESPACE} + scope: ${SCOPE} + name: ${NODE_NAME} restapi: - listen: 0.0.0.0:8008 - connect_address: 10.104.0.7:8008 + listen: 0.0.0.0:8008 + connect_address: ${NODE_IP}:8008 etcd: - host: 10.104.0.1:2379 # ETCD node IP address + host: ${NODE_IP}:2379 bootstrap: # this section will be written into Etcd:///config after initializing new cluster dcs: - ttl: 30 - loop_wait: 10 - retry_timeout: 10 - maximum_lag_on_failover: 1048576 - slots: - percona_cluster_1: - type: physical - postgresql: - use_pg_rewind: true - use_slots: true - parameters: - wal_level: replica - hot_standby: "on" - wal_keep_segments: 10 - max_wal_senders: 5 - max_replication_slots: 10 - wal_log_hints: "on" - logging_collector: 'on' - # some desired options for 'initdb' - initdb: # Note: It needs to be a list (some options need values, others are switches) - - encoding: UTF8 - - data-checksums - pg_hba: # Add following lines to pg_hba.conf after running 'initdb' - - host replication replicator 127.0.0.1/32 trust - - host replication replicator 0.0.0.0/0 md5 - - host all all 0.0.0.0/0 md5 - - host all all ::0/0 md5 - # Some additional users which needs to be created after initializing new cluster - users: - admin: - password: qaz123 - options: - - createrole - - createdb - percona: - password: qaz123 - options: - - createrole - - createdb - - postgresql: - cluster_name: cluster_1 - listen: 0.0.0.0:5432 - connect_address: 10.104.0.1:5432 - data_dir: /data/pgsql - bin_dir: /usr/pgsql-15/bin - pgpass: /tmp/pgpass - authentication: - replication: - username: replicator - password: replPasswd - superuser: - username: postgres + ttl: 30 + loop_wait: 10 + retry_timeout: 10 + maximum_lag_on_failover: 1048576 + slots: + percona_cluster_1: + type: physical + + postgresql: + use_pg_rewind: true + use_slots: true + parameters: + wal_level: replica + hot_standby: "on" + wal_keep_segments: 10 + max_wal_senders: 5 + max_replication_slots: 10 + wal_log_hints: "on" + logging_collector: 'on' + + # some desired options for 'initdb' + initdb: # Note: It needs to be a list (some options need values, others are switches) + - encoding: UTF8 + - data-checksums + + pg_hba: # Add following lines to pg_hba.conf after running 'initdb' + - host replication replicator 127.0.0.1/32 trust + - host replication replicator 0.0.0.0/0 md5 + - host all all 0.0.0.0/0 md5 + - host all all ::0/0 md5 + + # Some additional users which needs to be created after initializing new cluster + users: + admin: + password: qaz123 + options: + - createrole + - createdb + percona: password: qaz123 - parameters: - unix_socket_directories: "/var/run/postgresql/" - create_replica_methods: - - basebackup - basebackup: - checkpoint: 'fast' + options: + - createrole + - createdb + + postgresql: + cluster_name: cluster_1 + listen: 0.0.0.0:5432 + connect_address: ${NODE_IP}:5432 + data_dir: ${DATADIR} + bin_dir: ${PG_BIN_DIR} + pgpass: /tmp/pgpass + authentication: + replication: + username: replicator + password: replPasswd + superuser: + username: postgres + password: qaz123 + parameters: + unix_socket_directories: "/var/run/postgresql/" + create_replica_methods: + - basebackup + basebackup: + checkpoint: 'fast' tags: nofailover: false @@ -348,146 +416,108 @@ The `etcd` cluster is first started in one node and then the subsequent nodes ar nosync: false ``` -5. Create the configuration files for `node2` and `node3`. Replace the **node name and IP address** of `node1` to those of `node2` and `node3`, respectively. - -6. Create the systemd unit file `patroni.service` in `/etc/systemd/system`. +4. Check that the systemd unit file `patroni.service` is created in `/etc/systemd/system`. If it is created, skip this step. - ```{.bash data-promp="$"} - $ sudo vim /etc/systemd/system/patroni.service - ``` - - Add the following contents in the file: - - ```ini - [Unit] - Description=Runners to orchestrate a high-availability PostgreSQL - After=syslog.target network.target + If it's **not** created, create it manually and specify the following contents within: + + ```ini title="/etc/systemd/system/patroni.service" + [Unit] + Description=Runners to orchestrate a high-availability PostgreSQL + After=syslog.target network.target - [Service] - Type=simple + [Service] + Type=simple - User=postgres - Group=postgres + User=postgres + Group=postgres - # Start the patroni process - ExecStart=/bin/patroni /etc/patroni/patroni.yml + # Start the patroni process + ExecStart=/bin/patroni /etc/patroni/patroni.yml - # Send HUP to reload from patroni.yml - ExecReload=/bin/kill -s HUP $MAINPID + # Send HUP to reload from patroni.yml + ExecReload=/bin/kill -s HUP $MAINPID - # only kill the patroni process, not its children, so it will gracefully stop postgres - KillMode=process + # only kill the patroni process, not its children, so it will gracefully stop postgres + KillMode=process - # Give a reasonable amount of time for the server to start up/shut down - TimeoutSec=30 + # Give a reasonable amount of time for the server to start up/shut down + TimeoutSec=30 - # Do not restart the service if it crashes, we want to manually inspect database on failure - Restart=no + # Do not restart the service if it crashes, we want to manually inspect database on failure + Restart=no - [Install] - WantedBy=multi-user.target - ``` + [Install] + WantedBy=multi-user.target + ``` -7. Make systemd aware of the new service: +5. Make `systemd` aware of the new service: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo systemctl daemon-reload - $ sudo systemctl enable patroni - $ sudo systemctl start patroni ``` - !!! admonition "Troubleshooting Patroni" +6. Now it's time to start Patroni. You need the following commands on all nodes but not in parallel. Start with the `node1` first, wait for the service to come to live, and then proceed with the other nodes one-by-one, always waiting for them to sync with the primary node: - To ensure that Patroni has started properly, check the logs using the following command: + ```{.bash data-prompt="$"} + $ sudo systemctl enable --now patroni + $ sudo systemctl restart patroni + ``` - ```{.bash data-promp="$"} - $ sudo journalctl -u patroni.service -n 100 -f - ``` + When Patroni starts, it initializes PostgreSQL (because the service is not currently running and the data directory is empty) following the directives in the bootstrap section of the configuration file. - The output shouldn't show any errors: +7. Check the service to see if there are errors: - ``` - … - - Sep 23 12:50:21 node01 systemd[1]: Started PostgreSQL high-availability manager. - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,022 INFO: Selected new etcd server http://10.104.0.2:2379 - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,029 INFO: No PostgreSQL configuration items changed, nothing to reload. - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,168 INFO: Lock owner: None; I am node1 - Sep 23 12:50:22 node01 patroni[10119]: 2021-09-23 12:50:22,177 INFO: trying to bootstrap a new cluster - Sep 23 12:50:22 node01 patroni[10140]: The files belonging to this database system will be owned by user "postgres". - Sep 23 12:50:22 node01 patroni[10140]: This user must also own the server process. - Sep 23 12:50:22 node01 patroni[10140]: The database cluster will be initialized with locale "C.UTF-8". - Sep 23 12:50:22 node01 patroni[10140]: The default text search configuration will be set to "english". - Sep 23 12:50:22 node01 patroni[10140]: Data page checksums are enabled. - Sep 23 12:50:22 node01 patroni[10140]: creating directory /var/lib/postgresql/12/main ... ok - Sep 23 12:50:22 node01 patroni[10140]: creating subdirectories ... ok - Sep 23 12:50:22 node01 patroni[10140]: selecting dynamic shared memory implementation ... posix - Sep 23 12:50:22 node01 patroni[10140]: selecting default max_connections ... 100 - Sep 23 12:50:22 node01 patroni[10140]: selecting default shared_buffers ... 128MB - Sep 23 12:50:22 node01 patroni[10140]: selecting default time zone ... Etc/UTC - Sep 23 12:50:22 node01 patroni[10140]: creating configuration files ... ok - Sep 23 12:50:22 node01 patroni[10140]: running bootstrap script ... ok - Sep 23 12:50:23 node01 patroni[10140]: performing post-bootstrap initialization ... ok - Sep 23 12:50:23 node01 patroni[10140]: syncing data to disk ... ok - Sep 23 12:50:23 node01 patroni[10140]: initdb: warning: enabling "trust" authentication for local connections - Sep 23 12:50:23 node01 patroni[10140]: You can change this by editing pg_hba.conf or using the option -A, or - Sep 23 12:50:23 node01 patroni[10140]: --auth-local and --auth-host, the next time you run initdb. - Sep 23 12:50:23 node01 patroni[10140]: Success. You can now start the database server using: - Sep 23 12:50:23 node01 patroni[10140]: /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/main -l logfile start - Sep 23 12:50:23 node01 patroni[10156]: 2021-09-23 12:50:23.672 UTC [10156] LOG: redirecting log output to logging collector process - Sep 23 12:50:23 node01 patroni[10156]: 2021-09-23 12:50:23.672 UTC [10156] HINT: Future log output will appear in directory "log". - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,694 INFO: postprimary pid=10156 - Sep 23 12:50:23 node01 patroni[10165]: localhost:5432 - accepting connections - Sep 23 12:50:23 node01 patroni[10167]: localhost:5432 - accepting connections - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,743 INFO: establishing a new patroni connection to the postgres cluster - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,757 INFO: running post_bootstrap - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,767 INFO: Software Watchdog activated with 25 second timeout, timing slack 15 seconds - Sep 23 12:50:23 node01 patroni[10119]: 2021-09-23 12:50:23,793 INFO: initialized a new cluster - Sep 23 12:50:33 node01 patroni[10119]: 2021-09-23 12:50:33,810 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:33 node01 patroni[10119]: 2021-09-23 12:50:33,899 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:43 node01 patroni[10119]: 2021-09-23 12:50:43,898 INFO: no action. I am (node1) the leader with the lock - Sep 23 12:50:53 node01 patroni[10119]: 2021-09-23 12:50:53,894 INFO: no action. I am (node1) the leader with the - ``` + ```{.bash data-prompt="$"} + $ sudo journalctl -fu patroni + ``` - A common error is Patroni complaining about the lack of proper entries in the pg_hba.conf file. If you see such errors, you must manually add or fix the entries in that file and then restart the service. + A common error is Patroni complaining about the lack of proper entries in the pg_hba.conf file. If you see such errors, you must manually add or fix the entries in that file and then restart the service. - Changing the patroni.yml file and restarting the service will not have any effect here because the bootstrap section specifies the configuration to apply when PostgreSQL is first started in the node. It will not repeat the process even if the Patroni configuration file is modified and the service is restarted. + Changing the patroni.yml file and restarting the service will not have any effect here because the bootstrap section specifies the configuration to apply when PostgreSQL is first started in the node. It will not repeat the process even if the Patroni configuration file is modified and the service is restarted. - If Patroni has started properly, you should be able to locally connect to a PostgreSQL node using the following command: + If Patroni has started properly, you should be able to locally connect to a PostgreSQL node using the following command: - ```{.bash data-promp="$"} - $ sudo psql -U postgres + ```{.bash data-prompt="$"} + $ sudo psql -U postgres - psql (14.1) - Type "help" for help. + psql (15.4) + Type "help" for help. - postgres=# - ``` + postgres=# + ``` -9. Configure, enable and start Patroni on the remaining nodes. -10. When all nodes are up and running, you can check the cluster status using the following command: +8. When all nodes are up and running, you can check the cluster status using the following command: - ```{.bash data-promp="$"} - $ sudo patronictl -c /etc/patroni/patroni.yml list - ``` - - Output: - - ``` - + Cluster: postgres (7011110722654005156) -----------+ - | Member | Host | Role | State | TL | Lag in MB | - +--------+-------+---------+---------+----+-----------+ - | node1 | node1 | Leader | running | 1 | | - | node2 | node2 | Replica | running | 1 | 0 | - | node3 | node3 | Replica | running | 1 | 0 | - +--------+-------+---------+---------+----+-----------+ - ``` + ```{.bash data-prompt="$"} + $ sudo patronictl -c /etc/patroni/patroni.yml list + ``` + + The output on `node1` resembles the following: + + ```{.text .no-copy} + + Cluster: cluster_1 --+---------+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +--------+-------------+---------+---------+----+-----------+ + | node-1 | 10.0.100.1 | Leader | running | 1 | | + +--------+-------------+---------+---------+----+-----------+ + ``` + + On the remaining nodes: + + ```{.text .no-copy} + + Cluster: cluster_1 --+---------+---------+----+-----------+ + | Member | Host | Role | State | TL | Lag in MB | + +--------+-------------+---------+---------+----+-----------+ + | node-1 | 10.0.100.1 | Leader | running | 1 | | + | node-2 | 10.0.100.2 | Replica | running | 1 | 0 | + +--------+-------------+---------+---------+----+-----------+ + ``` ## Configure HAProxy -HAproxy is the load balancer and the single point of entry to your PostgreSQL cluster for client applications. A client application accesses the HAPpoxy URL and sends its read/write requests there. Behind-the-scene, HAProxy routes write requests to the primary node and read requests - to the secondaries in a round-robin fashion so that no secondary instance is unnecessarily loaded. To make this happen, provide different ports in the HAProxy configuration file. In this deployment, writes are routed to port 5000 and reads - to port 5001 +HAproxy is the load balancer and the single point of entry to your PostgreSQL cluster for client applications. A client application accesses the HAPpoxy URL and sends its read/write requests there. Behind-the-scene, HAProxy routes write requests to the primary node and read requests - to the secondaries in a round-robin fashion so that no secondary instance is unnecessarily loaded. To make this happen, provide different ports in the HAProxy configuration file. In this deployment, writes are routed to port 5000 and reads - to port 5001 -This way, a client application doesn’t know what node in the underlying cluster is the current primary. HAProxy sends connections to a healthy node (as long as there is at least one healthy node available) and ensures that client application requests are never rejected. +This way, a client application doesn’t know what node in the underlying cluster is the current primary. HAProxy sends connections to a healthy node (as long as there is at least one healthy node available) and ensures that client application requests are never rejected. 1. Install HAProxy on the `HAProxy-demo` node: @@ -541,18 +571,22 @@ This way, a client application doesn’t know what node in the underlying cluste 3. Enable a SELinux boolean to allow HAProxy to bind to non standard ports: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo setsebool -P haproxy_connect_any on ``` 4. Restart HAProxy: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo systemctl restart haproxy ``` 5. Check the HAProxy logs to see if there are any errors: - ```{.bash data-promp="$"} + ```{.bash data-prompt="$"} $ sudo journalctl -u haproxy.service -n 100 -f - ``` \ No newline at end of file + ``` + +## Next steps + +[Configure pgBackRest](pgbackrest.md){.md-button} \ No newline at end of file diff --git a/docs/solutions/high-availability.md b/docs/solutions/high-availability.md index 2e74896ed..1510e8f30 100644 --- a/docs/solutions/high-availability.md +++ b/docs/solutions/high-availability.md @@ -1,18 +1,12 @@ # High Availability in PostgreSQL with Patroni -!!! summary - - - Solution overview - - Cluster deployment - - Testing the cluster - PostgreSQL has been widely adopted as a modern, high-performance transactional database. A highly available PostgreSQL cluster can withstand failures caused by network outages, resource saturation, hardware failures, operating system crashes or unexpected reboots. Such cluster is often a critical component of the enterprise application landscape, where [four nines of availability](https://en.wikipedia.org/wiki/High_availability#Percentage_calculation) is a minimum requirement. -There are several methods to achieve high availability in PostgreSQL. In this description we use [Patroni](#patroni) - the open-source extension to facilitate and manage the deployment of high availability in PostgreSQL. +There are several methods to achieve high availability in PostgreSQL. This solution document provides [Patroni](#patroni) - the open-source extension to facilitate and manage the deployment of high availability in PostgreSQL. -???+ admonition "High availability methods" +??? admonition "High availability methods" - There are a few methods for achieving high availability with PostgreSQL: + There are several native methods for achieving high availability with PostgreSQL: - shared disk failover, - file system replication, @@ -44,7 +38,7 @@ There are several methods to achieve high availability in PostgreSQL. In this de ## Patroni -[Patroni](https://patroni.readthedocs.io/en/latest/) provides a template-based approach to create highly available PostgreSQL clusters. Running atop the PostgreSQL streaming replication process, it integrates with watchdog functionality to detect failed primary nodes and take corrective actions to prevent outages. Patroni also relies on a pluggable configuration store to manage distributed, multi-node cluster configuration and store the information about the cluster health there. Patroni comes with REST APIs to monitor and manage the cluster and has a command-line utility called _patronictl_ that helps manage switchovers and failure scenarios. +[Patroni](https://patroni.readthedocs.io/en/latest/) is a template for you to create your own customized, high-availability solution using Python and - for maximum accessibility - a distributed configuration store like ZooKeeper, etcd, Consul or Kubernetes. ### Key benefits of Patroni: @@ -67,13 +61,15 @@ The following diagram shows the architecture of a three-node PostgreSQL cluster The components in this architecture are: - PostgreSQL nodes -- Patroni provides a template for configuring a highly available PostgreSQL cluster. +- Patroni - a template for configuring a highly available PostgreSQL cluster. -- ETCD is a Distributed Configuration store that stores the state of the PostgreSQL cluster. +- ETCD - a Distributed Configuration store that stores the state of the PostgreSQL cluster. -- HAProxy is the load balancer for the cluster and is the single point of entry to client applications. +- HAProxy - the load balancer for the cluster and is the single point of entry to client applications. -- Softdog - a watchdog utility which is used by Patroni to check the nodes' health. Watchdog resets the whole system when it doesn't receive a keepalive heartbeat within a specified time. +- pgBackRest - the backup and restore solution for PostgreSQL + +- Percona Monitoring and Management (PMM) - the solution to monitor the health of your cluster ### How components work together @@ -83,13 +79,9 @@ Patroni periodically sends heartbeat requests with the cluster status to ETCD. E The connections to the cluster do not happen directly to the database nodes but are routed via a connection proxy like HAProxy. This proxy determines the active node by querying the Patroni REST API. -## Deployment - -Use the following links to navigate to the setup instructions relevant to your operating system: -- [Deploy on Debian or Ubuntu](ha-setup-apt.md) -- [Deploy on Red Hat Enterprise Linux or CentOS](ha-setup-yum.md) +## Next steps -## Testing +[Deploy on Debian or Ubuntu](ha-setup-apt.md){.md-button} +[Deploy on RHEL or derivatives](ha-setup-yum.md){.md-button} -See the [Testing PostgreSQL cluster](ha-test.md) for the guidelines on how to test your PostgreSQL cluster for replication, failure, switchover. \ No newline at end of file diff --git a/docs/solutions/pgbackrest.md b/docs/solutions/pgbackrest.md new file mode 100644 index 000000000..440b6fca2 --- /dev/null +++ b/docs/solutions/pgbackrest.md @@ -0,0 +1,341 @@ +# pgBackRest setup + +pgBackRest is the backup tool used to perform Postgres database backup, restoration, and point-in-time recovery. It is a server-client application, where the server runs on a dedicated host and a client runs on every PostgreSQL node. + +You also need a backup storage to store the backups. It can either be a remote storage such as AWS S3, S3-compatible storages or Azure blob storage, or a filesystem-based one. + +## Configure backup server + +### Install pgBackRest + +1. Enable the repository with [percona-release](https://www.percona.com/doc/percona-repo-config/index.html) + + ```{.bash data-prompt="$"} + $ sudo percona-release setup ppg-15 + ``` + +2. Install pgBackRest package + + === "Debian/Ubuntu" + + ```{.bash data-prompt="$"} + $ sudo apt install percona-pgbackrest + ``` + + === "RHEL/derivatives" + + ```{.bash data-prompt="$"} + $ sudo yum install percona-pgbackrest + ``` + +### Create the configuration file + +1. Create environment variables to simplify the config file creation: + + ```bash + export SRV_NAME="bkp-srv" + export NODE1_NAME="node-1" + export NODE2_NAME="node-2" + export NODE3_NAME="node-3" + ``` + +2. Create the `pgBackRest` repository + + A repository is where `pgBackRest` stores backups. In this example, the backups will be saved to `/var/lib/pgbackrest` + + ```{.bash data-prompt="$"} + $ sudo mkdir -p /var/lib/pgbackrest + $ sudo chmod 750 /var/lib/pgbackrest + $ sudo chown postgres:postgres /var/lib/pgbackrest + ``` + +3. The default pgBackRest configuration file location is `/etc/pgbackrest/pgbackrest.conf`. If it does not exist, then `/etc/pgbackrest.conf` is used next. Edit the `pgbackrest.conf` file to include the following configuration: + + ``` + [global] + + # Server repo details + repo1-path=/var/lib/pgbackrest + + ### Retention ### + # - repo1-retention-archive-type + # - If set to full pgBackRest will keep archive logs for the number of full backups defined by repo-retention-archive + repo1-retention-archive-type=full + + # repo1-retention-archive + # - Number of backups worth of continuous WAL to retain + # - NOTE: WAL segments required to make a backup consistent are always retained until the backup is expired regardless of how this option is configured + # - If this value is not set and repo-retention-full-type is count (default), then the archive to expire will default to the repo-retention-full + # repo1-retention-archive=2 + + # repo1-retention-full + # - Full backup retention count/time. + # - When a full backup expires, all differential and incremental backups associated with the full backup will also expire. + # - When the option is not defined a warning will be issued. + # - If indefinite retention is desired then set the option to the max value. + repo1-retention-full=4 + + # Server general options + process-max=12 + log-level-console=info + #log-level-file=debug + log-level-file=info + start-fast=y + delta=y + backup-standby=y + + ########## Server TLS options ########## + tls-server-address=* + tls-server-cert-file=/pg_ha/certs/${SRV_NAME}.crt + tls-server-key-file=/pg_ha/certs/${SRV_NAME}.key + tls-server-ca-file=/pg_ha/certs/ca.crt + + ### Auth entry ### + tls-server-auth=${NODE1_NAME}=cluster_1 + tls-server-auth=${NODE2_NAME}=cluster_1 + tls-server-auth=${NODE3_NAME}=cluster_1 + + ### Clusters and nodes ### + [cluster_1] + pg1-host=${NODE1_NAME} + pg1-host-port=8432 + pg1-port=5432 + pg1-path=/var/lib/postgresql/11/ + pg1-host-type=tls + pg1-host-cert-file=/pg_ha/certs/${SRV_NAME}.crt + pg1-host-key-file=/pg_ha/certs/${SRV_NAME}.key + pg1-host-ca-file=/pg_ha/certs/ca.crt + pg1-socket-path=/var/run/postgresql + + + pg2-host=${NODE2_NAME} + pg2-host-port=8432 + pg2-port=5432 + pg2-path=/var/lib/postgresql/11/ + pg2-host-type=tls + pg2-host-cert-file=/pg_ha/certs/${SRV_NAME}.crt + pg2-host-key-file=/pg_ha/certs/${SRV_NAME}.key + pg2-host-ca-file=/pg_ha/certs/ca.crt + pg2-socket-path=/var/run/postgresql + + pg3-host=${NODE3_NAME} + pg3-host-port=8432 + pg3-port=5432 + pg3-path=/var/lib/postgresql/11/ + pg3-host-type=tls + pg3-host-cert-file=/pg_ha/certs/${SRV_NAME}.crt + pg3-host-key-file=/pg_ha/certs/${SRV_NAME}.key + pg3-host-ca-file=/pg_ha/certs/ca.crt + pg3-socket-path=/var/run/postgresql + ``` + +4. Create the `systemd` unit file at the path `/etc/systemd/system/pgbackrest.service` + + ```ini title="/etc/systemd/system/pgbackrest.service" + [Unit] + Description=pgBackRest Server + After=network.target + StartLimitIntervalSec=0 + + [Service] + Type=simple + User=postgres + Restart=always + RestartSec=1 + ExecStart=/usr/bin/pgbackrest server + #ExecStartPost=/bin/sleep 3 + #ExecStartPost=/bin/bash -c "[ ! -z $MAINPID ]" + ExecReload=/bin/kill -HUP $MAINPID + + [Install] + WantedBy=multi-user.target + ``` + +### Create the certificate files + +1. Create the folder where to store the certificates. For example, `/pg_ha/certs` + +2. Define the variable for the certificates path: + + ```bash + export CA_PATH="/pg_ha/certs" + ``` + +3. Create the certificates and keys + + ```{.bash data-prompt="$"} + $ sudo -iu postgres openssl req -new -x509 -days 365 -nodes -out ${CA_PATH}/ca.crt -keyout ${CA_PATH}/ca.key -subj "/CN=root-ca" + ``` + +4. Create the certificate for the backup server + + ```{.bash data-prompt="$"} + $ sudo -iu postgres openssl req -new -nodes -out ${CA_PATH}/${SRV_NAME}.csr -keyout ${CA_PATH}/${SRV_NAME}.key -subj "/CN=${SRV_NAME}" + ``` + +5. Create the certificates for each node: `node1`, `node2`, `node3` + + ```{.bash data-prompt="$"} + $ sudo -iu postgres openssl req -new -nodes -out ${CA_PATH}/${NODE1_NAME}.csr -keyout ${CA_PATH}/${NODE1_NAME}.key -subj "/CN=${NODE1_NAME}" + $ sudo -iu postgres openssl req -new -nodes -out ${CA_PATH}/${NODE2_NAME}.csr -keyout ${CA_PATH}/${NODE2_NAME}.key -subj "/CN=${NODE2_NAME}" + $ sudo -iu postgres openssl req -new -nodes -out ${CA_PATH}/${NODE3_NAME}.csr -keyout ${CA_PATH}/${NODE3_NAME}.key -subj "/CN=${NODE3_NAME}" + ``` + +6. Sign the certificates with the `root-ca` key + + ```{.bash data-prompt="$"} + $ sudo -iu postgres openssl x509 -req -in ${CA_PATH}/${SRV_NAME}.csr -days 365 -CA ${CA_PATH}/ca.crt -CAkey ${CA_PATH}/ca.key -CAcreateserial -out ${CA_PATH}/${SRV_NAME}.crt + $ sudo -iu postgres openssl x509 -req -in ${CA_PATH}/${NODE1_NAME}.csr -days 365 -CA ${CA_PATH}/ca.crt -CAkey ${CA_PATH}/ca.key -CAcreateserial -out ${CA_PATH}/${NODE1_NAME}.crt + $ sudo -iu postgres openssl x509 -req -in ${CA_PATH}/${NODE2_NAME}.csr -days 365 -CA ${CA_PATH}/ca.crt -CAkey ${CA_PATH}/ca.key -CAcreateserial -out ${CA_PATH}/${NODE2_NAME}.crt + $ sudo -iu postgres openssl x509 -req -in ${CA_PATH}/${NODE3_NAME}.csr -days 365 -CA ${CA_PATH}/ca.crt -CAkey ${CA_PATH}/ca.key -CAcreateserial -out ${CA_PATH}/${NODE3_NAME}.crt + ``` + +7. Remove temporary files + + ```{.bash data-prompt="$"} + $ rm ${CA_PATH}/*.csr + ``` + +8. Reload, enable, and start the service + + ```{.bash data-prompt="$"} + $ sudo systemctl daemon-reload + $ sudo systemctl enable --now pgbackrest + ``` + +## Configure database servers + +Run the following command on `node1`, `node2` and `node3`. + +1. Create the certificates folder. For example, `/pg_ha/certs` + + ```{.bash data-prompt="$"} + $ sudo mkdir -p /pg_ha/certs + ``` + +2. Export environment variables to simplify config file creation + + ```bash + export NODE_NAME=`hostname -f` + ``` + +3. Create the configuration file. The default path is `/etc/pgbackrest.conf` + + ```ini title="/etc/pgbackrest.conf" + [global] + repo1-host=bkp-srv + repo1-host-user=postgres + repo1-host-type=tls + repo1-host-cert-file=/pg_ha/certs/${NODE_NAME}.crt + repo1-host-key-file=/pg_ha/certs/${NODE_NAME}.key + repo1-host-ca-file=/pg_ha/certs/ca.crt + + # general options + process-max=16 + log-level-console=info + log-level-file=debug + + # tls server options + tls-server-address=* + tls-server-cert-file=/pg_ha/certs/${NODE_NAME}.crt + tls-server-key-file=/pg_ha/certs/${NODE_NAME}.key + tls-server-ca-file=/pg_ha/certs/ca.crt + tls-server-auth=bkp-srv=cluster_1 + + [cluster_1] + pg1-path=/var/lib/postgresql/11 + ``` + +4. Create the `systemd` unit file at the path `/etc/systemd/system/pgbackrest.service` + + ```ini title="/etc/systemd/system/pgbackrest.service" + [Unit] + Description=pgBackRest Server + After=network.target + StartLimitIntervalSec=0 + + [Service] + Type=simple + User=postgres + Restart=always + RestartSec=1 + ExecStart=/usr/bin/pgbackrest server + #ExecStartPost=/bin/sleep 3 + #ExecStartPost=/bin/bash -c "[ ! -z $MAINPID ]" + ExecReload=/bin/kill -HUP $MAINPID + + [Install] + WantedBy=multi-user.target + ``` + +5. Reload, enable, and start the service + + ```{.bash data-prompt="$"} + $ sudo systemctl daemon-reload + $ sudo systemctl enable --now pgbackrest + ``` + +6. Change Patroni configuration to use pgBackRest. Run this command on one node only, for example, on `node1`. Edit the `/etc/patroni/patroni.yml` file : + + ```yaml title="/etc/patroni/patroni.yml" + loop_wait: 10 + maximum_lag_on_failover: 1048576 + postgresql: + parameters: + archive_command: pgbackrest --stanza=cluster_1 archive-push "/var/lib/postgresql/15/main/pg_wal/%f" + archive_mode: true + archive_timeout: 1800s + hot_standby: true + logging_collector: 'on' + max_replication_slots: 10 + max_wal_senders: 5 + wal_keep_size: 4096 + wal_level: logical + wal_log_hints: true + recovery_conf: + recovery_target_timeline: latest + restore_command: pgbackrest --config=/etc/pgbackrest.conf --stanza=cluster_1 archive-get %f "%p" + use_pg_rewind: true + use_slots: true + retry_timeout: 10 + slots: + percona_cluster_1: + type: physical + ttl: 30 + ``` + +## Create backups + +Run the following commands on the **backup server** + +1. Create the stanza. A stanza is the configuration for a PostgreSQL database cluster that defines where it is located, how it will be backed up, archiving options, etc. + + ```{.bash data-prompt="$"} + $ sudo -iu postgres pgbackrest --stanza=cluster_1 stanza-create + ``` + +2. Create a full backup + + ```{.bash data-prompt="$"} + $ sudo -iu postgres pgbackrest --stanza=cluster_1 --type=full backup + ``` + +3. Create an incremental backup + + ```{.bash data-prompt="$"} + $ sudo -iu postgres pgbackrest --stanza=cluster_1 --type=incr backup + ``` + +4. Check backup info + + ```{.bash data-prompt="$"} + $ sudo -iu postgres pgbackrest --stanza=cluster_1 info + ``` + +5. Expire (remove) a backup. Be careful with removal, because removing a full backup also removes dependent incremental backups + + ```{.bash data-prompt="$"} + $ sudo -iu postgres pgbackrest --stanza=cluster_1 expire --set=20230617-021338F + ``` + +[Test PostgreSQL cluster](ha-test.md){.md-button} \ No newline at end of file diff --git a/mkdocs-base.yml b/mkdocs-base.yml index b2245ec83..296e83c19 100644 --- a/mkdocs-base.yml +++ b/mkdocs-base.yml @@ -155,7 +155,8 @@ nav: - High availability: - 'High availability': 'solutions/high-availability.md' - 'Deploying on Debian or Ubuntu': 'solutions/ha-setup-apt.md' - - 'Deploying on RHEL or CentOS': 'solutions/ha-setup-yum.md' + - 'Deploying on RHEL or derivatives': 'solutions/ha-setup-yum.md' + - solutions/pgbackrest.md - solutions/ha-test.md - Backup and disaster recovery: - 'Overview': 'solutions/backup-recovery.md'