From 139e186ac2be5410e78c67696b5e4b6564309af8 Mon Sep 17 00:00:00 2001 From: TwinProduction Date: Fri, 4 Sep 2020 21:31:28 -0400 Subject: [PATCH] Support sending notifications when alert is resolved Add debug parameter for those wishing to filter some noise from the logs --- .github/assets/slack-alerts.png | Bin 0 -> 34441 bytes README.md | 78 ++++++++++++------------ config/config.go | 1 + core/alerting.go | 63 ++++++++++++++++++++ main.go | 8 +-- watchdog/watchdog.go | 101 +++++++++++++++++++++----------- 6 files changed, 175 insertions(+), 76 deletions(-) create mode 100644 .github/assets/slack-alerts.png diff --git a/.github/assets/slack-alerts.png b/.github/assets/slack-alerts.png new file mode 100644 index 0000000000000000000000000000000000000000..3f9e3cad1bb26472e9f49ff9c5e44e62936f2b43 GIT binary patch literal 34441 zcmeFZbx@mM*DhRxLXqN5DGmjKyR|@&Vl6HKibH`yaOX!!u;5y(6ff=&ybbOyMH}3m z;^7N@p68u2-#72fH|Nax@BM?>B$MR6_ugw=*SgkP`wn}jrhxa1>e+(_5Ac)}`AoHMXh-M4@;)(Se)i)0wR7T_6m|~z`V>>D8yF7S+ z-+ur1u-oyw#e)aIFO=loXn7j%wfK3GOt)-^2J8(8yJ~6me5IZa%4beZ?+| z&|)2JlO^?d*KZ*#D$Z|Q5ApZ}urBsR3)I|nwRPJRKL$SYla^?Esy$hFV!YwI)2wyJ zPauOu5Al5RjQsD2`QMDeN~D19rpAhMZq%cuyI<0pcl|3xE+~ni45>)sCU^R3KcB_L z06Oa^nxB_08<6;uSzdYwflTv2?lW@DnlCKT#ZVH)97aZwn=Wck4}I~tX=CW^^biTl zoiJ)qtl$M5M$iRkBI2^zn*9U%eh8xZ3lW{K3{;9SH{=y~k4FK!7o&R@zr!7od2J}W zUDWz?#X%}Z<$3T9WRl*kQ~0oJEwj7W1Wq$o-tg@Dt0zrCG|i~g{oFcWo@(%PaTmUxpJ2_7;CU z8dX0)q%gM6vi@BiJaugH2II9EpM&MwE9%2hVc)kkzBjey*>^a}=F`(3c}C8&Q}M<8 zkC~*to=Y|#T@GWvIni#ln!Q`o~;`-F6tCm16--` zQn`0)7O}rFUvrXalBws%(oQDdFQ4cS+J=Ri3!t+Se}T-Iy)l#!31MOb+~r z=%>-q44Eo}IPqcz$IDzWb-O>m-(Y7?d3X=kaJlee_@0cX{as%&-t}?vp=tjTvRVbR zKs^INiDY2~b%6h@D&aO?vecRAbIb=QEyNQ%vNGJO=RNgSY2Z?O7TZ~C@crVQM*&@+ z+e1rQ<+S(3L&6L(BSRc7E2miZx8y){#OP(Su=kXOV+DfE4vd>`2OXCexC00euz#A( zUY}JT{Y({{sjMURSqd;|NYn(1`(GRGUYt=e?YBKY@Lyn^Pd)b|oNGe~+LLc6tCHvW z7@YRJ<&Y6K>QWp7%hlz_Oopr}F;*#t)BbeKQrdB` zq)=s?+EbXjVG&R=oDbXd_j3@58fkPFD1M)6RkIVDw)syCzn}lw;*vEUYv8wIRyNdF zt~nOKf%oR?5@vu0u5DR0da?Ni4xcS&>1a3sQ`6Ug z7u`{`?Z_l$YA!!X$-N?7X7tC<~j zcT(R572erCw6=&Z9Sbel0xpa#BLAvw~=C1xiDp$60WT~(NvQwib z6r|E+<~_nvD+md-e9L~ORehYZRhS^PLco~%7NYE}S&Xy_+DiVw#2;+?3T+>&V?T%y zYw?3=M`)uDyp^%Xo$`PDw7jenxl<DQLjCx zTTYtRR{W5%jIf_6O9)m(W*|o^wBGUh#n4xYLfOi9GVs-@RvQknM2oA+u2a(+1as4* z%>DL45CJ9&Q;w_meH?A(i&zb9R788e3bh11XjLsM zXR954el2WvTdrgJlBP?%Kgo3G+_BwIcp=omW&7x7q)Ww47pCd)o#+fZCw*B9W7%y$ z{d|O;GLHLAyY%lBF7B1qJ3jJQRyyv=Vz$Ub8CfO)EF@Cshbb{l5Mhl{rN%g<&dpTL z9L(e+jvM=)6loTdxD>dxB?lvu$BT|MNw}V)zGKUVV7g?>ehMh-V6q-g|4Hq~{mt*E2l&HpbGI^*!d7=KW5}Sr z+&IGhVl8R4VLfEa>YH0U0l@pH#PL1B`^)MFErIjj6^-FcBZaL$c4SYE|2bs+2cEu7 z$j~vvvJhG~>6}(09Z*gNaE^pITH%QKWQTTUV{zGOP0x8>;u)jua~uba`dLyHduP)l zD3i7zlx|Mf#%*`c5o5caizA2sneTwg&M-=LQBEYx8N44pOpNJ&N!}N zy_8-p^Yeo?b)j^fLxybL-1qn+d6?i7t0^^SW>|Tnpn7pz{zAsela2Ltx~A-y%IX1y z-{Bu>J#e@0P5smTpVr-?I&aVx=}Cwt&9EE#&W%jpNBxeOA>F#1og+Wrq*IAz$(yUu zi07kpSx`yiZ&;&fJ2P{*%{rHaOx<>tmdAfS{txXAqkn0w z9mM?sQQZNi^==3;g8;}=GT*0m0siD;Pc>H3817xNC*emt0_eQItDhU!baOIGnsK-81U7K6u4W;AgPhlMieI*Cn|C1v14*oaJ;Q#MNPfhrD^(EayuiK(H8CRnR zfl~e4uT6O5yyL+gSMfa&+nSoOu}lm`Z{*@McdW6i!gwC-AQXG(YSS(FZs`eiv?h{p zfS5=7Fl}zukX@9>^5YkGjczPaF*e-UkFUNEXb4KMgESTfx4SX@ip{c>+un7IvIqPg zInD_qrWgP6+N|R9OmwKz#^uCw4xz!}J05dwN;L*D;%r5_sN@aa9QBG)V>o64{>7hN>^y1k#JJEdH{p ztI5$>eV@~l1LIwfTby_`Hv0MSn^aa0Lu(y&YbwiVn2LW18)!bptV18r*~2h88C+M^ z0W6f&4nja5$ZE7 zIdNN9i;Ge`DjCCfcK+rBrY8no0|Ci6_)r}B_~34cGFhc|wu=FQOy8e+Qu86{bOIwk zmav{i_nHC6H`k3=aclBiV_-NQNJ9Qmc{aL_ zU&EaM-g|AM+_xTLH#2nk$5izBQj}E+fF5&hjp~JITISX}Mgs?RgC%eN(9+ngQ^t>b z|HRQJnyBu`wc6tpU8i=;RWZ;)O|GinB)l`o1+K^U&AnW1h^>0^`FK8+g0HH1s)o{^ zbDvX>gYjgxO%D`54fGZRrjqK%k{i+5lqPT4yxw`mNUK&|DtsTMI3&~(vMPA#lAOyN zBbb9M9A76#AI>%G7g!h(H%JlyZSn?SLKjVlQ1VGQ8FIPg03-2j7rV_W(oFhu3*b!c zDr;r3j&CzG8MMK?oy{2#%i3^J=)E2VDkqv_+8(!5-JD_Cso){yT@l4QgS7!AwJw`P zO1`dXWwLRHWDad`GEc+xryE39G8Cz&OL35W`GemXe_1@qV2IYr(3*bLQxEdrY+h1A z%Jf(uxsZG`Z_;dVXT9C3v*b;v2r9UYc&Ltp^Y&~qCO;2-Wy_Y#ut{O3I)4;vZff{I zRZvO*JC+~}NN7BXyJQ);Id3K8tA>mN`l>Qo4LA_vrP#*+6{=NUz|@p+m?py}Jzl-pfQ<(eI=XQRhdR;V`Vz-#@i5-@tiMPNErpDo*fLl1~=m5!K7rTlI*6Ly%S z2k>hJ%C~ca@qE%+ZL%T3@G^)p7ZsFxZC{WvbIl+zT}9Jza4f?wm+dSey(%s6j}&1g zO6|-85PfR65s_*z5#C&y#(I;PH~llzH*S_Mxt!s~AQqE!x}i2WEmcP@K{3v&ba>ES z3tXbP^U~L5foVA#80@76%^8pLu-lSbZOm%u66JSvMxAZal(6kXDG+&-gZ0y4QQQ%m zNjzfYeQ;q$i)RIVZd^^J#E}fr269i?cC@&o8CnYi6en5k*`!)eTq`DT zmH-zo!y}%(BCOIXQ>w;#TN|S%T~Pcun{%L%oPpxC$V6~i^Q#y z^(M!OKKIJ=Et~@8d>guE&_g1VZC=kkCP?i}uTdiIBv-J_K_IHbr})dA&w@ao^QKp@ z#~m3L>4tS@K(JVjFDO4VxF8>IGh*d~(1cD1tf%0^r;azSPqx2O9y%s&3>;GbUNXy9#7dmR z>r)-qZ1kDoUbHE7Y*}Ee$cA`-i}0STm^C)3%2uKN zD0a$kouJ+R(S-@|L(wIyv!=+HgoB)ZhQ=;>*KJX+#BTUNL`+UXY>_W&Vm2Qn$)o5% z0B&q2Ax|t&4Y%CkH?nQ*y)bB3&}b&Xx?=-j6-vDz0(51AHq;no2lU9dwlPO{B~-h3 zH|!l>yS3-vMwXKWED_3#(Ns+(d1`c6zn5Du7ueLvvNXF81R&$0zG=@J&VZ?QtolOs zI?etr+Kx^uscWF&|_s}?bHwE#k43yz5_;6=qY%GMTh{&sG*Zki!a z-B+LiBM!Au_A&UFQ-OT+t8gEB`R2Xz)#Y~rT(c%dL%n74qk%#>{7Eu(yMNBj{BdI4 zLPW|tVo*Mb1nE&%RHU{#O{Fy>TLjCa>sBJq?kuHTi$)t6&Ze}#=&><_K=L2wzc$4o zfa_lDNWS2jHAq$VXJ3v#5>YuV^boCJ4+t!kj0ms4CBxerIM+uu5q8qBm1BnI#f%pQ z(jB8F@~7J2q*^BA;p7#)DCGy|#e zooAK>eSdymr&H^>p(J-6V>3pB4PV<)2m{EvzRd z1YjIZd8PNmIEn1mwBA2u=oR4GV#>BlLsqLzfp>Yz2=qcL~k*680y5kbTL@_!d-LcweyZ{xIm~S6_Kd@yw-SVT^ZJLRbYanc5w;b z_x%rvZC87s=uaf(8j$|i-!}`slh^-6dSZ0yn?&Cyjs$;+LX{!9k0Pqd3lM_sHKV@ObR9Q5CK-;MiK<+-l=_nE2 zskAS&cCnb17P8rGc;?hB?n2)hgCf6}pSR8SAraII?QP zvEJ7Ygjgx~|C*tcGw-z>Jt`=*Z?!Z(gk7hv^$G$YDlgE=6S||9PJtI+iON$9($j!b zUSgDQeVH$06x(D#VEls2rYswh$oaO}^H%rlMl;ql>$`?_(ROwQ9kB?T>p$62 zp$bFg!slq~@wq0)pJlq?btt}VuIzUsTnD6Un2PgG`-TpgdObkp1>VTWNj5Tyf6Qc- zF2!+kFmtYgz(iXZ$H*td#UFhvh?;af@e$x3fx{ouD&K~F7vxk!GAKTgQ((QdlbPEl zKsb{}(M-mihaYO`tE4)yn@nq@i4pchQdq?JYFvm;e|f|ycU`(II~#iK^e79c0Wu&< zEZQQ8ZPg}Z3V{~CKgwmw!J7XS@z3HQ6L~}5f-0>uVU2`foV>I0gVK6fMPfYo8M#sP zY^O`C5-Wi&EQ5m-kPUB4op4q0@5{&7%bw3(e`MWItf*G2@7`7D&@Xv3zk%JAQCD@z z;?b;Rx)dI>_&GxjG78ux_8=wX6ZL!zw|QU1xir$0MJDhH%>M$(;HEe-!^bt>GrUt@ z24k$PecWn>0LCf3^t8_S_-)`iRX5mW3C7O+Vywl%8?POA0 z=+N8%cf;W1QW{@fufa`SvMKS6VeTCO0gO=%Mw)!k=it{=TVOhNvmb-N_`gEkxM+!K ziyF=hzEs=qQTPi>QUYBYMt_pGSO?_t3^Lg7xFl}7O%yjcU^QDdQWQ85f(7!gax-gf zjG5~x(GUz_OXhy8-btXl#1}Ug8FXk^_oeuTWcXdK^e6ieGK5f#f#Y}`=Q^G4EYsSt z15?;Yi^g7bF0P7M*$o9Hdya9N9Yt)D68E?XFzP^XIJU(9#4(gkB^q zcxgP`4&msI4aQLdf@#df0A`K-Q-bg7%**KbB-i3Jm0F5*TB`W)rTP8#qS$33Q>Bu2 zbRx*9T|#0LJIvkocSgF>>YOi=NY~uJMMU)D&(N8>o(bTdnVnoWS^v~wEQy(2>e*>Z zrB!b|8w8Tx_Z};dzQAtVuWQO(riU2=EdVUu#(VWkxvi(iC29&H=l_7)cgJ5tVwADb zKNYI*-6>r8P6?|)Aj|2mG_?Nq|8qnq#YFvYME>kA6sw?A(JDA*D~uHaBjIfFIM89a zW4TwSryazIP7vjFKmdHc7r>9`vBEn3Z zgnX3%MF=tQEnPHp*NNyzkEYkuFB>-{gO_RS>NSLd)QfVHTD?byFVsVKQ(Y*?l`qgj zEdCC-eDrdRc1Jxdrhtf&$s9YEH4cEz--W7hBgdA)D3fbnN_>xI#UUNSb^Wtr9=6*N z7X>JxI%;@r`}Vvh)JcqRx;rrz=<7i2n|%3BxEz~Ho%!sND3{SDvF*^1?zK!tXpZKK za!SJWW*e@jn!RbsKar1%#axJfITOrRg=1vRQU!&7^Z*|+v z<@bqXLZ`nurQm`fux+07jP*PK2vf$Uwd5hKW8~Vow`ORWeUP*;pEDAoyj^HTuVp~^ z)&`p|Ro+$G{PITIteD5~UNajJGWFkPA3-ewNKl>ra_2kdXq8Su0*DWUIVnj@@!UdS z;c0Z0m77VA00~Y?->R)JQ=4ctl~UMdffuD|&Mvx8rbjjuCpIML?KOuLB#&{t*^*NUQIJ3GT1TLqBDS_Wx7a?_tQ9D9Nh5|w}tVIs7L7$+|ur_n& z$iUr!ZUQ>57IRyi)b;wG+)K-Ba|$bu3^M=h&Q_Sf)Mkc@6#d zF&bI3cN||_X^;TvS-HhpPP~^@4HoNUB^9^Pm6;0MDk?NT5ze zPR@*wo-IO<0#$m+!ox8yXgn1vGz#t?Vx3(mixR)a6W(eSnlRRZB?+3!<_Mdnpc@HVnv!R%KN|hf56xdUn+;!%2l_^ynqV=J(jl#E`9E%8#KdISCyr?W(jOri z#wJClL-K3EJ{ws|4tNYN_mjbPM$F?hufLlxdEX1aWhv5WHbikM5CoK{OKBBRkbeeM z{#6r-&&e|!LdkD9@{1T~uS}7Fv-7){of$OW+a;!T^I7~1n>x5>TW{j!*sln8?K&_1 zp)@b6+#Ikmf2!rIcKSzP1{Oo}l!kj10J}}50lL5r`Kyv1Be^uNfFoW_BE}?BIC+2ubE0 zyhrL4gomqWY7x*?uMFVeKGlA!+2CZ4u#^iY8+{*kny}00T40me)%Q~HL!=#MY$rn* zhxu$dA0qE_O>Se}2O{L`Vskv!Q2mu5b~Xqo=GOpkFz=_v`Ew#KMx|qTtP;IGphrPg zD8#!R*Wjs>2Xwsv?}(+892PIK_JJ;PA)y31q$NH&@Mp<}KXI;Rc^25bH>4{2g3G zTw^q^q|)_;qZ^xWRtBY@TTHba!C-PNgBdX=OVp4KRI|{Doi?+>VAN)K`g3pOtGq&x zhENm^O%oGlF?TxyC=kfHXUNeJ2?|2*!%7toQ9G&YFQ7c=?NsJl zHq9w{V-ZN?;u{^2D#o#TEl3+%He&eG8O&#naFP^IS@s57yy4RT>vmS7AK>}H70;vu7icn2FWcC5aWS|za>RNm6 zC2zgXjR{gM%Pd1*C{oT5D=^XF6@Datsj2u$9{|)>(5Yu(7&DE5N@9R+R>=>1;gD0n z5dRz@5~%=(clLS@!Mla~nM)C~r{6VoLhGjVWjtTimu~K=lF_++Pj%{L$OwH|Go+_5 zk-H65*0%$leE%)ln}ZOp@7RfDmrdv)EW~`WiXnQeF>V@4k4%mpmxHL_uz{XT$^`XZf#>Px(fLQC3?v(k)x|D;-H?IY|b}Y^&t6(|d-WDQ& zJBY)r=hsw#0r9*2xob8AbpbD}MHlrdyj<;mnNGf;pQt*Z43AOH%uLKwnfsW0#}J03fz!0u4nXF^+Q%f^fkS;k ziA2s3uFy2&hz7nptMsT|Bc|3qs4|9aY>pUS^5w{t7J>p#k(I!BO3O+&mW*qJpK-|29p zvU;`&#i~GFDhuEHaiENx;nWWT8Gr-Yw4jl$m|TAK#e668Cuhmv`TZiq_d?m7MiC7|@!!RIaK*knW|)PC;eMk^2ni{;;lZ!< zXbdAOV5uT&8Be{5vIn2)RRLZR1BmIGvCcV#x0=v_VM(jkA`YwEe&|jm5VwsZmN*zg z%d&_q@Y{S}DT2f+!1}?_t;PuXmAQrr{CgD21jr5~=b9#=&O;)ESr6U2`A`&(BzOF2px7YkXZ(cA? zzSVp$5A+UKGwrQWbG}fMF9NAU0x;+vFhK~OqNDZung;IwdqgWspY#2}FtSVG`v5qf z75zqcG@b5eVCqfUebr-}SRLzc=Li8dI+{j*MB?%P;TR+gRTc`e2*9oUQh{O`W4yY- zpoeZYK6tbv6KcH*ChJ{%`ME{uNLN%N)D44RK6R`+nJq;`=2dhyLnr$Bif4yOPLxUv z%~h1O$%8*|(=JD9BqCNZ2zkJ+MLdQCFGxb>#E|1)bn+J-W8wbneWF$$ivl_njdJ8* zu=P9Eu;7UxOB_;*HDuAxh0LhnCMA2^|157aBS;?d;R}bjtf`=zjS>6OCbaaE!ikEk zNX@54f{#DwLBTmT$;e^S3|BP~<`I3yq0(NDx$s-U8tnR77-RAuMM#>HlETQVq1sbW zUw9;10a&aM{>?rI(HrExu6Sz}MPQJ;ZYKKOz^o|?0m+8pR7XrN6QTZi{(>7roNG@e zVgK2s16BuOvjUsEy*&T&%~3K|gEZvFn3b#rmyNJIG<}MeoL8XG48+CM2uXuxgj!Z9 zy4L3a=f`I=1OQRF1bci#N^P@Kav#zrtAhF4F!pwirxV+jeMk7Xf?Y}cx7@u*UabqE z=TgKS8U+kKwAY(39S$Ngy_Ys>=z?^W;cYO%_bsjEZvyurQi9JCXHL9>mZlOqNg(f@ zL*z1%OIGW=ZB^zZT(1){8XhVd7pJh<>5*WtG#AX^jEY8bHa5+<7Un38h9pDf zjD#6(3N5=Y2MMV)aRq*~b^5jpGqlc9klJ|n!N(WM*=8qO_7T9*rNVuVFR%E9cU-mj z%hXf?={r@~V&z%l->{?s?Gs4|q-xIBUHGnbo>D>Pg1hDZSvOmmOoChU_e4a*; zzF69%y?RCb_fA%chyk{8WR$da-G)~QEN5ErV&!5R=@b>^n!V7RCkni6>E}_paP)NfXQ`#yJRrAJJ^#!@{#NM*TbT2T~r|ER4EPr!zuofW){jbsq_HE(Q> z*6SsOdkaVJuAc8`Qu!#Dq&Q)7Cw!vGaQ)e`LJF9Su%plaQGpFj|K=pTv>Y0$OBPl} z)}Q1;x4n=1Qp(=*y*5&Vg%iEG9bv){WuajF$zE8LW6*MP`N1#@+1!&h%!u|B|HWM}dYvPrWpQBA27C?Xrz(S_TDLx5=cD{>Usg!a7AMf zE(RtCP|77!`PZL|=KfTa=f-q@DVJjs%rBigg@j+F;ZkWzp(n=@RdFx0S-%njs$`=9 zc3K&kd@kKl4ej!-U8*t4SG@QQ7&H@?$76=X=!hB|@w`gVXk=atlx_N(Z0mRKW6BJLT~7XPaM;vaz`tXOZRGp6^Du#! z^mFT;j3g0r1Wy8N6h^N0;*lbcJf^nXco84BN1FLQYkjJJyJq%sa%b3MfZvc{K50ta zA!NtQf{qW%do!=?pU=B;adJ-cV zQ40ra2ml2N)icd|!XI;i;YkX{4)5-`mt2X}GT5&$>cV3(7FzwTC!S8IeU(r;`QCdR zzovMEzMy8BvN1eeQaBtHm|P*|od&F#z#(+uH=aisri%xtHPO%=t7c@cjq&J5RY_o@ z`*Mr`wnzqGFROxd*LJf1KOyx4Kig$0ByI}4C64)zZa^jH2j72Y*jtk2Tx= z`)d-6pw;DBmyO|~I}TDIVtzVL*?ep&taCaP zc$Y_w?g;twZVA*}=MvOt%ry^Heip!0u{URoC|*fY!=L#3M9 z#G7?6$lyzVY#SC_jPe1b?s+}Db_ToUDu3bPif%P;vg$8ju)R<0#S})eLi&rvg+6qy zW8RpZaJDm?dGcRM42ioxQ5tm?iARKgmFHaI?C66{bDZ}J4|lDU*C1!CGj$xodn`&2 zK=+d1Z*0w0soNiDGs;ZIOz16gL!Dao88rH^l9x?I?a|Jdg7P)cn&$#L=XWMeM_rIB z3;)+rU`FHqMjhHqP^3Vg`s8^wya@30!>F=89tS#~0@dhl8_!vnukiXZoacNbUw9!y z5YY?TPVFf{q{CIh}s`I;}%7&(THd7?6M z^vBS3J$j3eD`*N((3q0*=alQQ=k!5}NgN_LOMLf*XvFG-S>_@rW3 zg~Id|pwRV?fXe9-r>+~8aAcLwQ0|1n+=|-I8Y|EV@2|oKv7d7eQI$B!pr_m=x!k{qq zG94R4*NZRqlAeOM2duYIy=FIvP9EXdrlF%3Zg2I6syUoR_ z$DuIo^i`Goailt6$jX=VTu;GE8usj5HTs>lby;u1m?Pt-p8H}a3%)GgO1O| zqWAIeE+bx!SyQ?(Wx2(55x8Fz(B6w2rb*f z)Oo>t%n+C|3fm<-5pfdCUTLN{!kn{-7fno>4;b;TaN2aEd*;gn#da1rX&w1MC;m+&>l;bj^?W`N8M> zy6lGYT&R7rfROD&-{zgqVPx3<3(}#{rwQhL9T4p!GmR|B;y9`vQE|{UI~t!hGdhR^ z@4ZaLLSu#~7t;12BL=!g^*77D%CN$YWS|uDCs3BD|1dhPyIdwcv0PU05o*(b>0ng9 zJkv9hyZ_SJ98DJ~DqXe2xnBJiqJ76`p~UBQl9Hawa(r< z7e>aLo-YAX?Sq4ZD;$@k!pWo4IHms!{v}vJxgue#IeMpQJ7C?zkMp$YK^`8CEo|AH za_;WOszz%3BWTdSVKYFx>Gp~!k|8$pKs`?yktyvBS+xBwhIb5@GF#2SZQ2RXEwKr|J}~ zeh513(&}R70_Cz@u5Nbg-Hts!JWLYx)Wb>&xzmbPFm9k!w|8g{(^RPKKVl5ISrB8Q zD%FU03q`9hbQuBO(^VjyOXRxWLQpV ziYB_g%UF-hwxx>YBdSc*AJUh_<6rM;ZIe$A&|Ej*s~ce0KOp%R>T;d$Nq3?0A%~Fg zZ}K#2O0vT4wB8NyVkM(ucv5=`iO#8j*`*b|j&ZYbgGsBG(2IaPyl$!Mnva@tEGvWn zuUJ!$f{x%k{`wciZFGJLOB)0%MM0q8pORg*@vKWfAI??Uk^m-a-$GzOi9Ug*1e|;5 zx!;5S-?0zp>AprjWvo8@P)@=Zbt5# zcNdjS+u1-qFw)H3;*%L;zz{nq-86wnv~$0(N6R{mj+bZU6Bg^W&F4~0_b8!Nxp0;E zwPsr@-;9iwoyvYc^3EZ(J2oCLEoa_KQf;UrX2JQ9BNZQr~5-z;{5G=Gi={8R8-jC zlg{6r$QAKA3&QjKpeZ?m;Mk{vQ50yUI1pukI zkmKf!+q&(%s_LQ#Ip+gYi|GJX&v?>Qr$S+aCLWb$|K(5rd(@{*4IZ6N&uH=5B>`+3 zyeW`0Mw+E_3o*p-+ZB`yetj{%yTu0$95kv_Gs+Mie+Mw;YgvQ-v@j^-`^zocD`;L) zP({`FJ>H)ap_X3qEUofsg6er)q05Qe33{;`Nmz!2UqfUS&XKzf)mQty<`Rg`D;>v! z)4?c%G$7Neyo5@%Ym!%Sol`7IAK`jp2#fo?vmP2+)y>Pcd_W>n*X-nuy1EGzE?Mq~ zmEQJVY=fD1_Wup0i=xq8=uCdQ4`zkuKl@CXKz)Qj4mWoZM*(Rftsd=ydN(0r_H);% zSKv;ZpL)>#&e}q%@-2t^U|vJ?D-qxqf5C5)wg(;_CXt=P104&)pgenU{Y8`E4Q)~T z$x#^&I`jR%z+M3aHg7aBcjq%t+=wrYXB3tRwffjZ1j}qC+wo40uzNZxhwg#O-*!by zh6%oZ!yGnw@5oV*9#JC(>}xBL$88pwhom-#T)&{rP9tfzd@zgK(^%@qk019~@RiXL zMd}P$EX4jBnbpw8Wa=cUDltminur4KnmC-Mak6t7FIvOzf}XuD;|!b z-5Tz-AhO1%9kaey`^%(g_#1j58J^X2&{a71sr~T0HH3JSZ75STr(7qLK-?Ph>Cslp z`EtVHN>8&_9~~MxAf8Xr=#Sk0lzuhj;q)gz&B=WqWC1x8X+_roGo?z665fO+jk;3F zN?%hBVFk!LHow2*pP7JEXZq#SM$(mmvqk~U;7EeMInreHOn<6^Ppy>JR8%}-*7!qM zNbqjcQfH+!sS7ilOCm2W!iGSY{r zsH3M*b$z}PpU;v`l)Z9_C;XD5wDpAo)Kj#|^zo22%-2;ds>b(fSZ?2y%tsBMSz1_so{Q$s4sH z?^@+oMCYZ^C$tnHo}#rbtoXin*CO;!daQL#O6^xu?RX@t@uM7O6EcP3ve)9(oesOD z0Z!r*Bmc;@q0S%$p(<$Jt%2F0?Ouf&oo#)xG5oc&(1Iv)AT4A?29o~0RM381#n=1x zOymy&=VQ}}ExiGzp+O9x8PJ$6_ETB2Mgt|Nh00n%z9yPD1Ob(Af}`W@o>PP-0+`(a zv-z%3zZf1tz?kCbhtr*B1jfL}$zt$$NejPqw}5=%Ew^mFb@27xs};E@Hqmy-?d&ju zA1dG#WYhW!n6O+=6@V5zhyFDG;}>u#g5PA+^h6~|zhMJ?otQ1?1_UM-o*Yl-AKD_k z6p0<=zuY*0BabNjZXWSU~n*=iyAW7!L2Mt)cYNcQP5}p*?$TPzF%R{?#|3 z;s5DV$#4)~N*BYq4H_9Oypc-CY!MIq%$7STsa7N=?zOMTD~x{K*3XOqC&IygXT^O| zgow36X_Wb-f`LjGBj`9IKW7CTj#-@8xNbl1#`@u^_(`y)(|2>muh#}iYbzFb1r}e3 z$sZK812(_cG(fq;R{f%-q8^J4WSuP^C~$$3Y27hsk08rNEb6-~{mchb`uKfX=|HjY ziDZrSuTdJNWaY``)yHNGj(h{nhT7{GmKGPR{bRLK3GBLPw8|$(r;>=+jq4m0j%V$7 zPuIB3FO*IC=E&PZAItR6?Cz!uj54HraT-PZlmJY!7u9c?(dJh(R$lC^Tkoe+s)O5p z@#-xk z)!BO6R$)g!Itj_YB-?qr$>Gkz1kz&sw1wqX@PW3k-o4`3_#EjM;#7IHGn+#0{$i08tJd;kbBp|Iq#n@z%afk{xsZjWJ1I=4jp%6r zDpTY%6QwzX-+$NHAnAN+xgwv=@h7&CA@}MLR!Oy<$y<|3bhlgFfe1~;Kg!m7R`RT` z_J;*mH9|>_rpI_!Z=JlxlTDR9fXnYF)-Rs?Gx^p{9&Ol!0e*wCflc|fBVA;U3%_)K zJF;uq)?S%Lo{K<~rMTzIl{v7+ZTav?pAqIPs;R&!k@~k42{{#UXFron-2Y{ksz?So z*!=O$kw*w7eofFy)6?~5m?j-SVCQVG=-Epa+S~hS@USDORu$_xtYg2kc zmETyu%W~o!AfX#-qg}t>bIGqEbfo^x?_c5naRSsh#Ue>p3u|TL19Z_z_xxo8@UNND zcdu=DEr`=*Uy&$|4Qtr4;f@c!ouC$`4jN&vcrmr&%m2Z5(dlj<<6P?6Vo;V{>+;-lUMQ6t3CZ$A1bz&Eg2Ay-GjIh|v;_j=o4kxFT}d)cjMn^XcCM?5Q2jzU z_)GPW&WGUF>?Y;PlTW=$^-|EHg0BT#>565%N9L6VmrC`}go(SO0E$13D7?6UGhIbD z*WUw&JZjD7FX{qrcNDRI#<`q0-f_7H9Kk4jm&1y}ld-YMP3wkaJ)TX~tB;qCUKp0^ zi;p-v4jnhHErK?FM-+hV-TCs`Cuno`yrJ33X?6;0aZjyKJ%=DDEG6Kr-poO1E^e0v z;B(RUhP+NewG{$D=kj|#m1^JGNbQ0zKGm^tqVFHsP*7>7ZUg|+;%BU zaba?&Yw3fUY(oqM+#MVp#W4rlR5En*_go*W+@@0P5`OOV9xg+lE2p`-Jt$@{W&<@_ zC#!`xMp%Jn&q{7K?@tPiBR4FAKo*yJvHzKz%v?6@W4k93;8eEvyQu56Vn zqlOph(`UV%##!w={QN(c^1C~He(??~DjIbuty{Mfa6xzYvkf(U;kh-Io!!|%!hPV6 z#F;tczOx*Pjz)Fg5js9Yx~K{G65l7eDG8s}V-}+Ps_JI2^q@l}5^L4F}vUtqh`P5%EH@ z$S5*8D~o=2u0DDhO{_myGDRflJ?mwq zn6G%KU{(p{E|TQ#fSqpqt{yhC=_gC4qn*_K8_i)f)1#FXI{16?pMrIK5!0%|Qb{)O z#r-*PNJR3#o>$s_+!It&-FQz?o2i_OB=MNrQ%%~>K|gs6 z5}a&gY+h%MTM(O?TaRXAHvL8)WL+fw+ZeNLoAVL<$d-2Y#^&BH{u(;4^+Z!9g$W>< zBP?d`g|EjmT2;EPmZdhroq*bBkC8l!`gOR5Z2{Daja%~eX`0c`Kcxp52@7Gx-wpp% zGsOn)(SrBxn%1m33=Aq(=ui*o9K9&dZ<8BBB`!U~jDYBK(o_stZ?K-%Kg3V_O%d@; zH9RqurKS{zkqXB6YtA$@q6mvTsN@D;MV-~4hUcTFJ+4nBnxA5nA2?!>zxgwv#Cco3 z)EQvd`h-y->ceUEu#>!ho23obul5k|M<)W-=ey+YFRg#chQ!deh&U1%;5y{SCwsa-;jp zLLm*Fo|XgJTK8=_$*<525yv^s8Z{CZIV`xeR0Co74Yv(M@#&H-n2x6{|NOyN>I@8w zo@~&_y_&7{BowM9RK=M?CL6|Wq|)W51x~5Tz&AUqTY+!n)`lOQVmjwAcp>9n7hF9m ze?Hj%7?Js4*||s|l)yaAm2p<5-(d`O(_#Sc5d^fRaBc2}F_6I+YD(T4Iqs|pd)^n! zxzUA8$kmU!X%pG&fLr`4lSrz}b2`TEXk7EbmePy0X!75yicsmAFHwxm`J`@j!>H?b z9zNoqc_mxoRbRq+xUKH{<|q|k^61hS%4=vGr*Y*=%Y;JQ8#Df&_U=0x&aQ13{*@wx z=q-pgf@mYriI89rHA>V$^v>ucO3H{bdheq5-bRnndq(e~j81~+?LFnXuKRwT^?mpI zJ@>udwZ4B||BRJ&#;|9f=dt&(_wn2I(yL<+LGqX~3UD`$$d3`-!mQxc)jPnb0QeYw z;83>6uH4s{J@Hx6bi2B}u6&5crFKHdoYHMp<}xfTI^juk`K$B5>b7}bP#4!XW)}11 z14k#pj7g9;*;D(pwML+ecm4VbWF3@FY!J5Xd-VJZN6l|>;uq4A{%}yjf5{u(Z2YTb z+Wt*D?f&jO9TGd0c2x`OdPo8hWrjZ7^CL_3*}Kcx>_tasP(~(+jqPBy45tnjcG!wL zHHbgfRl-gdawYQ#Y4RucnTC%R`5=_B2`y584{G>!?7rOND%SKtqQrJ~}Pi z?$;WqZaxqp*gCnNk)ym?t#myaYnyi0|4k&Aoc>+i@~>XMD*}6HybM}a4+`XP(JnV` zpK><;;nhO4uGm4msSM)HS5|~bNBBEq5x-O_N@eb1LE#kQF){j#I7W2;@{`h1QPBmH zQMJ``YcwWptTn8MC|~|~i!TMK^&?7!J*Pa<9_(?g8gYu~XGm6Hh^Oo26&vF-4GUrp z6PV3!z-JKo5}HH8#MXAEc0P;53u5(^&#%|MnD}DcaXI4PJ( zZz@ba*8=PzNAVC3Ls_PV(zgeDLlUd~sM2HWi1i<2n7L~OfsPQ8a?m$$Kccl`KJw+x z_)opf2%-<0I#PJl%u~mXptKdmlIxJq!T^@-P-;{nQSexuh5zv{iP~XRzLV8peDfXA&XLS#ko%4%HXvOu$$rOGLB{1vbC3fiDSWcq?sntpb zc+|E%M@NIJ5F=f*0!g^bwnhB>$3qvOVkmF%&w85`!8B9LKf;s87R23v5@z3TWtCXK z^9_xr?H%mJ9eIn*-wgg&vIDU_-(3eT_Y$1_xi7Bf@prm`9g{oH&m^eE3Eq`ox~(3!<@s+ZExzKtxPuiytGzQ1>4>5td?$uYUymXhP&D{t0uA-&0&#n?&*&k zP1)etu&nmP$Y1849;H>vBKlaPYhJWD*<0Sdjj82-Vq)S^WXXH3HKSdsA-z)NQ-)+I zLHu$Kx^&H@07i_t$jrs|dg;}VlD%oDE#4=1j=RTJ z!Ta%jJ}J+|Vr^(NC77Lyy@wcibbW6%zo-z>E&QV252#o5zEP1cq5OYBPngOuQJj#Uq4@@LkHW!>)KUOF(e!$A z)xqWYa;P-Diu7s~6=1K1bmqy8P*v-HHtCXwNxwQMhd zek#n6!(}Bx<$(5iITm~d~O)O;+{9?g*Fh$RniMy+LDA~MTd$xhu3BgPQq{gb(n zLkM$<4{OHV+bi9VgU#8eh8a`psVtz@UJjLNg`UB04NDUDPWW{=(aAU3=UHlymDb#EmipAM@wEuh>s@}czbbw!2R8VT zO7+amK{a3xB`vT`^0--R7M+oUjew`wD)$~++`AVczpQ$2Dl=iV>HDMpZo~ZK8lVzAelkgKf=4_6C$tDX`V}b{oXX^5yP}(K)wuU0W;0s_Qt4L_p(rg$Be4q8{g<8f z45`(X1&-V-&}Gza=C;GQ^+De$*CWp{tN|sc>WjJ8m*=|Z!Ss`JEA@sM>DLcKLZ$<( z6M8U^Q8?nlR7thk^YujVV*L1c&w{_a>7Wl|{EbfkTB6RwrhyRala06NHDFhRhTq#3 z&B0VxW3M|To7CS*OR*Hke!`zN|7_NJ>Q=TJ27AI@|@?Q5~t5{J97&bNz-$>lhf%MG8R*Bu*F;t4;a zN~5-EZ)Gq6WH|j|6qB?SVgIFnEhsCDtKO?JZeGGP4 z^X;6BCTU2*s2vBFvk2V1vX&|&@J^qf(m}O;<{ZVk>^dLa>@jOu&EjY+ z)y!M4wC^o$Vr>+K_zJtF$wFA~JGPLou_*19X+A;WIlV7%S)IsOVvj`Z2JUpN*97JW ziE98QzP!0ESS#ug{q}k+ScLT+`m6C2-n?K^d%e$}9#p$u`!VcxNV&19C=1y{CpGte z=J8RoydNtbn4IuVv1cYbd{rSb(RvGfiH0lZ9^r3w%fnqlQ^S={)?X%aZNbYPq-lw1 z4X-?pO)Psh+Mb->yi(f8XtKvGiP`V=T21((Ad?Z6O1pNK+IkW0WA%0@f*WhFt}Sae z|KSgIpUa{(bF_*^9)9};4~lKBr8fJeDm&25uGJopy(x;Af<)YuTOVfFM)Grfy()Cj zlZhh-tynTbMP8A&>9Sj~_9rjl%hS!w2GFM-v91LX4F%^XHR-7kb3GDEC!fgi!Jz4`*bXj5 z(|scV9Kpg^zwox^NN?6tt1NPbt99=iyuTb%{AXcg%n*~d-NO9?)HdC-A(PMc@pmGO zm<4^AAwfFh7D3+>F!sa1^8Q-w`p0c|YGbWloA-h05fP~NM~^l=E7~r^J;*77giEVfv0UeB{z=4}RNrl(GRE=^Np4B3CTx%T18DQ%|?Hrn0U|Gbwg zy#Q$QD0?J(o;5*K?xQh(^1L5R9egQN%EF<~jJ=84b9P)Ba5ti(K?l{cGDfc+dY{JI z&GUSA6Ok9Ye*!%|ry9%>KFDZwT`q8B+Hleec_@_qyk{lS+X1FD8UtNQ&r#n#B3nUN z)58AwSkZQliPoo7Dc$y%h|^#j1IyzMslU`08ZMlUzPVtOxIu2Htv?eu+PydUfs`vE z*~4fNC+B?Mz@Ll|Yq4aR_w~aE$@W+W>2GXf!&&C1zPPUxq4cV6EMhQEgxR%i`lm(! z$21E@iy+NF^M9@lR41vq!{E0|v7&1^a260q9Qo8m3Oo7T+5ZHmf=r3iiiZJuN)Z&{x|FB+*i zwe=1)`|R$t{=8FF9F%W3eDES^FuKvb;*|kT^ix$0Kcbj=*z$BkgC;zuCmje7s!a>* ze_#ECU5Hz6^6EDo+j8()#D4Ki*R{J0b7A zHM=XgDwQTHA8N3DSh^z5wlFlr)$=e?N`i}wJJl^({OV(2ct{Wt`GzS&&7HO`NPxEB z)H{+6|F4&EQ^9Ihe#w~(iqXry6-8s)ayt`HKhR=cwIIUn_ju=brxa$H% zN*OC_Y|mQ10gUa#$mc%?L{wV#j`&N+qV?SF0SxWocqPaLuh) zOFC85Q(79;Cg*(O-HS-$2v4pZabL zx6iw|(a!(p&OM1=q0A2QYLg9*SfRJ6b{m$@!v^ct{irnZM6~lgeTD9=j=g@_ob1x` zf*b&4LtZ^T0bIXwu4tG=`o>@m^iC^e`4bK@~6t|&o5vYp&=M!-lGs>g-(cq+hiR}k89t7 zWOfeJkv-3CG+&ckm4~}$rS)O7VXY^%6P+SM3Nwn@Ad&xu3G`R$QjUmeOtL>cbFXH{ z57eqPf59$R2_+@b^|n4>2iw2m-2d%=2Nu7VIzx?r%r*8b&jSfgocI)AN3(28`MV=K z(>?p^H~YOgO#y90eZAV6&HDGs?BB1$^2mEq#9EXRq8Xt8Go$ne`i2)Y$f|AiW)|Bp zJ;{Pp_)w=YIU`_({i0IiLmftoF}(`kSDJ+zG!Qke^T4=+;zp+ zKeS=)Ixn_A{XN!xH#?FY80c|6&|D}t$h6k`!_Wy1lA&0)1tlw22fUjiQnw6Ye3DD9 zAQA$LrMCtB(22Q2G-=s5UrUkY4`YjsLGu~Sjy;Yr1q9akpx%TSn|t2G-twXKu<%WX zr1i=~Vc&=28~%Hgm?)3lM>#QBFWN@yQaK|UZ&Y$S_Ywx=N;bAYC-1zY9aaSu?b7XE z0?D>0EzQUPCKEj$=5bk$>Ml$<_Te8)(rIX$t3>~@2iq))kTM+An#Dk`a}_HW8|DOJl^vQsL30JN`-q)o}; zDP-gRx6JB0#6(BE$g1K9O9F-wZjwf>_u14Ase#sH3$Ob@#UQ$A1%o6eCSO#R4d8fE z#I3dl{kfu2%^Wn~>+*9niUho4Vob~K-I9C?$Cos5;j|)Jw~?-ApKhR9?HA=8LJO5p z_491AD^;Z{-6qzzW_zj~3pYrcLG6&C5S~A0=RW+TV1t9!8uT7;<8Jy6SxPM*cu3fE zZAyHv9 zaRAKLe(vS%Ro#oC+bYGN&%0R7QcYYBU{GFjPo8sJ@LLvH&nD>2qbcj zbV*jwIDY+u)D-NgDj|hafIIe#1L`Xuf4U@L_EjuDay>djsU7kuoN_=UQo`axmw~yE zL|>N>wGlI7f}K% zh;(U#{8xH)h|BI$kjj`Bpz_p_Q6b7kMHZE-)-B5bUZDG>oR3!KdBd(?aUl8j zh$C4*BB?GEZ?A_D+LH|ld5&|0MXMFg^GHq(Ctv5-j*vmc%wby#j__J$@cl_UAf&yN zS5iVkThCvFN>7hGEFxc&E^Z-~D0cWgQ?Abz6RhrgG0o{p! z8ARU9+zjc+=8J?^tknHb>`YYNXBhe@dgFP1ME2go@_C#t$7KmD=T+GNXQFhvApC2O z0J&dB{8;H*n6a2OfFrkK4k2_I6r=B(kx{z&EX#Y#V0DX|$in+Wyx`+1HowLi+(Me8+I53LDQ~ zKi%>OGTANJ>6Mz6o{Xa^MY4^cuz*Bp>X5o|p>hE0aK~B;yY)ntulz|RGT2C#og8co z%o@5$!EErA&G2a4E&>)J^n6;GpK%&Bu&PKtu%G7OKokz>7>;BXh$kc~zdGCaLjf|a z{28mqVnNY@ZuW-}zVMS^dNw1^4eu)JlC}{F;BbTwD99V@Og||w<=5U0IvLth zsmW{&?EvdMEKV-}>g_Fxf-Fynpcw}V11B^Cm3+#aHC`Yeg@5mx2VKS?SHrs^sa5Dx zoU@fm_O)XvZRASFoI#IJ)!w4Yij+lnA02IbMRCCuumQv6P+A1hreiF{IFYJ4ZT$C` zp5}$lV3y!2_4M!KNTQS2d7ch4kSuQVjix@PzK~nYphTTg_=bJcHRlZ=np&R1n{?A; zQXG0255pBnk)v3nXjA0wfX4gqDs?V{NenR8bXAv1^5K06jhg{{5YVYfcT)kPRnYrr<(zEc)PZvTrEIh*O8!1o$Ahs<>86=T*#{2TAk zaI37mM;V}JkepHz9adpaR;Jg*tf=OXgns~(;H`f(_65eUjqi?Dq!9n4Ypw^i94#Ql ze;SQItqD+03BN9a$6-wQ@E^XmB%+uTUzCe~)>6>&Az4osKM<$NB+>&)xAbs!gG2{F zlQdGLf^3)UGPq^Me|*$cJ3S_0CY6!JsHkVg0q>kgH%9_yx^CQz!*Z%PLtgrX7{Bla zhMO!QO5|pEs85S9T~<#Kg5>FYlo5SOjQJ6AlgqO9R-?PNkcUOe`hoX8qtk~b*&KTx zBBF>A16ua>LVzF32GGX6{zUhjOwAt^W)l=~U}zdkN4G6fEOGqC6e%U}h2XuwQ7=5_ zwds)qcXMuwShv;kpV;Y^iGLJWsCIQ&9-<3?k%HH8blPiUQslMEfSn`pHliSCxqpe} zV<`j~&f$l9Z}@_xeB_XIL_%-`OxR5TB{d?b~!fkt?s0L?BU1s9_bBR=r zmv_47{+OZ^NV9AzZTE&7jS=vE_#!l4&hl9SK==~`wu?!_GbJyBw2BAqB^fBB{S=Y+ zz4`kI1DmC~E*Nje2+)A6U(`QgU#{Fi`lr1lU$SCEGtY2tNpVI(((7rt7E{J{G94Tq z`dY*~D4mX0v)r66jNwl@;j%D@Z7x11U;Pwg`6M&rr+iIs#;Yeaz>2Y|*9&J(9=LgH zI>9@-D4f6Ll!)04LHKjU91`rTmy8WvIWuQwRLM3O333S^I{3?JqT@1Hs&b>%1f6{A zYpXzeA@$7_|wHUQuf|B@%? zmf9ov?8vCc;fHz zOad$EneTt5H43IiPS%KgW0(;TWx2^mNmEkY_AK#*>&UUw7OMR#r{uJ{O8lR{No*Wo z?38z{X+51A%Hhk)Nf_}M6<%$7@0BbEe!wggc!FtgYxt9fA-nHt7p_q-w#7wu;KQnxH6br+4tj@p=mKEOPyi+Wj^5Fv?>~l< zku<)j?B^ZJ?$e@vjj$}>fbv0pQzuE+XH=;wuP2Md@~0akGWO+V&<^itKxZG?L>k|< zSASLWzs)S+x%TRrtEmw~Q^%TYV$m>H{6>*Oo*6V5iFbX51%C7N*@K}s4{vF5G5ue| z=KmTtzYUv@*C9zQ*^&{e6lzxHF>OC0iy-3)ZJ7FUY0$LJk|d(`3(tm}myZwr;lmBj zUuYS=(1dG&;5r>65ZeF~erHE^af{o~<(B}$B99WZV%C7Xs1b6zOmY<#Gcm6L(1HWZ zcqFUxkf>Gpxrwl91CTcuS&5(kt44V`{xUA~UdMKK?|mT3dAqmOC%==aWG0BUx_Y54 zFJlUG9B#is#ufXy+c66B@|rMM1S)PR$5WpAXj~Yu#YakN4IeQv>e!(qbvL44-q>G5 z;K59m!(92FEc1QRWtH8}?*0o{`2SC@c?1QN-G6{Ei0O}7ClQPi=iGOR_s-qmag(3+ z!5Lsj=ao+vB}i(ld9$S7OJj<)C_Gapav-P*d4ZzX70XxL$V zx3mitIjnAY&nPGu8+MHWbd7WXr<9G-(v%uG-SRH;h%pR*+Ys+bw@Ul;W2S|0(n~{# zGEG#c8VPfNi^k>azfwG$&~JXU5?fbZHy8nm(CAS;|eRDN_fZ9NJrZT5VnJ z$sR#*Jzk2R0@8W+=%~J4EKIsT=D{O8l-qTDcKP!Z*JN-x2eG_{ND@1B`Jvo)Koj*8 zQ+))eMJPgF`uYN34q!|UE;r`C4O!qHR>$64bHezrf3_zrbZ?ClmiBo_~1Y#Otv~=!<6dGb~Gz7HMiKI-x zZdCGjINOP4Y=YTb+CI0$sDn7fJ0#Kz?ketcjYWUc+ybbQ`>UN?&}EBHBHZo%UJpA; z%kU`qB^Ia_79aDU8m)ZdP2QV$pzRRj=dkslKjNPR%l{dCrjQ9dmPatYi3+?+=`q(J zgByZ6>jXm`X&Xu-K3XXN!uK75VasyG)i?nH6_Z}p0)G`B1YfgZO2x{D@VQ&|NDMhp z@&i|J0ZLlhnr}(gJIq%c$c4=u$rwHxUl1&r302FHDfO|S+?f~Z6yjE>(=IO~XrA3; zP)a^gm=)I!(nc=#k=^r6&Z&nvI9IC5Mmec%B^BUwBk;%Fm8*y;spvQT1K6O2t4>DJ zs~<>s%=G{9mOwG_!wmDmH|?kN=@+$|We+{^V-BGR?9>aeE1iGFMZB~FXIr2U_x9?b zJ|aYXF3npDUHiUfD#!qNe}zRAr~=yv<6alt7Wt5 z6QmU2NON{?@JGpnXQ5l`&MSGkP4a);#Q#Rw{RA9AM_!sBzp4$KIdwZAu?yy=Nn|%* z&y-SqOh(EI^k3w1>f;uY?dwmO`(jUW z2d{%#8wbF*fn-bp%B+pFgsUT;tLFPclXOG<4^yw`>kIVXOyN+ToLv1Y-tGT0H1ekF z1=E8EvVuF248R1dV}O|(x~%DZhOLjA^}9Qq*^f%h2QWOPv-8UaF2w?j zQ{dA7Az3q@z@}rd#INmULJN?0s#L&+dPj%t+I{oR$^+s54c+$n!JgY@m_?+g2Qb}T*@h_-`$eE6NoV1(T!SV>l&OtlJT!rXm(NWZX7&|F$zj~`bkr4t0Qnu z_-C{8Clc@w-9{o}+|};YcmZ%Cf^GWmC*-x8vbR{xpwVRFPjn?O0T;cZWi*@dNt6v4 z@GQM@IbZY`zQET)yL}>hhClIh)gW(PB%kG9xHomiw=`_*zH$o;@^Gq#3PuhtH5ki; zk#YI)vK6jbw(a#x6gIdwIyQUm+lXGu(S&#%T8?J>D*y$Pf*)YV=!W;NR*u_E&ki~r zv{Hc}*XKC*wjalO{#1P+wBhXh;xl>Z__58+%Wi|@!}Aiw`b?ieyb1G_`qOD0vzC9F^0;iJ^ zgDG|&4R5J}uNS=lyF)vvo@LLNQNql45^sFr1ErZE#;IoETUn#<6A+|Vt<3LRlVyyf z@l?ct;P;FctWj|%{d86MzSa%Yp;YCip>S|=SM&BW@vA#Vg7q1 zxY2g06Da#jWMs@yr(zH?WUkO{zq7IomvX%c2LI~a$a7@^wcQB}Mj~;e&mBI9lPB=b zN5?M|JjN}FCG2Fr7&{gjPM3OPIC-r-0w|Is(%%RBv+Gim`DQdj?T=8@+u~Mx{aX4Q zrO-E5-@r4?4j~Tuan2RM@`ZOj=0%doetexzdEBu-Ld@0GF=lfpbq@uJqoXZLuF%Fv zwDMK7x*b(S`S>z%U^&qG!McNgK}W(VhM_|S4A4T4SC4eA3xxmFm}I2~Q(6KK#HG%h z#*JSdQ*&1~M;$(->9FKYRdJLpdhrZ!U0`ZP#hl%_k~u&)AKfim2P07Y`) z+0{X9TVTe^=hpH{Bw3@T7-anbaOiY0Yfl%9vj3=cNYTS!5p8)xWQh9nH>r9bodr zfX7e7ud-Vf^Dw*}Z6ETTNm$kH6>rCt?c(tq{`i|+`i)ZCAJ!Y4So8Xr)fS}QlXXt(QDbNp1@Z*@tJQlV0| zgB=PkRvdL#wEwGp{s)xXe@AXi){b9r-ktpjQ8oyuzkG$LPst}w$52Iw$tK_)y@GUg_!VsTc(RAgYa&VSpjNkzD)q2&?_dCPaYM5F+}iRXVyxry!-X%n0b zTSO-m^8OBn0@oEed;1r8zhsX0{kVa`RqEU+vxt5pRe`?c z^Ukrv(#2ZcaM}xndIsPkK+4j=i5ukX#z`4Qb~gKdKV!+ohpT~ zCqNV30T@!p<>l{X&-{OQ5Z4nl1xMzDA}X@5`nrc%e=b`SE~c&ZYn|-bob!n2o^!){ zSEfr@8yl_Wz0+W6iq*ewb~+&M`yUu?Kxg}NsMh@-hC*L)DCqX&z@|fBuBwF|_QxawGS%)|jQ7dPr2Bca zz=N}`;e18ncwiyd?qmCu2m8BU$>#^pxZk86Bs|u|KI>Sjwmx|WAZ(i+hKC`O;LP9C z&FPa)hY9C`#W%v>YdNOyh`b7goZ4~e{si@W4UI8`g&xY+_h*e#X&ScoeE)ctQa5R4 z-`wJ&Jl!Gs_ZYwfm>>LyIO#4Ufm6FKyj=3CVq>mb8#y_Z@5P zn9xZ6qzFSL=pVTR)9f>aD-_j4H_qt19$-7iNp7~!hw3bXAreYalIQa6l3J) ziYgvN}6a!v*jh3baK$|>SNca8Z0nJxax8G?OQTVN}{6mRZ&;^v3 zeH0$$y7&C4+vDThpaTE;aj!|LC zkf`MxiY?m0x5lxncp~OCKtR~~2NL7=<`G>sMrR-p@j#F!74|YOJf=lZ|FN{b$*F>P zRB35vUW0pKpE|TNPAo^bwVz(?I`C!vhY|OFD6HG9kfMySKdZu(H-nV?XjR0EO)BtM zHw~upUjdj74rtKvrnS<-BB(S;pxT|rLp`85OCf7?!;gRNcYt^T+|m)~3iDE<3qthe z%z+k3(;fMD#qFO4y(ym(wxKT-GK46MAW5-@`{pH(PWm6O#0%G;Uv+I+covQP@5`@3 zuXPuZ9;1tCLp$%4fypJ0W0i$Ynyke)gsT#GlvMgJQk3d|ONi{EV}fUKr4VeNZHxe~ z`TRQH0CX8Zh2%dOKEy}bFWH=XE>-pHYy+v5^+exSd1^+G6kU)F#GQ_ScGjg3SS?+d2VG6K?KaXBw=9Z}7*5ZPkA!mQ;{YZQd~&X%Dy*#O zL!w=3aP?~i0vSA`AMzu2&_MNi<$7G(Xp-|rT z%%+c7oQTyS{ajqd4(Zn!EVUnW zo8DtIh^hdVrstU$2CK#Ss|zzjmv_?VF6Wn0zPTJ*uF0Kq@~KTnYn}h9(i*Qffj;$T zx#^2I`VHGPAadY{es z2^qrM>DdmhesKLKr=L4JMU;@rxULXL6L~MA7j=;i%O)8F7!ObU?KX% P0ADX5N>arVZ$ADnbP~~a literal 0 HcmV?d00001 diff --git a/README.md b/README.md index e3588728..41344680 100644 --- a/README.md +++ b/README.md @@ -67,38 +67,40 @@ This example would look like this: ![Simple example](.github/assets/example.png) -Note that you can also add environment variables in the your configuration file (i.e. `$DOMAIN`, `${DOMAIN}`) +Note that you can also add environment variables in the configuration file (i.e. `$DOMAIN`, `${DOMAIN}`) ### Configuration -| Parameter | Description | Default | -| --------------------------------- | --------------------------------------------------------------- | -------------- | -| `metrics` | Whether to expose metrics at /metrics | `false` | -| `services` | List of services to monitor | Required `[]` | -| `services[].name` | Name of the service. Can be anything. | Required `""` | -| `services[].url` | URL to send the request to | Required `""` | -| `services[].conditions` | Conditions used to determine the health of the service | `[]` | -| `services[].interval` | Duration to wait between every status check | `60s` | -| `services[].method` | Request method | `GET` | -| `services[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`) | `false` | -| `services[].body` | Request body | `""` | -| `services[].headers` | Request headers | `{}` | -| `services[].alerts[].type` | Type of alert. Valid types: `slack`, `twilio`, `custom` | Required `""` | -| `services[].alerts[].enabled` | Whether to enable the alert | `false` | -| `services[].alerts[].threshold` | Number of failures in a row needed before triggering the alert | `3` | -| `services[].alerts[].description` | Description of the alert. Will be included in the alert sent | `""` | -| `alerting` | Configuration for alerting | `{}` | -| `alerting.slack` | Webhook to use for alerts of type `slack` | `""` | -| `alerting.twilio` | Settings for alerts of type `twilio` | `""` | -| `alerting.twilio.sid` | Twilio account SID | Required `""` | -| `alerting.twilio.token` | Twilio auth token | Required `""` | -| `alerting.twilio.from` | Number to send Twilio alerts from | Required `""` | -| `alerting.twilio.to` | Number to send twilio alerts to | Required `""` | -| `alerting.custom` | Configuration for custom actions on failure or alerts | `""` | -| `alerting.custom.url` | Custom alerting request url | `""` | -| `alerting.custom.body` | Custom alerting request body. | `""` | -| `alerting.custom.headers` | Custom alerting request headers | `{}` | +| Parameter | Description | Default | +| -------------------------------------- | --------------------------------------------------------------- | -------------- | +| `debug` | Whether to enable debug logs | `false` | +| `metrics` | Whether to expose metrics at /metrics | `false` | +| `services` | List of services to monitor | Required `[]` | +| `services[].name` | Name of the service. Can be anything. | Required `""` | +| `services[].url` | URL to send the request to | Required `""` | +| `services[].conditions` | Conditions used to determine the health of the service | `[]` | +| `services[].interval` | Duration to wait between every status check | `60s` | +| `services[].method` | Request method | `GET` | +| `services[].graphql` | Whether to wrap the body in a query param (`{"query":"$body"}`) | `false` | +| `services[].body` | Request body | `""` | +| `services[].headers` | Request headers | `{}` | +| `services[].alerts[].type` | Type of alert. Valid types: `slack`, `twilio`, `custom` | Required `""` | +| `services[].alerts[].enabled` | Whether to enable the alert | `false` | +| `services[].alerts[].threshold` | Number of failures in a row needed before triggering the alert | `3` | +| `services[].alerts[].description` | Description of the alert. Will be included in the alert sent | `""` | +| `services[].alerts[].send-on-resolved` | Whether to send a notification once a triggered alert subsides | `false` | +| `alerting` | Configuration for alerting | `{}` | +| `alerting.slack` | Webhook to use for alerts of type `slack` | `""` | +| `alerting.twilio` | Settings for alerts of type `twilio` | `""` | +| `alerting.twilio.sid` | Twilio account SID | Required `""` | +| `alerting.twilio.token` | Twilio auth token | Required `""` | +| `alerting.twilio.from` | Number to send Twilio alerts from | Required `""` | +| `alerting.twilio.to` | Number to send twilio alerts to | Required `""` | +| `alerting.custom` | Configuration for custom actions on failure or alerts | `""` | +| `alerting.custom.url` | Custom alerting request url | `""` | +| `alerting.custom.body` | Custom alerting request body. | `""` | +| `alerting.custom.headers` | Custom alerting request headers | `{}` | ### Conditions @@ -121,7 +123,7 @@ Here are some examples of conditions you can use: ## Docker -Building the Docker image is done as following: +Building the Docker image is done as follows: ``` docker build . -t gatus @@ -194,33 +196,37 @@ services: - type: slack enabled: true description: "healthcheck failed 3 times in a row" + send-on-resolved: true - type: slack enabled: true threshold: 5 description: "healthcheck failed 5 times in a row" + send-on-resolved: true conditions: - "[STATUS] == 200" - "[BODY].status == UP" - "[RESPONSE_TIME] < 300" ``` +Here's an example of what the notifications look like: + +![Slack notifications](.github/assets/slack-alerts.png) + + ### Configuring Twilio alerts ```yaml alerting: twilio: - sid: **** - token: **** - from: +1-234-567-8901 - to: +1-234-567-8901 + sid: "..." + token: "..." + from: "+1-234-567-8901" + to: "+1-234-567-8901" services: - name: twinnation interval: 30s url: "https://twinnation.org/health" alerts: - - type: twilio - enabled: true - description: "healthcheck failed 3 times in a row" - type: twilio enabled: true threshold: 5 diff --git a/config/config.go b/config/config.go index 23638975..2ba7fe9d 100644 --- a/config/config.go +++ b/config/config.go @@ -22,6 +22,7 @@ var ( type Config struct { Metrics bool `yaml:"metrics"` + Debug bool `yaml:"debug"` Alerting *core.AlertingConfig `yaml:"alerting"` Services []*core.Service `yaml:"services"` } diff --git a/core/alerting.go b/core/alerting.go index 530a6dda..5f595f13 100644 --- a/core/alerting.go +++ b/core/alerting.go @@ -2,9 +2,11 @@ package core import ( "bytes" + "encoding/base64" "fmt" "github.com/TwinProduction/gatus/client" "net/http" + "net/url" "strings" ) @@ -70,3 +72,64 @@ func (provider *CustomAlertProvider) Send(serviceName, alertDescription string) } return nil } + +func CreateSlackCustomAlertProvider(slackWebHookUrl string, service *Service, alert *Alert, result *Result, resolved bool) *CustomAlertProvider { + var message string + var color string + if resolved { + message = fmt.Sprintf("An alert for *%s* has been resolved after %d failures in a row", service.Name, service.NumberOfFailuresInARow) + color = "#36A64F" + } else { + message = fmt.Sprintf("An alert for *%s* has been triggered", service.Name) + color = "#DD0000" + } + var results string + for _, conditionResult := range result.ConditionResults { + var prefix string + if conditionResult.Success { + prefix = ":heavy_check_mark:" + } else { + prefix = ":x:" + } + results += fmt.Sprintf("%s - `%s`\n", prefix, conditionResult.Condition) + } + return &CustomAlertProvider{ + Url: slackWebHookUrl, + Method: "POST", + Body: fmt.Sprintf(`{ + "text": "", + "attachments": [ + { + "title": ":helmet_with_white_cross: Gatus", + "text": "%s:\n> %s", + "short": false, + "color": "%s", + "fields": [ + { + "title": "Condition results", + "value": "%s", + "short": false + } + ] + }, + ] +}`, message, alert.Description, color, results), + Headers: map[string]string{"Content-Type": "application/json"}, + } +} + +func CreateTwilioCustomAlertProvider(provider *TwilioAlertProvider, message string) *CustomAlertProvider { + return &CustomAlertProvider{ + Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", provider.SID), + Method: "POST", + Body: url.Values{ + "To": {provider.To}, + "From": {provider.From}, + "Body": {message}, + }.Encode(), + Headers: map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", provider.SID, provider.Token)))), + }, + } +} diff --git a/main.go b/main.go index 23d72080..ba8fe163 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "bytes" "compress/gzip" - "encoding/json" "github.com/TwinProduction/gatus/config" "github.com/TwinProduction/gatus/watchdog" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -53,12 +52,11 @@ func serviceResultsHandler(writer http.ResponseWriter, r *http.Request) { if isExpired := cachedServiceResultsTimestamp.IsZero() || time.Now().Sub(cachedServiceResultsTimestamp) > CacheTTL; isExpired { buffer := &bytes.Buffer{} gzipWriter := gzip.NewWriter(buffer) - serviceResults := watchdog.GetServiceResults() - data, err := json.Marshal(serviceResults) + data, err := watchdog.GetJsonEncodedServiceResults() if err != nil { - log.Printf("[main][serviceResultsHandler] Unable to marshall object to JSON: %s", err.Error()) + log.Printf("[main][serviceResultsHandler] Unable to marshal object to JSON: %s", err.Error()) writer.WriteHeader(http.StatusInternalServerError) - _, _ = writer.Write([]byte("Unable to marshall object to JSON")) + _, _ = writer.Write([]byte("Unable to marshal object to JSON")) return } gzipWriter.Write(data) diff --git a/watchdog/watchdog.go b/watchdog/watchdog.go index df8403cf..c237dcfa 100644 --- a/watchdog/watchdog.go +++ b/watchdog/watchdog.go @@ -1,25 +1,34 @@ package watchdog import ( - "encoding/base64" + "encoding/json" "fmt" "github.com/TwinProduction/gatus/config" "github.com/TwinProduction/gatus/core" "github.com/TwinProduction/gatus/metric" "log" - "net/url" "sync" "time" ) var ( serviceResults = make(map[string][]*core.Result) - rwLock sync.RWMutex + + // serviceResultsMutex is used to prevent concurrent map access + serviceResultsMutex sync.RWMutex + + // monitoringMutex is used to prevent multiple services from being evaluated at the same time. + // Without this, conditions using response time may become inaccurate. + monitoringMutex sync.Mutex ) -// GetServiceResults returns a list of the last 20 results for each services -func GetServiceResults() *map[string][]*core.Result { - return &serviceResults +// GetJsonEncodedServiceResults returns a list of the last 20 results for each services encoded using json.Marshal. +// The reason why the encoding is done here is because we use a mutex to prevent concurrent map access. +func GetJsonEncodedServiceResults() ([]byte, error) { + serviceResultsMutex.RLock() + data, err := json.Marshal(serviceResults) + serviceResultsMutex.RUnlock() + return data, err } // Monitor loops over each services and starts a goroutine to monitor each services separately @@ -33,33 +42,39 @@ func Monitor(cfg *config.Config) { // monitor monitors a single service in a loop func monitor(service *core.Service) { + cfg := config.Get() for { // By placing the lock here, we prevent multiple services from being monitored at the exact same time, which // could cause performance issues and return inaccurate results - rwLock.Lock() - log.Printf("[watchdog][monitor] Monitoring serviceName=%s", service.Name) + monitoringMutex.Lock() + if cfg.Debug { + log.Printf("[watchdog][monitor] Monitoring serviceName=%s", service.Name) + } result := service.EvaluateConditions() metric.PublishMetricsForService(service, result) + serviceResultsMutex.Lock() serviceResults[service.Name] = append(serviceResults[service.Name], result) if len(serviceResults[service.Name]) > 20 { serviceResults[service.Name] = serviceResults[service.Name][1:] } - rwLock.Unlock() + serviceResultsMutex.Unlock() var extra string if !result.Success { extra = fmt.Sprintf("responseBody=%s", result.Body) } log.Printf( - "[watchdog][monitor] Finished monitoring serviceName=%s; errors=%d; requestDuration=%s; %s", + "[watchdog][monitor] Monitored serviceName=%s; success=%v; errors=%d; requestDuration=%s; %s", service.Name, + result.Success, len(result.Errors), result.Duration.Round(time.Millisecond), extra, ) - handleAlerting(service, result) - - log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s", service.Interval, service.Name) + if cfg.Debug { + log.Printf("[watchdog][monitor] Waiting for interval=%s before monitoring serviceName=%s again", service.Interval, service.Name) + } + monitoringMutex.Unlock() time.Sleep(service.Interval) } } @@ -72,10 +87,43 @@ func handleAlerting(service *core.Service, result *core.Result) { if result.Success { if service.NumberOfFailuresInARow > 0 { for _, alert := range service.Alerts { - if !alert.Enabled || !alert.SendOnResolved || alert.Threshold < service.NumberOfFailuresInARow { + if !alert.Enabled || !alert.SendOnResolved || alert.Threshold > service.NumberOfFailuresInARow { continue } - // TODO + var alertProvider *core.CustomAlertProvider + if alert.Type == core.SlackAlert { + if len(cfg.Alerting.Slack) > 0 { + log.Printf("[watchdog][monitor] Sending Slack alert because alert with description=%s has been resolved", alert.Description) + alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, true) + } else { + log.Printf("[watchdog][monitor] Not sending Slack alert despite being triggered, because there is no Slack webhook configured") + } + } else if alert.Type == core.TwilioAlert { + if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { + log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) + alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) + } else { + log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio isn't configured properly'") + } + } else if alert.Type == core.CustomAlert { + if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() { + log.Printf("[watchdog][monitor] Sending custom alert because alert with description=%s has been triggered", alert.Description) + alertProvider = &core.CustomAlertProvider{ + Url: cfg.Alerting.Custom.Url, + Method: cfg.Alerting.Custom.Method, + Body: cfg.Alerting.Custom.Body, + Headers: cfg.Alerting.Custom.Headers, + } + } else { + log.Printf("[watchdog][monitor] Not sending custom alert despite being triggered, because there is no custom url configured") + } + } + if alertProvider != nil { + err := alertProvider.Send(service.Name, alert.Description) + if err != nil { + log.Printf("[watchdog][monitor] Ran into error sending an alert: %s", err.Error()) + } + } } } service.NumberOfFailuresInARow = 0 @@ -90,33 +138,16 @@ func handleAlerting(service *core.Service, result *core.Result) { if alert.Type == core.SlackAlert { if len(cfg.Alerting.Slack) > 0 { log.Printf("[watchdog][monitor] Sending Slack alert because alert with description=%s has been triggered", alert.Description) - alertProvider = &core.CustomAlertProvider{ - Url: cfg.Alerting.Slack, - Method: "POST", - Body: fmt.Sprintf(`{"text":"*[Gatus]*\n*service:* %s\n*description:* %s"}`, service.Name, alert.Description), - Headers: map[string]string{"Content-Type": "application/json"}, - } + alertProvider = core.CreateSlackCustomAlertProvider(cfg.Alerting.Slack, service, alert, result, false) } else { log.Printf("[watchdog][monitor] Not sending Slack alert despite being triggered, because there is no Slack webhook configured") } } else if alert.Type == core.TwilioAlert { if cfg.Alerting.Twilio != nil && cfg.Alerting.Twilio.IsValid() { log.Printf("[watchdog][monitor] Sending Twilio alert because alert with description=%s has been triggered", alert.Description) - alertProvider = &core.CustomAlertProvider{ - Url: fmt.Sprintf("https://api.twilio.com/2010-04-01/Accounts/%s/Messages.json", cfg.Alerting.Twilio.SID), - Method: "POST", - Body: url.Values{ - "To": {cfg.Alerting.Twilio.To}, - "From": {cfg.Alerting.Twilio.From}, - "Body": {fmt.Sprintf("%s - %s", service.Name, alert.Description)}, - }.Encode(), - Headers: map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Authorization": fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", cfg.Alerting.Twilio.SID, cfg.Alerting.Twilio.Token)))), - }, - } + alertProvider = core.CreateTwilioCustomAlertProvider(cfg.Alerting.Twilio, fmt.Sprintf("%s - %s", service.Name, alert.Description)) } else { - log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because twilio config settings missing") + log.Printf("[watchdog][monitor] Not sending Twilio alert despite being triggered, because Twilio config settings missing") } } else if alert.Type == core.CustomAlert { if cfg.Alerting.Custom != nil && cfg.Alerting.Custom.IsValid() {