From 93439e90dcb825d32d863e05df1d2ac3c18dfdd1 Mon Sep 17 00:00:00 2001 From: Pradyuman Agrawal <107980189+pragrawal-px@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:24:12 +0530 Subject: [PATCH] Add portworx (#1094) Signed-off-by: pragrawal_pure --- assets/icons/portworx.svg | 29 + assets/portworx/portworx-5.1.1.tgz | Bin 0 -> 66769 bytes charts/portworx/portworx/5.1.1/Chart.yaml | 26 + charts/portworx/portworx/5.1.1/README.md | 239 + charts/portworx/portworx/5.1.1/app-readme.md | 8 + .../crds/core_v1_storagecluster_crd.yaml | 4247 +++++++++++++++++ .../5.1.1/crds/core_v1_storagenode_crd.yaml | 191 + .../files/portworx-cluster-dashboard.json | 975 ++++ .../5.1.1/files/portworx-etcd-dashboard.json | 1189 +++++ .../5.1.1/files/portworx-node-dashboard.json | 1529 ++++++ .../files/portworx-volume-dashboard.json | 2247 +++++++++ charts/portworx/portworx/5.1.1/questions.yml | 43 + .../portworx/5.1.1/templates/NOTES.txt | 24 + .../portworx/5.1.1/templates/_helpers.tpl | 287 ++ .../post-install/px-create-cluster-token.yaml | 134 + .../pre-delete/delete-storagecluster.yaml | 41 + .../pre-upgrade/retain-daemonset-install.yaml | 97 + .../5.1.1/templates/portworx-k8s-secrets.yaml | 8 + .../5.1.1/templates/portworx-operator.yaml | 103 + .../5.1.1/templates/serviceaccount-hook.yaml | 48 + .../5.1.1/templates/storage-cluster.yaml | 606 +++ charts/portworx/portworx/5.1.1/values.yaml | 269 ++ index.yaml | 33 +- .../portworx/portworx/overlay/app-readme.md | 8 + .../portworx/portworx/overlay/questions.yml | 43 + packages/portworx/portworx/upstream.yaml | 8 + 26 files changed, 12431 insertions(+), 1 deletion(-) create mode 100644 assets/icons/portworx.svg create mode 100644 assets/portworx/portworx-5.1.1.tgz create mode 100644 charts/portworx/portworx/5.1.1/Chart.yaml create mode 100644 charts/portworx/portworx/5.1.1/README.md create mode 100644 charts/portworx/portworx/5.1.1/app-readme.md create mode 100644 charts/portworx/portworx/5.1.1/crds/core_v1_storagecluster_crd.yaml create mode 100644 charts/portworx/portworx/5.1.1/crds/core_v1_storagenode_crd.yaml create mode 100644 charts/portworx/portworx/5.1.1/files/portworx-cluster-dashboard.json create mode 100644 charts/portworx/portworx/5.1.1/files/portworx-etcd-dashboard.json create mode 100644 charts/portworx/portworx/5.1.1/files/portworx-node-dashboard.json create mode 100644 charts/portworx/portworx/5.1.1/files/portworx-volume-dashboard.json create mode 100644 charts/portworx/portworx/5.1.1/questions.yml create mode 100644 charts/portworx/portworx/5.1.1/templates/NOTES.txt create mode 100644 charts/portworx/portworx/5.1.1/templates/_helpers.tpl create mode 100644 charts/portworx/portworx/5.1.1/templates/hooks/post-install/px-create-cluster-token.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/hooks/pre-delete/delete-storagecluster.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/hooks/pre-upgrade/retain-daemonset-install.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/portworx-k8s-secrets.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/portworx-operator.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/serviceaccount-hook.yaml create mode 100644 charts/portworx/portworx/5.1.1/templates/storage-cluster.yaml create mode 100644 charts/portworx/portworx/5.1.1/values.yaml create mode 100644 packages/portworx/portworx/overlay/app-readme.md create mode 100644 packages/portworx/portworx/overlay/questions.yml create mode 100644 packages/portworx/portworx/upstream.yaml diff --git a/assets/icons/portworx.svg b/assets/icons/portworx.svg new file mode 100644 index 000000000..c4aaac2ee --- /dev/null +++ b/assets/icons/portworx.svg @@ -0,0 +1,29 @@ + + + + Group + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/assets/portworx/portworx-5.1.1.tgz b/assets/portworx/portworx-5.1.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..63429c7a3d5a99970198c427e2a43b1209d5e885 GIT binary patch literal 66769 zcmV)LK)JskiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POwycH20zFpAILdI~Hh`-?rFNZHP{)t&x+j_vOD&doo|Nzd%; zoERb@2_uSN1E6emlC{ojoYy-~aux~~f;U~9+svM5uRR@;AgWL(6sihUg^Y1|#rSpq zXo|TEmw2B1<#Ye`_V)H3+`9+=-P_x%{&#}qG3m_6 z@`~|T3_1ZCO2+YobOLmGbf*6I>97-^Orb3#NhOM@n5id=s1u+lVR?*FEa`$E;Xd`6 zdk+oBPJr+{esHf7pd&V)6CP2VbQGaqH_%zr(tRsf&LaYt+96MxL1|`ckkc*F8{yA?-n`(sfzM6QsN?& z2&XX;8Hwn4i8NP$u#{6G2x1^h1;Rp!A+%&UN1XhTQ%=;!BJ6Zxk|k{UG9w&I#s^69 zoS=r^Ep!YmRnMOMHQ$+Tj*?tRvZ9xkZX*3%r$gs>LaO^hb3djPVq@Lc-_WW0IP7HC zb~LA@-VW}m?pF3-hl8l@s>3ppiYXn-Q@!~N&=@D8J~TC~G0qbykYuLKiaqP4?G}X{ zDlRae5czU=#5j35YzJ{F^bAU9k6sRu{^I}Cfrp(&>NgVdl_0VDFs1^J5`q{=KBq0e>Ta$#{7NsmED940Va(Xl+(M+^c&JIz1fu`^Y zir|~DgiYxcprbK}Lo}>F_bFETtkcP6Vhy`kybODQMyL4V?CB8goY4i7I6+4&m6)c4 zqaoo78j;=3$ ziciw`jL}pMS}3W$y_gboa&dHwNE&CVafpUfmM1Y95kym1;$z0=SR&QsK&ecFe*Y)6 zqyqIMupjK4xmI_Fh-u% z5O$2reE~YL`Ql=c9ARDclI}QMUwY=aUiq?$9`TZn#9GI=sU-F{3KeG%gy%|Ax=A5j-EzA?9T9dt9Jr{5L>^5M76X*!ob(X>Q~&2m^w1UrVd}};Fh(v6j<&p#}O_mx0I#ka`L@0D{Bx$7C7Dl9X@K{fwTyaBB#yDfhEp3cj%kEtXD#nltLzKDcPD_f6aMD-q1*qQ6j0`*bu9W?6_%fx0-CEMr-ofIyTC>DL?2IMqr$Nf(r}6ig*5 z(5X^{I%@c*YGh-qh~FZsZy~ieTI+z?_UI=fr+@poGsh{OfEl%_t2Lbnt=VaMw(c3X zJSP`2pCBc$Ve-ubB?FXYMPfA4=9M*rLUc?yi6nZh?Zbr5$i{TM^Niv62~HF@@jE?q zM{%`clskG?G%5g25{C$=?&i6|Lt!dFNN6DvZ4BJ@*fTnjVIaF_#A?;vu}+b|v>x~v z|LET0j1G)_dU|%J)8Tn4>72aGjB?P1!y$@Tl90&iQ8Tg(G&UKgTrE#55nB*`#i?Dc zrf+M1hlpG&QTUFscXG-Jj>Y5sy&r?-PLYD%MN zv8oLOVYjf6GjEj5=L|1cC*g~B% zgcA#;hmOstdkXXCu%zP<0a}72;~|}-YV3_a!^V=$NcxP9i5gi$Q@w#FB%R2q^DqPx zk{V?Xz87u`I8ocQVj%UWnnq5>oQP>*PEaB8RjQYUs9S*!WfTyl7i)PRv2;u)IR~l# zx4_O+s5Mg9tI!;0+S;8GboTQ6;`f*5e|vZM>f*O||M~mHyVK$D)ya8DzAiAQU=JH> z%Q4}VJ~SKh8{pk=&7c9EK25aA~_`sg+4gIB!i*Ar0dSjCt;_< ziFK5>5)42d8JhThDQAM`LTNWf6Gd|}XM72ukIr7{mmSrIGSP^Ta5{4fA=7Yxx)1I> zr(N?f%JKo~9^BuXcRQUZ7m_)Bpj`|V*y%4&f=49L21s!%q&Ea3W>LmsO&%jgHl-T}I=RO4EFlAgGi8=S6Q`8%9NzcduoGL}wI;>uQOL~ zq2rUY^OK{)i<9F4dL=-sf;9;oAsILzNV;$mU~^8}%nojnGGcS>-k@>fpd|T1&CR6tO9_w<0n*2OhrbWs9UZ=V z^6JI$vlAmZ0M7yH_KA%8+0}KQjs8I*sd$(K5#iE*5N2fFt#x^LbaXNtzI%G|_jjkq zez(7HHn(C68R_|yEYHcfc#2F%N^y@kiG}xOMwSDFVHXc(WZCJ=sfZ5wWDR%SLeHs) zNRr@`uw3!^B!{yJtytMzF~h?6a4i?31>qws={!U))wkuuu z2OV$u{i@~AQPVzK%fH|bhmU9O+&E^D2utVXPO6M>G|RJqr9sBYydSYNBAFC@00#Tb zbeRzzv2;QBgrpH@cTQ^rPrVkiGj|lzhS;JwkpcV)&(}ftFx=~D<%P;S);C%^h|&y` z(P)P9&5cD17e^7}F>D>I=-a&ARNkn44cLAiRM=%berIM(cT9VNd^%SN#0WNf_1l4U3%3zE3@pkYK)GRJzJ7aK6# zp_AmE_)}(Rixe$ZyJiEcCY_TB6_PJsoj)6NxcS4Vk)4D_Bo)N|*_pGHYQ0biu#B@g zkyDbdpbbp$dV9xOJ6@mq%WJi0o-5TfYCep*g-)(BW*kM1zvlW2HC&t`a&25Lj2B$( zO#3qjQnMpY2$$M!;ms&FV5`4E+Jm9J#_N^jl7d`xWk1|ZL)IPnTC#r9pefrIzP41Z9L%2AY5|+e?7e!W3vq!O)3tg60@_CUm<373kpJG%Uj%(mz4{YykS) zxNtEXXux3u6*D9<91%T0Z;~UN>iw00ASWeEh{43ks_y>Yuk=Z`*S){@oIWvXwN~CzJXhS(f>Q~d*IsjnlQ*Bk z2AqHu&#;^hP+t-u9Wtwswa%&|mP&Hn7jVR z(sRb7k*^(uBuR+k`{k;583WX{?I1Mac#KoLkuS|Q?zkA8^f}hiHO-%EJLl}>5!i{H zP9$jAf!7+8Q@}hLo~q%oj3sQc9BjQK9t}_3*Vf;NCRi$>8C`am;!I4LJaWp5*Kw&M z+Y%K`I~_O|poyA|Eys44OLnHdRJhEI&ENcu`S8%f((RQI&!X9>{&p=f1hiISs_4g< zhuRaWgN)F|TI%mg`GqJLb6njk>k%(YI?`s&=Hi$?e44i$I z^m7Fz!yM1WDmca}(+$Dh;QDrKne*hai5^nmV>TK5u=m5>k2sgp$79Aun0GoIdvyBr z_3@Kd`E;v%!rGZF;*mWQZugqp@=tY%!d2R+gF;19{Q-G;+HY7m|9#99J&i~KOC7?k zHgSPfU4jbpP*FhJUoY%j`zH;sRqF_<4nh$pu~JiUVq8Rn`iQ|-!GUpfq>T_)jB7O+ zg1r<()rodE`*B{djSuV>3LfD`xmZQnCs+`aVL4T_;#zo`tO|V`k2yhEf+MBz?-0uq z=D4}kGQz#c#80YsGqQw_X^NcxPLV==qnTPC;}xfKhWFr@v;N60^#%X6HScHOGS-ySr2rc>MDK&dHb8J=M6 zoOr=f@<9Q=U@qwR!xO4K-P)W(=3`E9{4!0J&ar#TsW?@Y=fx^~5u%dJMe$9bmw4s; z!c&Se7+m#zN3z7S(mG2Sd2J0%0cLkbmiBCxxkQNW$sajRKAc=rAw{=x00aNE56O5; zA~`_a7i<_!Nt`F-!x>kr!h7(7om`VBmtG8lh^uOzXW*| zxdbbCTf-ZlLWdPz-dKHP-E27^OXe%9xOW4bMdCNux&yqSifcOs@!b(Jo!Z2^J5dk`9kjDE;zftS*WvZ#r0Y!L@uV1Bj zfoTGgQZ<;<^qk;$IV2HFV__pBA;(tDBukBau(66|ARUBalTfRaXmi}3u&WK+-Ch_k z71}`{*^0DYlY$$fA-pqxs&8#rI2sjsvKuECd#5i{svZQ9(uCKTgu5$JoGwoZx-8T2 z(hNeaEqw^ol7HmPY@?b^u}e2Gg8>BvoAC1vs!Z6*^f0aUBoGufWmo+K2g^^Z-XlU!Of|amnBzU3)$rYx$riv0I8H;U7 zsj}|%V#X3B1PfVX;t?4$ZX)erCLtf|TB4CMGa+Jv#bP@aDLk!${^``kjK!u?Dm*n) zPC3gb+AdR1!?G+0xxa?O%Cb}>&?i~BWV*>w5gXiyTsaY|JkN#Ta}mb+AZ^pgLk(i# z2|jSf^j@YB=^Z9lcq#n)56(pwzj%3ZG7ROl{9H9A*3^GGxOeYwkQWU+3>E=jdFQR#MACX(cgw{~m?rVF;?z2lPkIBtaiPcAQ{5Q_H+KTi|(`s`blK zL~t}U$(WVAnNIQf6@2JisClYaW5!rw7i>($Odu>JWO}-ka8Ftq?X>fPp>fV3RUK;| ziieohR@4TyWH~6gS}^HdK;(1p(EGJZKv*L&8$s6V&Pa!ni|%raA)K zlpWBaY_=?jg+x<5d_^@2)#Sm0vk#X1josk-pCl+$-oCMG1Fx0QRU|p zN&%$PF=zJ7B$H}GKw0O^i}LbW-&hA0>n}iQ512KF0tM4+(*dBc;H=>a`k7kX5iEOi z(21_5G$Cz8b4v9krUb{@ziFs(esXyH{3M*m=vxcvFZgTN|KCkXk`XRKnSGv8V5|54 zgZ=yWtMwo6-{1Sb|9_R=`}h67qF;%zS0lzDt^4ro)YY;YVNqm;Ey@M1tWEyc{>P7< z&inU)GU#YZP?wKz6cz;l(?uVU!wNdr$B#kr#1iD=$8KjzIF;a+_w_XEU-DEnQ!;a~QlD$Z5^7|;%%te* zX6W_;=1nL4heoIxuV!c}YJ5NEh8Trp|ML9u@ng~L253bxf$+== zJo%bTT+lJJ41If!|G~HdO4FEJ7i`>px7T(2c6(0KmR9?=RmN#5$Ef=sBK!~0wH=!; z*U)4_S3wv)^lkT*jDK%#S1k5vvwD`i#a9|$8|s@aMlemk^@`k-@4>G9LUIzOQ`^Ctz*!|E8@k$V zQf7|?Oo{Mz&#$qye>(DT)Y!u@-mj=fuJ_S572aQ|Vr z*F~{jMr9aAP)BTK^>CA!Rp`wHw3HF!~e5 zCJA}x0zSp(6Sw~c6RU)UKVkmJ8+U+qs&BIkG-=K_7v!671D_-XKHmaa&6dscwuN+$ z1g%Q<|CyxzBgad)OyiM=1pPE}SNqzI#jtK~>r&g51tGvjnDPuFC zos<Nw?(9^H}76cTxA9rfHr==)qk>-eEol z@&?^jt=HzhcFjM!D{)M<8}ViYwFe_m4{>8^nmE;SmXPnHY$WA~rnVr^)+BwRzy0^3 z>`KyZ$DQBVvPQeD+40|&4}p}wnkuUZBaYLU%}wH@Eh2*_m5S)~voUuhZbO7;WLXSC z^?I0OQ~V;Iqx}cy14@~$q4NQa9^4~owBFRfCjZOD2fuVquJu0Kf=!!XwAav z0jkFBAl2KAfX>KrW0bdFd*PwCJf^ zYt`?j@}{Ye($&9^(a~S*cV;w=2k1Z9sAF%W8FX|OBIs%m1!dWMr6L+?Jd2`PM3x!9 zD88I{ZCKrhIG*xT2d@=V{qM~zFi-nIZy z9AwpT^(QI#2OD{(m4+~iCiN9w-d;V*3{ZDOWs2vtZ^GyX)#XNffH`=1A_KR3vv`Io|FEahGdE16j)L48YMQ}6ic$={oMwxZ~Y#>77clslm_D5Ju@AS}}AiDF`YX-#^E3iLQU_zdZ4Vph=@#EV&hJYwY(d`#QG$B%F z`~`_Z{IN@g4H)`|U}=Ws^zohd9Byw4p-9i0zrOwWJ`@tm9MpVWg;l)yKRf@w5Uve; z3^j3=-QBmQ)yMZ;)cyEAboY`eP#@oS|2yq|d{5H&;~n$?O*qNWh0@ACpeUCpi0_~~ zC>S504?4U*NcVRuIZmw>Omz8dH zrp}u;==KhYrVK`r>Bdi(ZA6xZvyiG+|0 z-LF0W)})PDN;*Lhl=NGt*_IC~YfA3+O2N%l{Q>=NQw3#EoiX*O!Xj6&iRw zBciT)SF#~mb>Oe0LvN?RN*&;_q}^7-gMQXH_sP@kKoya$ZijHuaw!T{*h22yXfua9rl%y zQSDXR+TFL6*)<1k|9+EfjwL@~o<*+fK}+W!rk5?{-l*)xo&`3 z6Vz7iDo`u@Gjq)6U<+5|wz^r_-M6v{Y}env*`{I2p})-VA35oE>{Pedl2x8=JhA@Q z@Nj?st@^e5|2Z)Wy>HgLiEgl!JT~+F|7Cssm%Dp=_bUGXyZ86+e)s=>jbG)wOWzA) zYfah%ta5$2jBI-ukrim z(eJ*6(pF<^=d1g!xoxSr8CmhARk!boS`AIZ#MyPh$KTboFRG@gf0~;V?cyNl7>GZF zlS1Py!x=@E)vaVPeExVwNQQ#I-KqIwBUH3K)Qusc50F!HQ^;1zKD4|fWus=6U z2c(`?UR~o~bnNT#_*T$U7L#uV{L85{E1_!9`mF}-gzG;6*}t5?X@cLlVNq zAipkef2KLz9xlj_36VjT%OIiRYf`va$6p0rA*jFHYB5(r93@x?yIfM7i7AtTh`!h` zOJECgZ;CmIzdqb}bnx|org@Tli=o~O=FL4;$-2j+g!ucY)9m=vP< zTVYV6#aDB%)NwH2z7;97#JMr_PqO)qo4BSV*dk|N*8^d($*#XO0$+Y(-<%X`zx}r4 z(A@FQG9}*#fkI~9oDerd%TSe zzk7EdKB&Zh?Cl?XkN@~8KXW_`HM+?kO%={uiM3s{1BdGgW3xO%-Nk+vb-f4+udG_} z&s{iiZ%jOKpEhSQe!g@>6QN)ePrgUKdB0Wb->O!hwD{Mr|N93IEA@Z&A3oUo9{>MU zevR?}wpwKC(p{U0T;rin2->nGimI0b?1u;UyJ+V#Rk*GexZd6U__3BWr*~+3*_$E8 z8HJxrH(ItU5!jHdZ@`};i05~6Q^H3_Y4 zO~*p#9yY)Cgl&=7>NyZgfdE=V+rT@dYJPG8KC{*iFe7peXesz&;2eyO2K`-LbnnmA(3Q8^vto0 zrq8@U+EpVh$9DSEQNN_r%WA&x<3d+gy=wYe|6F{ijmOlN=htHy8)l_;L!z3I!Ta~c zRpi}6b4NckKoU=MT@>@0>g!gAWd53Pq^}Y$N?c~;?u)BI8$upSMk7=%s*O}V4+13= zm>{+1aIYtS?B!gS-zNmega)pLZ9WEo4?Wcd^jDF>fVnUFqeO##&P{=XYMMVr{%X=|GU4pzkjdl|KES`-T(hpe*XDiV-cWTeJyb5;Qhk*bGwP?!$W&x02TVO5I?WHdXb2iS?;bPv7e`s@DwLc zUmrgyb>Tm6|D4DuK1t&(i5lPraw@pJp@B z@TvE-{nNungbz1C_+=x)FE>Fr7i3HmGK6&pfkA#34R2R=1_-v%5&`G+WoTW-%IDIs5<9UA)G;EQrjb6<__6T;F2)NN7Y-LF!$U zU#yW?#{_KMio7yUSA6>qmMgJe|GfNS6*X}*CAwK@B1+Gz-+OBV7UgqeRp|EqB58QN zqBnQL*WfYoFC8ncJ_|k6%Qtydzp38y<$r0k!!s5i7B*0wjV#`wXDNjli$i-yrYCZZ zrf`^1Yg269v+-JmUSACmK3n%C&yC~6&(^(CMd~2xbkYZFD}ZPpR1N>^yQHb zUZm+HEE;&om}e{(=R}fJ+WW>=eI9bxw4ITgdLGh0$rims^9BgkU0G5%OKyOOaS|qU z1bT{Y7BW6*KGN=%4)jcSaEojNvx`Oqk$EPU-IhX?-Ouaq^h&+d^elT9+PtZT?-h#cF08Kg8qw zFRaQFPz`;VAv7(@rq>?Ye0S{;EU(S9&W4g1HEzYJAxH_uGA6Ez-6I37W#yNkbB` zG!_LG<>#x=Xz8H^54{^1R&43g3R9Mb5Xp;Dn(Xd6?%MP#3HMZO^92a(U9Xa1(V~dd zfu<`6Qth>cBK}|gV(P4$GMr1M`Rs_zGnSH6YN|KASb=A2KLuvakeOmpvAxt>G+V5@ zXlJ$I={IOF?9}UKo6o;4cZx+}qqQ4wr4Ov}X7I0$%ImWv)%;JPzjFMf*GhkFCb%3` zX?E2S;fHUUk;cY8=jV55&82mbSY``qr&K?PQ+=x2k%R9-uVV{@r(8iGvL3x@ieSupZgDTRlUIGPA&p!5{vSBBY7jBbNhE<5_U6}A{kc>|3{xPLF-ylwWg4lk zdk80jA(Dz*wU%&IEW~xuQ9{s|vpIx@r7VUUVijhnu%Ye!WL;D`XSW7>m5_Ecr9Jhw zdkV}~CS@a+xAJ(=lWQBG_wUvFH9A4Xr;?J|fqS_YcX8ZavJ9J!;+p{ZoLup++2c=Z zQENMAiiJ%^>vwG%Yh*+T%j2Q1b5xZ7jI#xe3Adgg^JfWl`>1E9|u5nHr@QWnO=4*`@J3 z&^tUqsHsH(tMU7lP~EW0C!#AyUTRtlzVx`jsoe2866_Rb<=2KAU$B^nGs6Fir38IY zf^-+{*)N-$m6)=nr<-E34uw^)8-SL@o7Dxp^N4S_IsLxZY|*ic3t|KB)4T7;lVjYYq%EpN<)i$SXmPshA0;Z?8IyZ3$`f54Lha z4wK|}DyKz*dfyEKU`7AhiL0BG6|xf0wL@4bU)x1%U9x^HsrscPD&uPeD$u%%bICT8 zwVTS>>V{oCZ-Ve?QVk(2wgS=|8(2E=N{KS-XM^e-H1+8unEy@>rrdn(47a1@g!E)C zD=?ZqT4$n^`KpKx^?{j6WM`iC;x%m$`Gv;RE-`AI($PGaWt(7on~Gw%s6u^dqd4vKd(oYur2F}CB+a;#w8kn(cxm$URhug+ zr)6Ls5heROo+l_oA5f|Ue=5;InJ$43%pKxZ2jg{ldtt&?Zx?Z^wLPcR5QmTb-gGAv zj}p@EV)8Hc@;>l^pZg_Y&e0O+s^zzk^OJW+&rVNX zT)Z2e9G#zBY>#4g9pFE6PRd9%cijm1^!SSag>K6LFHT+@egS}fw~h4oR(Xm3XMcP5 z^8E1V*~z#x~vpU9}6T|pR)Y6Abo8T>oU$h_M`arcKY8Ol5EPAN?a-iIVrA|Wmx z;g?=%AMvhpQNqKDV&_@w`dj$lN@17wdNaxxnthQ{Gws+^4=ZY$h$X_J@XjuL>!V>) zxwk?mY3SR;sXJ-ymqXheMo`uk${?^8TM40*#g>Cjw0*+|+iYb%1l#ZGQxZ4?;ghDL z5%sk7Ok}+!S@xwHMlGL7G^)2?NmX(Rbjatfe%X-Hq)$!&m^X2iNobRPW#KVbdzdcU zQ(g6)LJXCK(u+nRL?xMfE-sKr6}jQWRR#d-Z=1w8oEzEvMK$u$XatXH<4EYJKWA~C z5M6R=%7lK^@uAn6*=Vw3%As^n+rAE}rk|qd>!=nl)gU5VidHI@TeQfH7fq6=ntDywV+~K_}n5?cd`v;IEIaiTz8|mV0 z_KL}r$i<1-mrQ_>mq9_UGsfj}A~}sj`HlXjFky-mjOOkNFbzo?k|bnKBwsecC=!It zt#>7Ts16ri7mv&FPBm-_C%6Gyi+!pi3$#JQX`sO_+L;ivLsMD8!q)!VMf+YrVo?!LZAuNzTMyjo?w-a(Bj-G6!qh{< zv@hdP>ZyuIlK^PVNGdtIJb3D5J52a=C0q~zWxCnIE*>yj`@~Iii*$H|TA?)pD`n_` zAa9B`YXr32s4Bv(rogQL^`Jf`v)DtOIZ}zrohyurU?FQ#$p7 zL1m6y?X1A_-G*tAC5)V52X9P9P}kXl9M3C?z-|3dO)bpnGb%KLgyw~t-x9Mlzr3Ac z$}3$x_DX)0*UC;pQvJ!UHBG#(X1X*DoFAKx_iBy6iC1m5Uv8$mQoIPBZvBbx;jSqL zt9f$Ge&nrLo~M$|$;-@(b}c`%u8i8st$bQrxf`t8O;+vA*6ijK7!xdWPJYGGE+hX* z>*!0L{i}`L_wO2&t+Z>wm-fv7wd`3qFMK%f%L$3K-c{Rhk`o~hA!HVHjds!9-SmTF zo&LMU7utG^ySs%U<-PZ}(AG}Y>Bve$XzoOiWIUvkl*FNAGm<`|W1`l}`sP%KobdJS z^!=@Vk|+Wl+%_nrXbFM#akruDNga*bMqTKp-nTx!8};CS&^QsrzneTx6;rcfX0~p^ zRS#52A-8=?mJn`I1d4q2nAq~^&_|Zw2*L*%Pp17YoWkrLZKf;KmU;<-kB|l7KU)Li zeycC8h2W}hp>G?_P3=_p3ZAc|dd=-um(#jLXf2Z%q4JBCJml7|-#3Q5eCt|<-+$Zx zT(N=klf&cZC*eH){4iG6|GanbVE=xl{^!BHd-uNA|NI)iTj4KmHFA3K?VoAn% zQXW;t*@WZy97`JEBw4DDL_Q}PfgT#ugaA|q1(^$sbV{B;F&yP3OC+b9<&!BK*So8r z!%pYcEp*0704Mr_N+LQJQ{vV5MmsJoP(9o=gcfegJzm#1*ZAm^u@d5n%dl#bB%bzh0%k!J)P=aS;dM58)n=5|7w|K#)bArfOIQxjJEd+^Co$$qX^v+<1W8 zs11L{T9Hsab2^zybpGV>)wjX`af?w(hDM}VWhR*~St<}7YWv~C|J@1DVUnOX zS&6USuHX(sAb0ItKVqD`TkO9x>()%{yNJh7IUD*q2+?_-YR)#`i@9T*PH2h~ zbK8!-sTeXZudqP5Q0Ap$3-y5`fu@T_x77EW^OI*Mhr^S%=*`h@hvye>75-lL@WtH_ zos$J6S33TuOhb{8C`6}Y$nDgw6{;yot}_xz68CKFejjbc{X9?6rH{^~!e7_*s8P3a zg>p+v0Bz~2E-_e1nM zrh5M>?mLhSMQolag}HNVN6|o7$&erfk%badm#);c`#45$C4dm)JV}<9Fb4&cF{X*= zx!ha5hDFApvcKsRk65m=c2(<~lN(d~K?s+<*?SoZzT0_~iab*!Bk^vh^GXmifd zDd&yfTIp-p$@SAyc8eRBVmq*fQ4AMlc{IqNiWZW5>G4&S2{XdSjL$2K7$Oiir%sp` z@}|?dU=2G%Az^j#F7VHy%lR@e4=#b z{ZbH_X9I=Vty{Oyi4h%i$7kA<;!P!$P*}O}2OimxtJn zAmbziVynkEMdaGNQ80S26-LrSV0>6Hz=L^_gRZRHr{Dg7fiOi5#sN|SmIeC1}Oq=ng8aYFrazBCOkduWtP zlcg{6$%F{0cONIcH4UKq6wehXHPpFcmR}f0WHJEBF`*Jg2}_Bi(S*urJ_6#n{Sy?e z36*;DvWqVWVlv7nCu<%Yf16(;4L)=}pi94);U)TjY>2X*rN8NOU-SXp4G+Toa38^E&G%G4) zAFlLq@^pxH4(Iq!mZFmc3rVATXS?Oj4%<3oDLRDy{>*vF5Nz(Js9{|1=hvqvXy;eP zCP}HIlXODUYColo6MaBC=O_RFSEuJE$GduGv;|GHb@n$!_i>&^TF>H&_mibxNCtJ+ z)sg9j=y{$BnT`kwLxEFrT7lk6{y+&`NuUa4t_=-P`9fG zUfMA~6!O}t;id*_*X@1gyL!U0`h2Lz;CeBtOP)eZG>H+U@PhuWH%JU_9iw7b7_K=Z zxDAY=yh#wo0$q_LQUA?~Ok(3(h^d%WA?fd+=O)st-A+k5)4g^NfC{C!!TO67Q^DOsX3&LoK>`haGSgb_i0#<>Ckg4d{5)Fp`jMC9~uKUaiF zLnir_J$RMg)tB{c7tTDjxO47y?}|+Y!m5g)$*Dp5a1n)X8V*!B5<*vQd7=T@&E2q)Owd@%i%@E(b*XJX5YK0ViOnxjCvNU_% z>Diq!b;_|wYlYs7ECY>DhA9`?MOiq|z)7NeL`-)X7JTMM{%irst0t#UTnZ1e?$FmS zY%mVvPQNX4dMRc{VWGomg!hF|YXI6ItP9&sr~qL=p1<1FVOV8VxGnnS5-dAO_?)Ih zpefUy7ifVvONeF-wY0#hh7?S&sGXf|!^4!4Y(jo|$cFOt6uKr#M~%odhbc~!exqhLF|HO)#+<t^j%op*YSHRY9kA{_8FMuy zsk6Hkh_k;1ZZb9i;s?H1P)}<<9Du48JSUP6Z^KYc?5+g|F3Yb!E~D z0H)PU2^1$hZv@Ab8r=leC&BS9QCbhHTrjmAVEcti9t=~Yx{YihI?6dGsZ5qVb7zA6 zM6GwwCY)E?s`tJ?Z=k=ap;4Bsfgl+!OIA|UKtT&r&caA?yVP}7n$}XBPt&skp8x&i z#q0gY3!Lo#2>(7%|3277JLs5X92D9Z4M4+Dc!^7W<{_JbS1kzr#Rqg4$8FfMiVM!I zIhw*w`_yxxIy&macC|L%yfvyx{aPCk-Nx@=+>zQtuB;^E*>iCH57o9_-W~i30IJ(J zra5});uoyz{kLUCc*N2L;S-WZq&Q~voTlYVsr0$~ZV0igc?qVUt8c~NqHt@$YhRqY ztaY%Hu~1w%$@Ss9WF{=2pmFs}6FhIH%!bxCerYN1Dg&smo*CBkL^Y^Z&6A%gV#h`I zMW?k(M!Xey%%T}lB458O33UI~5cP@e!EP9yA=h~8SWTVqv0D?9tMdpAZTi zUPHW0iQVTWja0JI=?UY7tJ0~k+ky9!N1KNOE#@Zabb}sSwo6grj3VX~_DgTGjItaR z+hiLMom2|8fioSwTkTxdX-UpGB7H#ntGhROn@(rS2&YUtNwiX0f~CeJ(3FBg5#a>Q zv7WFGsC#g4x{{Yl8Ker{b7K-?a5RA!qGE>NA~-!9Z;F+j0CuG{_j2mEx~9?arB_5=JvHU2uU=FnG=o_l3R~<}72~bkOH%)bTfkt3O zP@L1+iNYu2RFJ;jrIi8F2b*Keco8&bw{tK@ZfPh%anup9`8-ceXz6Z6afZ9$Ie{nu zoY5m$^YOVV#o%1NkGjLz*)HVAi+7-I|ppKa-G^5aZf(3yB zDJJ~UhE&k8afd5A2=YK;-}2D$LFJYa?uAit0*~O9d-XX@8>-Y4AuyiX$VF3P(e%kV z852JEtg#-u+Y{8MB#7*1EOMsF(V>naZkgE79xROr>MOGMrM?}L@G!y~0J&=aad-$6^;)n`0cQ>8X(6-|RT z$3QV0!-VJ;@a4l=>y?7YeE*;7X^ykPliuEut}yMH!xD{_ zwr!r{%-jCFytK+mjhDj)O0pIp;PUkSc%j#spMh5HO`k{Egf*Tg)d{g(S>?N)aHcwR zx`SFRn5Nbl9VBEyl5)VH20+u8E@+%<<8#7Up20Z=^xreYQiC$Zi+lDsuh*f@Q{9|YPb7-ZM|-LTX78U6LI7c~3pUkB(k)sZI=HmsrJekqvrunw?9m(3x_mpv4_ z6Mrfpe2e&@|6T%55d+;E`|Jn2KHd)x!u^AAujfbkJ@%vI_J24C_a8h8AMS-e>s`_Ld?_+Aw>Or#7Xw?#`f)d3jXRWHQ= zT)=*j0^{VVp=PJ!CQge!kVoZUaFgDkT^cuJW8+V1<=uMX-RI=f>T0x^F>s-x+oC`& zT$&_dy3nE1+Nmh;oV2a4CWPw}<|n0&>VN-SmNGvjRLFph%OSYp=boRP@sm{<)f)bJ zLyEMtL7FBFix(op%|=*BtxU<4onEPwT{0e%Na{^2L`UvJuGSz)O_-?5qAV}M#evHV zR+~*JJy`v3V7?1*8bgwBU=!fX8=6i6Y{?C+IEmPTC5SJ}hOjNNz zQrectmfKf-qjGC?#>-mMTb-`6a#5taffi^oqs(awlm)^9v0NsQ@2|)bZk;J^5Yov` z4nJQ?p`YQf+CSL$lhmlPZ1XfG99=p;o#lKuFysnf0zMlP5~@WWlls69@+-qmL0?0h z3T-d6?lcE;l*}2#UYo2yKZV<=WC#nvA_{sykt60ZgESQ3rx8Nh^DXoQ3mPHGS*{G= zDPu|>H9N;|{x(QBXI$vcFH?pJ1EG|cIA(boUv`?d(U>m-o~K@R6@+ALt~vC!%)5F6 zaNeRLV;svoO_jn93Z&AS5(qCDpgVUT-rc)*$9*t7Ie&d}K0w|5gNNZ>xEJmZphefU zcxf}HHI0pg8lY~`1k#?-+2O@+1Jvh?$!0m*IZ=cipfw~lY(WK@Ga=zb1o9$^@NFz@ zmcETMo??@z+?<;qx;<=R@7mSTc!Z;bO$2~^W5YclRGGg;2YdUE`g;fcy@zP;!C-H1 zuzyzxi2Wy8=jIJWGk}4xASQ;+&y92YD<~S1#uazb{W_(2Bn4|+ZH$Yt-b38EK z2IE#3c*(;9qc3&UIMgarust|t_f>S%C)Qo><8J4T9&YofXgX#CR0Jf#WR&n$I7Q2oB*aeqyl3@sSAlYW@ z)#-eT0!mxeGPog(RrP=N_wL?3*ne2A|8uZ+cmLt{`afUehco)xicg$TaxF=!<db#6j+j={wbskAn&V9>Ew`;^&7go>@gasXS%_@txsaP<}Vj+KOJ zZl=36wa_#&<)R9@^ez?hX~WBBROs7@vLxp?sbL52lwipd%&Twp%PHgFWi==SNXV#z zkce1D2H+tE@fLTG%g-Np*Jv#EbvK}wc%>k1U3KqCKC0Dq7ZS@{4A6U18Q8_GoN+ki zK4M8ePaSG;kzqUlb;_iOpB}p-Z42)3+6=l^ugqOo zws4$~N|9|tLwC9@4%rk7QUX7O7h3>clv8@wL(WJPnvY75UzeV30$Rb@6s$|m0dG5H z=>?q=A@MvbA#pe%B?OfDt63+SlDVt4u9nc@+3D-M!_s5>l`b{q9_ma_Nz0}8?dmLk zjUOYpEVnZ9mdhRJ zC}E2I=+Y1@{YmaBLH+bXAZ!&eQ|-A2{q4!Uo{`pF;AvA!G}*732j>W@PDknLI~) zY68&}8eO%Q_1(*EuLT$hCn4$<2D=Xa?yebv4zex2(uldLAC?%c1LWGPfMvmJH9#;< zW#W2yVRimOCcWXGoqbFa#+IC8zVex-McP+?pNDA-dl)ON?P1I zNDEkivkA*I+->7f-&j;u($96SuIsv9gwWrfG8S(uCRKk#$eS{G3H5W`7m2;d_>E9p zk0yeRXwHf&U)qpu*-9G`uHs$+EWZi9 zm)(1NdwcWlWs}UKbNYGM;TiPgDz%l9<%`UKDvN{Dl3jUmDgTtbV|c)96bb<8X->N0+DL7fhO~ zN-slOiOVaBW*WDSdsw>b@+RXx^#VoIc!eFEu6$}^%sI)JP#obM0qWB~H&J#!r_%Xb z%kk?WdcmY;<(TNTBIw6>YK8i;aMxlzC6{G~%B2xpRnq9aA95aS4&A4a2TozMg5t|z zmz>8Jgx4!wjERi&`HRW;^Y}-DpW%nw>JFte@qhghrRTQj5)XVPkaZT+l3{fsuZfIuk)j0V5EwoT_z6 z#YSLCef9iEFCgdy3coB_zFB9D7&@UL9DypmnK0xtQ}gIk7ejqni=v)VOUV~qR-#A> zf#M?&5NgY3Kz5I*s$`O=pdN+^d@ce}we*?iLb`)lqNNmw5&&n>36Y6v(XNv>L%&f` z)&*sFLf-DIO_L$Zp!jGP>h3bAKM&Dh)in#XdK5j{2u@Q;CJk1ZHnQOT-Gka&Q@wnE z_SS4?3Ora+a6qM9pB&gvKh&aDUvvZkm0X%o6SJE&I>+>Nc-3IQPuhP<)u%QMB1Aj3 zrUfKm_lnG~o|iMJPu~SYEZy}=!lqWy`l+d3q zU$B&121wIJG-D0geY4<`CVHoT3iKBdi#`Ky94{f8<1n8W=X-BfOWfd){ zb2Fl55PDYgRimb3Ygxw{F#4&VCqGv>u^i_5DOUbrTH{>FAmdvpFyxvFSb3Eq+~B%} z{VpX}IzeJfb008VVcV@RJT_m1rAi^|7Ex{e1n9C9u&%e>OB=P`sP&dVXg0h^n$Meu z9+V^GTiRFznw%*9=byuLR<=boUc2dRqUz5r2eaSUSS|q8Y;ci zl_m4ZqZLJh5nC-+=CS9WtHleO(XM4dZ!Xvm0hgjG<)T5sOQUb?&YA3Zfb2p7A%_aAn6K$aDN8q z+bJ3;t`sQGU_)+dO6%5i@iW6#dwvKBEiU?3gHe`>T^v2*CP{T{D~Xl1BN<|H+A18> zuK_6~lXHPcD)Qni#7W(Udi3sk7dN-t0V!h-)08AIL84(bxAa|PU9Iz&uGm@8)>6#o zQ@E`bE~`j`*p)u_3YNoyw3fOn$LOVHtXenCo$8EIve3Ez<{aN%rQz@YEO4I2S9M@b*Gbl~+(e;2^>D%xO>ve%ANw2_ zPK~M99_!uadj^{6aPc@S2eO;$z44{1+^6(Wn(Dj+CDnVbfQ1LNbd6k*c;~vc>GyiF zB*Kv0prEg_70TnMYV~QGIzX2$tKd@U^IrJ7cP!?J0h=uBM)tm?SV_M{sUERjhPo6> zDsd#6pqX^un*zO}Sp&=>I7WS-o-K_O@@Y^i4YxW^~;e%Ew?Id+N8-viLr$ z8G{oI>+)a5M4_^j>tP0>@K8!k^MQr^6-0Mbfi2zYUw&dd74D|4WLc!ykXXBVENiX_ z8!|)62v=qj2-JFF`zFY&h%9M9(YrvPboAG>V$>w5MFnbN3m8tI)V)rOz)Qr5P`pP| zUGFVnqA?n1wF|+~VcmN}r&{cYuI~OD)V^E7+4<*n1lJw&PHAUJY>Ms~HxyLTkW6pn z?n1P}*M|}0%{3~X`VJJO4dql!(TK<^LQ-AyQEd!VqCW{|Fl}&?nl;4BClmWM%cgo& ze-mk1w(3ua#VWh)P^3hL3$&`%7XQ?`-?fOMQmONQ(m|ICs+)YVN;#D3`J{hY`W8bf zJDgl#iR-TAg1GALbra8N(_QLYU|z%iRqZUC6SXTxwp8ymVyUZYV{eiGrF|xlpmVA{ z;r6V`Q@~24bsDp=z5GU;(f%l5qaq`llm24A52b^`3A^>|;K2_EyOx1(>Ql7db?qe_ zy3#?Xla#r~C)Xs(KZCWmnU$i*G$aXRVO|hEU)k6~#+>8j%63$ebzN;ZjBN&^e{(FO z>B+TP?4mvLxe1l^B6}pbTVyFHk*L1K5w&dZRL&c8~t~A_g&t#|GsbZ{_i{EZvo;v z<8KK(@b;yZ&GdHI=(}?zepmT8+GnOELjt>-octofrF%`_)JT_tDc=evnDZbnA!2~% zDPqPYqNCwJ9_ahfX=AD|TE`$8tmIABt}fKIUa?urbtO8U=<;8*6mthdwUxmn(=-Ut z$vl%wb0P?jR& z2{ef#Z0?!>CAv*o72R4h2H6zTP^@6U>oXs1wGQVOw6P*%d$c6uh>fUInXb zUwwtofpZL>8&2Iw{i4DUwpUD`f2LvBTuuqUqJp4*C_cxC>&R_0n0C%AFkBmay1Bhp zKKZIb@Oc#iZS*x+YOjKuMoV;CNt+!Lj$xKI8(Z=0WS&nMS;|;Z%*ta(IYi+JZ603}TE{k11x*A6d{y?uO{U+|eDGK#5N)dF(zeP2D^!&>q@> z&sHL}6*8&2$V(Xc_SRsrngJ*zaQ zRRK>ex)UAO?(=rV8sAX=ZlVoRl4>{i*bd<4p53N6X36vBka|}=^)sl7hu&46rdN=` zkO<~33oK3^V=VCq3!?MDPA-m)ODVt-&xlaka71FLkIELNSTV$R!-XM^bnuD#9IMp> zm-a7N?k;jtUpb__d&tR7H{qSS zw;MHgg`*@eml0Kw@+(E#OF^(#UpF#=PBMxEcg0S`GR@+;J}*rrgCh57Ty$aRtE<<} z@)3z-f`Sxz)j(}y(-i7{(gxDKv5!Q*sr<%@8ZzicX++!Bl2Len zN~Mj}1!XoGLv0+Z3d^j?^fG)}y|rSjgwV$uAqS4xv@GVbG0c z_OsQ9`R8YYmVz;+|FMCj)MHz^x^rHCn-KF|M)8;AL)@;fJ zIZQr}(WItWC$ed)rB~6@aFX~5Rk_iPq%FIjr{KX+osrna7jI<~>B0O?|3CKLzB`WF z*c1LLILe+wGM6*5v)SCNpPb~-BJInKC38ht-rXcO2D>pc2)luH15IkY-tYb#Q~_x8 zyfuIsQk2d9!=`4Y@meU<>#y|U%C)rfCO_!Z%w7Z;S1=(z!(Gz>6p)ScnaQ?b-zj#^ z)$Joq#8Fp(LC~@z^958wv2vG0u$LYms0X5=ONao+tO}#j`?O)8V)I(c64-S+s6WIE z&8dDajJ;R#QJ&vufgpRTNd+wy?EHae1f4wYy#6ISqg-0{bpQ=;wE!THwhT#ZczAp#-UKzU zKy6xt;97Z{eeqKt3=NDYYuI0v?8@u?MVr91=;Q@J#({dIvMa;Np{e&{vlWMF-_^sR zc`POYb(d^@IGGTrFn3Dmy*edxLnwEuff_m+`l;y$yNqv)46tjgyX^JjxClGoj7Da= z4o%>;s+ZuvCf}#!x6E)dD;!Wr%!b%{t_Uql^M$88S3U(+6Mzcg>gn}V_wp2~fiW7s zY|Xk(Xd~3zc2?&B7(@H!nfd&NDbo!tG%uJl`M0c)bUKX#!E!d%iO@emsmVv(>$Ou7YRjM#jRD=DjQ=ndPcPsDP|)s&hZ2pVob^seD# zjamqk0;aDezR*ZnZS&(n1#Rbe;$}!U5Lg>|XE?dk@VG-ra^G0lZJ1mMvVoeCGY7Rd z{MPyWY$53#GsV3}Tyo)sMQg@3eQb&YqL3JCX(`zJE&X&5Rsgzfc&|@ajt?;lKlUSg z##rFr==|bjI<$VJ<^P<97?EBJ>|@CHD|15CAs|e>Uv8TV5;Fi zu_^hI8lg^#4bszTkCW3IxZK!aZ|7{AO%&?Uvy#wSin-FE@#dZTliV;0pecUM7OYmR zx^u4g?ag+yx%O< zn2{3pp+TdUhjl=L0Eu9`F}LR8l_iTy@iqVQ)0nZRWE6A|)^|`52PhTE-~D)}Av!R8 z*xLhxKLFe+Jkr)`U;z>;rXX%qj0enFY0rsfhB34|&f2i0!M8V7pw}p~#lUytHqA;vc$Qzm#4D_PuL zZm!j`$0$v!4p{ouolG1d7H(NDpKv-vZ~JyoY0$&Ic-*X8SKmQG*mrN1#E%WVV1GRx z99DKo^U7^0V?*{lk--c$^}3KszuCr5Gqxp!GX&q{%#HvvpHQ-DJ1`2FzTI)$G&b7K zpF*j+Xtbb~mqVOPcWl~uhaqX^aw|&(R#pG$=1AnbgT^3-U^)0lrMC|10t&0pE_Et`F$ z%>KxP-=yr-v5n1Y)J5UOHynfZ3e4u?t$t8f{Lt$KGlQzhhiutJM1J*g*Ugu@+Fa#$ z+c+YZuH<)I#pS3cwd)q|pbcYm_cIT9Y#w4^g%2tSIH=;4D;fa=(Rl@h$^k-usP6tv z%VuJgP;BKN$G^e9Ru$Y&`@(M-Ex#32M9TIU|A%nzF%B6uCTyvd4V}~7Xho3?8sR%` zWCs&Rnux0+`mk-sgHCKr?Z~kTn>pArV23f+NH<03)kFw%!Y5)_qiK#7oP@k&eBNet)epF}*fM}$mE*gbsG z_Yj{X?7MJ;N!w}17|jU}&{;fpTS6AO{ z?z>a0-4UUa5di~nmw-5E+74oKRtw%BqS?m`k6Ddya1|S&_UCZ$0*abqHE278hYyU| z5P`8_Xc)T^4$zFE&l7`)mMx@W<7zC(BDQ2f`mhnm;d9lHp^&97M7M%WXX~c=+M^qK zJXl=Jxfxq)$169kig)@wTujC{9v^K(00!pR+NJvLGN`o}R4NlYWiL4kv%D_E-Fw;w z)x+xiD8!ZM;E3y#hRb_QAfp_Z;z;c z*nsnXn1>EJ?_2(q1J74x@yiVG4ruk)8|3X5;aT#3mi)(l^h>(6htIR*pVsoT#Ni%L`e34))&kJfED==U0Dn zo`b!&Js+PqpM$;QI(d#MeU2%`=Q-Frc0kVuGoh2`gSl^K4{g;nj=Iq!CXbxayjwO~hB2tAMJDEpn&}1WwxgXkw57X`dE~J^ zJYwl<+N}C<%~*Xf@vhbXD9H^_=PESMJ|{IiCpA1LH9SQ3Tay}UDY%xVv4g~5_04Sq zmtBFgUTZfd3hOn~E#F90cFI{Wa5H~9!MHspkz9S$?vl7osr&gKKl|)87Aj)Rg4L`IDop&huQ%0@j$J4c!CB2i zJ*4?nv<^YV^4n3bI3bfnRd*wRYv5S91Hga>{Msax*h+PMng`m)ClvIZ$s;tT6GJFxpxBK3l`_gsw)TuZ?7ubT?= z9XLjllFhhaCB{mA0!LD@qHDQc8X16n*x@V~(!jNS&DYHUo!t-hw}wqJTSX=E9!@LGS{mptwk2X=kc@;hF!IlE8=twuv( zR+s;5OB)W}U%A}fF_vk@Tri~|Z*oq#AhlVu#x-pV_?UCk(3T19Uf8AI5LYM72_dhYq%30Y7@UY%Y2^ZI{Z|MTqp_2rwGQ#)?^E@*?oZG^)#G?gvb z;8oybW%;sfWrOo>J0YxghjSm@?el=mhDijrgxjFIc}oSRO^@-{!Sc7AfIOLyrb(he79#UQ18byAT*~{wIoIe zyrY%b3@yiGM=AYXu#iem2b>5J!*(-_SYB(cm)AzFO;|hs4k?+Cu1Jrzo(l6VhQp3e zvr|1>NWheVl{;3;m7DvO>vyD8tci=V}E{_@5ay0j8WaS45fro)RF#)JZ~;i`mL1uD_$`yJ(jT5-+QDKu7ZVJK;3B5 zIu?yKy+Yi|S~TL7?KHj<;RH=v*~8ga7P zcZT*>sLe=+ols0q$!oeKv{KRved7$~N|x6{eP&9GE@{LNF36443$pTqQJX@!q1Dp> zaHOx2#zkV|n%*)f3tX_0iDG1bV+&KpN2CBHyph6?eF80nMaQUK;T?6Sw3CtP$oz#A z6X4*TVlATXw4F^S{%d!`DrU{|ceLUq#T?=>Du4uHsy$h0NtjR_fW#QBUqfYhfb_A$ z{OxNH>|>I3#lkZ7fq$JB&D>KRYVxxP=k~ez;sYQIn&4!oLfvL-lnLt!Q4Q* zdwO&0MsL2gxqS!tf#KZXwt8xFi!^@Rb5Y8BmB-k>$Cn+qDGW=~T+CI--07(4I@H`Z zad+oR0i;5N)LG?yV;a2`7omLe2=Cp-T_cRL2fxVUx$C@E^i^xx)SOn^Yj}sxU_ZZp zeSJ1;zz}(lB5A%Sw5+*O<4yzf+_Rg7l(%Fz_RK`_xq4}-aAIb0645pn6&Hbd20?gc z{Xh#p6GX-mv>uhY&0}#m*B9drtt~F*LHW=6w#7kGjBFi@mfPajeIRng0&HA8l*kWz zVLH7dW4@q&*jLRNb8Kj%J`yqKRgGn{PU5j+V|6%!dfZTR8PiST+T0{k?j=!@hn6Zy zvSdjPe%nR-Xb@;P7J4_PT@SJ3`bbKT-4=vUQsJX0iEN2Wz%S z_vOxK1B@RH;|E;;Wb$ZRR!hykZ@Fd{qJ%>{Hu4b)*%_d3IwOe0$82|?_yr5TrD0mJ zR=)3evEaOAHS~ zr`BNFNMs9c2#6Q7a{S0odB?_bMUH;=`^AyD&-_5I`A;4eUNMaUzhL;TsV)}wz|Fgg z?y@zp4^7KJ$|OJ7@V;98tbVS4Ui@7CeE;+EgTEKS-*xb}3O`pTe>vG6p%}s3 z25TQlD6iKdYfG+fiE3!UJRJQ^C5zi#SwC)Jj{ozspK(6F{oC{(Cx5Y9cKp|$U;q5| z&+mS|`1$ID|K;Q_U+yxgTc!AI%^ceZ?3Uw#Yx%mn z83*?phSr@kBe_w{$}fn**KKNmXPMXmkWV?vLDnBG2qm))9NI0b7)1zOuH4eujtW^b z(h0$gkWSm>l$@_mEvaV+Jwycp!fXJ8csNNeEGintCaCFQV%1pP=;b+2alag0*mvH; zc&;%^1C8ooF^OwUHJf$S^|ty9+a9Ay^cEOsk_*yQhCRp`tPL*3tBUga9jlv)YBn0t zy8gt+xZ0!l(gM@H{0*#I$=lt#5N|)JS;@PR$Of{u6ZlpiC8fwccz$gti(?r0v!tal zj|~~Vp~Wp81B@7zhT>NuTwc7e7ktgAh>Oe$A5`#d@M+18j+Ad4$37*m%{6|*Kv6Mk zpOdU^o@3kaiD8bI7?>OlzCsO=#6Z$46pH`EAToZOC?XRq(o-xFlPp%r(@(+Kg0$4M z3hw<%7JVXC@>aUE1bd}%MI2|c{J{cF8mX{y!=RNKi|Sf`z?@td77u;dq%udE?}BV0 zWY$vH-z4{BPqd%NJsv?e$mBpx;D+gYW}~66-Abn)a&J=sQ>&4fLDuMMvq5!`ISUp} zT2+C)Jh?ZxVjEemF%ZG4kc0FQELGO5xFVgfDIaUcd-Lh;6l*8*j+Z&-`pKpj?U`T{ z>BaF$l}TcRd+DL%2w%R+92JVzb3=i*e^li2>F+*2g~MFQd)B%#2I>^Uhf;5rd#)F@ z-Ov9!)FSgCm0-^>GCdgfnP4q1R;HfRl?%C?dzmNW=JFdh@e<7(hc;RJ>QExi2AM5U z-g)tT;75bR;g&&oh1oPwPG}_sGiuRz8#5(WvJ7JQ zlKX-!YMEh$KC6y<*NxFk1(Y2W*7;?$a_BC(Crqa_>wgR(Kp$w`R7Mw-l2lU6Pp0JW zf}9D$K5zvU7jdr=wN3(VW2$k)F89(5+fbnAzh<5=_umSsf9sB4)}&l>S`vCEc}Y5D zMo$IlM9V5zK1&xjOHy;C>1~9YX7#zYEWF<)pe}(%C03G7XkMX5&G&ot{#DKiMupva z^#P-RG%cGTXM$GEuZ&QmgS##LO!>Vy^@npoF<8--yM*IRg=HTto3WOO47Ox_jb`1< zwc-B(#HOa?jcjX32mw}Tl|(TbK)+#5jCdQ$DY@w&2)L$8yGXOH!nC)ViViw#Ss{he zZC4~;&;*hyZ(4cFMDH5EgT)2ff%So!wAxTE=8)rl$8IV$Y7s&?z);=R;#I&( zS5daVTH-iXA_Sp|im4%+f)!kR)DR8m%80y5Goj!2l36UXeTP8~y!8Pv1Tt)38#;l< zX1F1dEyM25nO2Ur-TiKuxVQDew0ulV^j0d=KsQd^ZnToPlb{dBEk`;kBrOczdo9Zp zAjzTn6g1%ulq(dbz$$`2NaiT+sii~eu{W~i^b2~IuXi+l;hlEz8yw~Yx|@7Hu4=m{ z6Y?|BY?uy4YWU)HbfFyk&R}Qz7VO33>RMZ^2aj{QZOJ{18z%S#`Rzc2w5p7tjanEH z5cm=~R`!Fb-NNbZA7OuoKaUBT+x+}?O^!X@csfsP@$c-8ynbhNt9B6ZVgV2;*1}S& zu4!bOF3-3}FTzkIh`K_6kW-Iq<`q}L?z8)Gk}@@P5dbfQ(TKxwWOs$*w>2iA)S_78 z=9~w_8dxT>d+I^pPJ^S=&&%Du_NvKH^xf09{+7)!R$%baTA$fH`LTvG0`X+f;ne71 z;051j^}re2$M@Ni`Mzl=yyU%W`_TBx=y8>-(79>GBOc4}$%9#@4Ub5lU44dF5?7bd zgmYNT;U7+7azHjZc5nGV9QeXWO&N|%3a9I4fu?aC$!A;hqFmlr@sPb zxwMkAUhuY@G_=)AxEs}JIFrPtJ+4&Jdzo0(w_HRZi-gaBGGJl(5Gue!6v7TCPu|hP zW6VAcN}jCFep5jACo_9TW-0TZ0KFu=VAds3u1v(AAGmUA+Dxo;s%6cKWbrhj;$raH z775ibel-YovVx;g+OlsKYigIIkxf_G1al|vfSt>X;FR#05uv1*fTwO>L5eGMlsWb__h5Z- zQ7wt;ih?n3Cf5NNe1Byn7hxDa5ZcKUQ4+E-l%`eHn~^O6Ns#%=rJ(UAW&H_Mh-qjz zw4MQsP%@L7b)rC&Ouh$M2C(@9br9@j3a!n2hUpebW{~xVwyY;kurM!CiZI$f?w8^E z7CiAQ8526P`MIIR?LBQv1t^qMb3}SNMR_aIISvTgC{ljp|Q@CMBO)5k>h>n*&U2jmW+rza`%Agdc%L2j!~SCqFs`!ZLgA+z8fJ2gD&o>di~GCsg&?75CySO*jK(@ZO8*6F#yMh3c}h}wkYVVPAOgC z>Yuw_xUL>!s?WBaXWP!;dA995+jbsp_Q2-$ zTmmoI4$bAkV(iHp$Q!9`m=)+cH2X3bYV8#G@jYEm1=IJkz2#!=ymW6}?_akx3X33o ztj(jx2Q3}fv6Ir-GeBGxdEdvMK`TefMlYG~i-I*e!{z5!d`hkvDzC$3Af+riUl|~= z^%*3^YsxFt-|z1>%NdDH@4mJ7ajzvHv#!1RE4H{s6hT^jhIZ1hgoaPi3NtxkOUdk* z)|LPbsMJx&J7&}LE@r->&=4C7p{|nHnc!U@k-~y8CC}T!92rU(w4Q96$gLSovP64))&=|(7z``uXHkP+Q%K8} z<#fYw%5B@i>v`a}ki0Bt+f%G-93ELbGb)1#iG9LKs503Jv%XEdB&ic|7NYV*Gf$g>9;J(oxn<+TL_H zR=drz9qjUfhi!)pX4-V4jeEFq(ak>Dothn_n$3AZYSzw~Q9_DE4v}WvP5A6Z-}%O@ z{mgZ@EPWRyo^+nJp%28UZvfWe@m2S1YFL*9aSA8|*0eDk>;D>Y1P_|me0{c1o!|47af0&FIxni-ndjHB7klFz%ieN@mai&HKxzVKqtEMjO3e6V)UW^5y_v%7zM`6no8D(~*mxE2Ow%t%g9a zx!^x#1Owq{J=7Z*pEUS}0B;7w8C#u%E*++1C-&&kDa9LZ%{kXD9Jr8m-3hLjF9Gs~ z-*j5G>Sf99SoKoz`GmH`f@@aju4ON2!zZv9P~byN>oO&tEP;E)h3^k3#>MPY+l39{ z^Be5~q269xzsn#O8zF+#A21HOH!1ZbW44cr875Dplch&4^_8FSn&w7wSKSRbkex4- zXgMXXfX^g%WVj!~L{7=2Ag^f6s#jEP2J$*2!9X6##BA_kNhThFa99A!;_be6jH+wIe2;Y+o>S_fLlx8JmUrl0d4?q^D`4NypN+HBEoXO((DsJwmO=plq#2MP z@=R-(c(2j@2BB!CFIRbk`Rr0FY}c~nvt_R3)aT8n0^$cG7^g)gYQI&2#poka_iMBS z*$d~EEmh!2@^WAUdp(eOb!SKjhm1`7KS zyS5L2C$!QF+07RScd}plNKI{$H=$1EfC^L~ZeWTmW}^2ZGi}V>)$&|~KQJB*G^DA| zKWtiF)0S6D5R^q}17L6!G6UCn$?8UGQ0Cz{4JulKssYTz30B!>*9ljtG|r*W@r<)p z{RZeKh1??W_YHbdHwph@X7MitOXfpnO`c1|{(U^$-=~cEo8%_UG@Zq|yHYWJ9}nO6$zc1Y;`)Z*T6w~4 z45sf8Q6a0JQ9R#GxV%|C?Z)zr!sN~BcL4JdiN~8c!F80_5iH)Ue%3I99~6f-6@wSB zbhAnE;P|^4VD`z_EDwvz@^F|e4~fT;g2j@7u9D$mOG8yjL{!N^3CWm4S^{MvF3O|K zAvM$}2}dCznJzh?Vc%ee45Zq}L#cg4gxXYe+C94fuRu`0dqiZ~M?uy1sjYEvh>T(@rHV!Q|kM{-;pD9SOc?5#}JRVBy z;~~U88aiwuGHjrKX4-GYQDGl|M7ATst|Pndae0lQ!R}wnC>{ycLqHwC2Jp}*uzMrG zK3f;IKhM^M?a#Ay;UVXch@bkf*r^G)soBNM;xSXxmBA>y)FA?DLhB<~sTClT57QBh4F711;i%`_Ja8lPw+(;ksD-)<6zVmd@4 z-9w^&A_%7G@?|P==_8_+CLxxRn|!R@XtdG+;%IhvaY&^@w9)L&4uVpeoTL7b=%fch zCd~yh$w4HIMI+6sgB5{9>QF_K+a5rQ?M9=I_*N{Lz9qyC(LK=c+26|_F&_F(NXSxoNkJy=t z*6EiiT@>Dd)R_t&w4dHMg1H%BXeOPwF}%$+{LG}*4}-OttDa_HY{uhj<}Eh?CuT0T zW;UoGkHO_Vrj19VkWnJSRBpl4amdKjKvAJ{WkU0taV`%cdOoxUvwwm|9#Um8 zO634;av#6#6G5m200^?^vBXswI$BCjFuC_pP!*79Ml9YbVn=O zJfcSI8|u@ODg(5D1>KBh_$x)6pcod>qs%=5T2`WSBiFH!<jLK13DvW=C1T-A!Ua~Fyh@DbNX%0=&yftU*e zw9>ixi>n?m2$BUOTXqZiV<$B|wN^kFG6v*fg6-ORMW3e<7Gw~lqJo%`S$^9t?||vk zO?}Wx3g&tOLqAM0)G2zB7#0 zh25n{iqT4=o7VH?^((u)0{$?N;_VHP8ZmA)`%aXst(IU72V#C0qLl*aFh_{)Uv{^( zQ(RuTZ_I3Zku{O`ppJ4m3)f;@mEj80YNFM2#eZU8*+TZ!k9C-rPLAHfPp>s?H7k$e z);~$G9<1iZ4B>uosH7y+(6RX}gx<(bgL(>Ph~_S~+DeX}vZ6{~wep6&<7u(9gQXa> z;*!`d6yZ~}(RtwSq-CMzy;h%+nidN#?7gTo6}Z&a zVGLmmbAFd-tU>V=1`K$-j>eDF+OntCM&wXcLAF|v7lVuBg++8{)LNYS+byYs?Kh#j z;~tz;Vr`y#b^|Zv(&O%H_1&^bJf?=O?;x7v&RO<%cQH-uBV&ouUDK1)tTfZVgZxShF-Bb)CiU<8zp&KGq99O?t+MmT*7J6V@*%G7YRagV- zQ2Bm7yYJv;hC@4~qJ9|hBkN??PM!mjy!JqZy?rBy+QmSFPB)zUa(HB6)aOB zR)~AsDQC245I|o`i}>Mw;ShuXyA2Ac7-F^R^bcS)tfk;unw6e5zJCW1{lkbabyT$* zEX8{V?j8ge3R?Goc~DoOk)<#2vU5Mp%)!rNVdBNI50!UOCym^B<;UuC^j=*U>4z6z zR=jzb5Bg-NqXBZOlKRTGL{I+fUsQKRQxmAtCJ&g+v>= z0vt-JQEkyurdhvSznpi*-7AJuzcEt>wmjyNOHs6E>RqHXG~5z=&*>|(J{BQ_iHt79 z3A?eB9l6J9QUnrBrsN%Oz91JubuH^7i~G)ETDGIKpyquz1?;cA&BUBwgHv+xftm{? z>i&GeKIq@SI3+JW%#`_`&@=U7N-k@k4g&SgTBNQUW?zVCna@Zk0#xh=Afkv-y!-b2 z+b_sM-jhl_L#-Env-s~hPE1Rjdc3;*ZNEsU<9f>g{wnObJHL`m)> z3Omi2f4I>9^>@)k73-Pf9YfNT}i&t3F86qY0sMaq&Odn7> zUz5)dy+^TlpF^Hv-ZBa*E40UHhPa;V1vM7~sk=(^ri$4hIADfsmAO%Uuoo=~;Hc!B zfp^KYYO_`%=^Bp0#wK4wGJ0w61%kQh2P&i!NMGz zPi6~RkTcJ$vge4P8#q@R`}Hr zS0HLpkeY0M7CEzA6Z$YT^4F-W&?Ghn+%a$d?oBs+Wa;oRi8|U3ruy_-6Z2y?k3%t- zf;EfT=z$yt#j$4b)%DQs|B5vW^3}E1jCytvWsQPqDfAW`DN^wni-?g)VJOxw5@kgl z3Nh|YLpg8VzLM9$r<4@3s#pJ*3T&Hp~sQPdXwE?Ax7l0NR=JheW=UsQOP{#W{r`v$a)g9 ze|NYe*MraF>MT0*ir#0Sh|YiPV3QX|s^ED_WqpW*@bDMN`ae}|1Fzs zT7CPm&&Z7#H_EfL(+egvw`j&zNZPm^2tD%D);~42IRzScxC5c>%+*Rrwol7DqmZ!s zm$sh6~p&SR>Ve%2=;saWKKB6dmfF)0pe)pHY`n#i% zT;K*!5-pzEI>^=r1*vefr8U#6wW-fqns0^qm6qh&hKXxUi`yu_;15Z2*{ zkC#Rzm@UH?BQEUi`*hCzlaJ0o%;(p1v@GOmGA_QgBK!D{!Gvv-uv}&L(b+-fR#aGM z3J0-wP+llT@bZwY*5;ENu01=9Xj(B%8rIg_f-Oryik3kTIfyYq+JpC(lZZ=$5Lswi zxnftg`tfIE^*6Rmp2Aw?*f$hd z4x5iE*YrQGzkSoAFAU!Y4c{;GG&HNl4EYbg|NWGl^A>Q1ImGk>N(%-p>V*Zh(1gxr z=oF8t`-ItrC~nwfkDAXHuDGZ<0TnMa^k&&R5>3qIfOV)~JYGbGB{(G&zhz{mptNX@ zmWXKw5iJ)~m=WTDdfHV?KY^IZXgm1eY{FYB67(ryv{;Z^wwxf1Hgtcfn`VSnI(Yu*jevdyiOP9E%^`_jAAC#lq!?+25w@tlzvr zSh8Zev)hSfz&lYfr3jC%`B2mL)@beaW|)#IvmhS38Nt2pYz2I}gyCDcI(qC!)9Dd1 zE!h&KX>0k5`8-C}NpLg`j@WD3060D-$N_@8Q6pd^mYvchz>sv5ha?>OZ3y&sJ0AjSspu;G^0~8Q<DzSj*DbjJ#y-q zU)t!V!R1wh`sA3_l)Sc=1|C;>qh!@->?8$q;q~zD>E&Pt_zjnd{~6pe3C#mFB8DPd z7O_{8o%SP?i;hOdpTN2#gk?lqPg24%VvSx;-Qe|L#;zoq@rrxuinfs#g3dnmiY#~Lh&@4k>cd|l}{sg!HSncvvx)cb`si>ShHX? zYainMHQl)|t^zO&)`5{qrM#0fg#2qi4JcvjmC=_!=8mg=v8vd5P^%bWh znWi1YG*T=##OzjzuN36ON^X%Z#Z{-GBDF-|27!=%&=8;(yZ-tD?{TAVnLSuBz-#g`mRI2uW5bTE@c zsS%~T7x%O+&#p!@qlY4G|6VBkyN`+a9C1l)UR#6Ual}6CM&=D0#)dd(l6w`s@&q5?gRdIv$F5ZO=6EFx*E8N48g(0Y-lhA z?xuhGUKRTLDk1#~6SH?O$oBWY5 zQOGha;>3)%RT{+1-f$TPH`H$IX<-5wv<731C!MSk1&G@C)t;hZFA#bhNV^;!4m(jY z>?;XYyyjYYTzICmBFFgQv}n3`4rSk|SuNY;sc+0QGHBiY;v}A5x4vLOwR}YR)30rO zIcWC`x370LgYmvEnmzO2w+X>xBnTsb7MQprceITtHXJ^g0%ytZxRMFvM$&8d;gegz zZ}@id1h<0w2QdD4i`X-mI4}#&-k-n~&EfMna6k7v`O<^<9HamtK5pUmIg$MZ>lv!o zc^=^ymo{}CD_wnQ0q8kQx^f^yV(06YQ)6_Jp8`;u!NlI&SUB-mY4v?_| zXso>fpkb6HZx5NHU$c^Tbq2&1yce$5J(O0kg&SP775*%ubytXu7PqLg`Xz_zDs(ffp1SSFa8#GVYx}y@C*Oi@E#6|qy&+>BQ4)733VZ){yK8e8t=Td z&fd?(vaq0UJfScX4hhK%asemiKhEao+~7BCh7oL|Q3bL@Ly66kELp`gq%<;LOACYR zYI$eq`R{^!r!a0|q-j3;g3yxz2!bv3FBmH^ZL(s%leQD;l-!)SxaP6u<%|fUbMAjhVzKw;xMUUsy?dj}^n((tS6OKhCa7kYBfAVdPijGlA30rS_+M z;EvPQ{EMz1GwFM}1ju0c#hgqb3MeFdTx4azZOZtXJ<}#Rcm4 zb3cgNeCJrWutE(=nHeXZAi=QuhLym*?dZV)Z#GkVhCGWV*9O- z`nUeTxCmvCi6o(Sl9!}|(tL)k2=EF;EsCXtn({7C5I?VUFf>@y zAUrJ%GQevq0e|Gp+ha zZ@g^{W402QHI~6u4BHj&PPX=-%}Nq(x{~(BG?GH>qU3m)vAp$bS*E$BS1M2Ujr3V< zn0=|h;)p*;XupQGLVP0R-wS$|?`G)b_K7b!2?LuzcazUkJZ}l1$L`5=2&aH0xW{nH z>trwG3gn+}!(K`@P|64GMc2o5595XnenEaa5a6s@;B6G=gzJ!p%JHD&*2|VN=c4+aL)}hOr{4-gzX6X z%8bKx&yXwp6^h?kAFqR#Ft_B{)mYuX^#@j93YF2RSlVb@z2I#*X=tmLXgxdivq|nC z%%4wuIXkE-wpwWT4Cn-w*tueD6o3!jf!$7Chn_kfYZX$G_uVsCg|J?ovyFXXtB@1? z1i)E*v~dZjxvZiDi<}rJ{0PIJm&r`z3 ztoh+XiI8Gq(P+HG(CRBLkpXQ&RHVqbR(yq#hn<{VUD~VVSV)1)8GoB1!lM0jqvD8- zd3no$pQK3;0eYho4!J1JO$HY}n4?rz6fEhmTJy2vn!~HhE?7-t%W*?wWE1_9%rGT0TQBRc{AM zXxWn-SY?+e6&Ss__fT12pOq6|7Hgjeu9leEn+JT6<=G>Imy1*=b1*8>8g+B>Eodd_7*j@Jtd zpE(n(r5YUd7Ra2>2wfZY-~jEpG$E;Au!P(2totA!4ciWwiw%{ydHDz?XZ>LQ517`iLiJ=SbNo z8uL9sOk^=tzw%RZ&0KlOa5072r?r?TbBdK#)QaW&-&iB)V8P1sYt=$Xiy(JJup8U3_S$P+W?yD6i7|mX7p#*Co`h z9!B|OdDZ>(V~M4ZeBqjPsSXeZ*OsXk z2~0+yx>G~Au>t~u4jAm=DdRC@a~OT%JF79y1DqchFMEzA-nVuhXO&UGGcKm^ zYUs!9qcM41ciJ4=sNlx%$p{o8tCa9I+@$~xoJf&=j}Z(FHzeRf#|P~vSr_ohJtX+# z2<7;!i{`+kz?a|;OjgTixD%{jt45a74M!%wb%;uorOgJ=19D5NjWv8O%>vxIlp=pS z&^yX2$k63N@iKQ=XmGrhdlKh>)~@iU>&k1?0jq0s0{@se)yzg`^t>ps@lTR-Wxa-YX=z}919Eo_ zSU`Gu!z%J)$J!;4cg*MJC#?X~w*mnhGtzDRREn&et4)F0bMc{R8Nw*3$p zOO?#I{BnnuDomsr*sjt7p71$)bDpj;9YQxhvvjj*avAp+EVFM5*pE-C@-Y$QmMzm= zro<*a5<_0pH;?(kgKGO3O~+UA89w`&4w9;(htIYO>A06rayTU+bogLBIJ<=ajb)Zs zegu7)_`N>BGI_EYfEV4eWn>|!^#{R}rBDmrU`c(&fWKI3BJXL%bEuI7Ge;Bqr8p&T zq&EM%fE@BnIycX`Wa^F7(9TEMR9v0|Y^jamvgTR9jR0`N%yIQ&^Vvt9Xt*Mmg2>kH z!X8J<4jPB+fO&0bQ?<_U=a z!>4BL9Kgpu`#~R`tUtRfS-b)5`@HW08ruM7##MXk9BDm zCxQ}KO&h}r|F03W;L!fR43h$H)s&nO#l^g0VO!4a8FZW6ycwZUOn>b79jzE*vrs`8 z>XZOnxgx`-bFTY^R45gC*TjokwtR8A$}5vwesL*Y^rbike9_Ch5Lx>IzDulTL@YwY zs#=k!COoE2d#H3S>j!^TQ4EO5kGNQEX+!1pgto7P*81QmizT$?@=s`Z&)@(6NKAu( z$YMU&Uu>~V_iQKv3*~nlH0J=DIvQ52fB4qDcn<*}az?&NtDqjV%xj|oS(r~DHQ}PD zx)P%hpc{-uc$}r*>oD|9xKrTrk@k)MkhYg^ok8)jdK zXr0eUCjtx(tS-Nh_oS5MoiJ( zq=>2$h8+5me_y>KnznPMO?yOi^$m*x8-Lgmw%HZ%k9-OI!7TT$TmROz3o${Es5+Bv z;li+8-F>WNY^dkGl?6JXRaDMVT4j9PwC}%vlV0ifsDF1GE!@ktD(~ZBkudb}*fe(% zGtUIGL*jLrqAcIv^D<-U;I-kBIlKe2WW=l);UyE=u$eYS`#Od;PsjyFs9Wd(s^`@= zT=Zzm{iem)N6mS5hAhQ20%`&W+3MM;MEu#qZDQsN4a8a=jcm&gg4FB3t z*nsp^UbxMU&Y^K$>K^ByeCZ_*4^zuNu)>0UHB~p~qi~Ta!UKYECzE=QZNH*Yx^HZ4 zn^~kmEWmz`;kQW?lqnw_SUh@g0cod*6xJ^ZhETGGiAW4gDacnn{o_I%e6MWr*r+l< z*_Oz?>if9zwu8Gdhyq^7s#Ju1&@GKZKzTnq#G_1dy>@72{7_8w)2N(^`Keur*F^s3i8Z}fvSU)y%3Q!aHH`<5Iyu1Q#XBIPXJNKvpN{L)1 zB|TzDPj7pI%?;bek>Nc(N>)PwX zBmMQO3t~Tw(lwo?x!JBgX4oEUT=vy>_N*{EFIey7E4jKoWKG%q4dtuao2wGm7XplE zd?`C}Z;P^69r7J-z91JubuH^FMXq!fV}nt#`Xsr)8b0a^r!ht}pJ;YpHL6GSaA@j@ z=rYREziP*i)9fC{(%;ZlgCd+x*#1?=l7kP#VJQ1@=5jqwZD|Q#MAiK$#<%3rUd_!J zQtwy379V+8vb&e6q@SO{K;jV@hBK#%y!iaZlw9+gSG27VkkjLoy5GflLHwW~1n09C zS9X1 zv&y6uEs&F%RWj!Se(^DuKZz|Iz~U2QnTffTR%|!x{~oN@M{6fF?x@nCW!*Kb)f z8d9|)J>p9%UJ9r<3; zu`BcSps&%U0Sm$i&^^4uAw|}bw~&sKevJK_9{dlTxBcgMl}5WSg!FbhB3mp7jazm! zogPgg68~s=ez+KgXw#>BCATzA^j6pr+ zO&7Z@AFCW@qrJpm`k}pidvSLD`eIs_hh(Kpl#S#f4r5`sX#Q14lkS6{Zmq>ejFB+? zT5H+JO3s)2F4J$Ykl2Jy%=#Z*!PJW!-BK^ya`iLzT9jTS#*06>WqBY0dv&ei{CYPG z4=&W)){CGDZNu7{V;;#s@{UpWw#w!YzX^vI%l#pa5#gy`9EwpJE_oGkgoZTY-Wsy$ zrezaf330|N2GvrW+%eLD^*0PGz0NL-ZB)yZQ092hFb5ZiNd8r|>oNu6G1|g z$pp-g$Nw_ANm-qc@0lHiQQZt5Dm1xYNU#=l%FZ)(zkwlO#bn#YOVf^#Ln%Y{~Qgjf)Gr{D0eo1(?~x$oj>CHfNn)oO1;*(CI?jS_?tW9)tVKdfEs# zn-Qgl_HDiTkFzU6JF~``N8{e61+=*tTw@k-_6ped4={h%(qqj2VUnT-{@y6h64e?0 z-pJL+^7qF0c^@s!YR_8}ILNQT$IAuRoNC!-dB_HDd$U@nw5;*;f;pW>qgT8E!|~;8 zz2UyH4``vVKbOL2gs}xPhM6a)M6i21M()}nf3l`*s4-Xl`HMd#WKIiOvp@HR#z8kN zyx|=}1eQ&(YtW%NE zr&QRP>y`>-_I>C@JhEX0N90%+O_cxq!Luo=PK06`j{hBQ<2OE=GG?d`>~@D!lx`KZ z@A>@==XV9)k#D$ykiNL1i46ic3MRW5N%xA#mhk4Ev@F{%P~MnXX|s*W*J3Y-|P4cdxFXHAVjThtELv{jb0K!)H-M^c0fb_OL`A+S4E9>p!@tm!G*Y$j~rz z2FdgqQx(jeV3`gObtI!I`&%(x#(mB)_*Lc@xP)zFo5^Gxz6`)E>#P_v0P*j*gPg>L z0}dP2`eC{70~vn!kK>b>m}@@bSK$^N%9s>$_=kr+42-kMwh;e;sj260uK)SX*_RjJ z!0KZx-KgzcFRb>I6a5{}KgP~@Gzo}Qhq+jBl?(FbtLuT6e9aF&W`kSxlJBbqi`nS< zJd&+GfL$LLCCrjRk?3~%@HXR!hXHi3U$yBdl;haO!p zn~9A+kj>TiT~r^fgZj8*r|57c5Brbs6Rb*jP!EY|F)!ZUzq$(lv1Ku%nEi(3CR91e z!iMXHC?OAbj~r}2$fMmd{*w8A6h|RSKHsRrKaBpK!oNJ_9gH_Eq?TfjJk3oF3`2IV z;1{njBf5&Q$N9!EfadDO}hElHqSvQD`lQO+qZ&kx2TWUKFGn*$d%e03#~&wjTG$}l>+PlO9x0~c7S z^}UT7L{V37s7Pc6aehbdCtqO(p9T}yUvd~hbn>D^)NHdboer)&;?c_Bj_yTk1T!1o z^F|70)ZpxXJU-{fc`#nrO7olX&WCKyy@l>Dtm#)(W`Cs!YX@4urtzdLEOS@WMq#Oe zQr?Su+Lq+(YAn(yDY*_wJ{Ht{m|oYv3qi>c{bBxb!!_1{#`t6%T4v&$m7)(zFpSPH zHQQTYtrbKTnpUpRKhyV&#mV-cefAmJ*FXFBfB$y^^%qN4@Orf|g1`E2pMO3j|NHE9 zOoA98v;bWSQABZsuz_I*>XiJC>u=xmh2ibDQ61Qd6Nbk8a!<*55X2nXq^Vv+SEUnD z6FQqACA*sVmRC7&G(KOz@^b++TJgvV5n!xCABjWy!*F)!&)sEG6 zT!y2o$I&un6t#j1GfM2Rr%J`502?1R+`uw}bYHDyBCg<@t!U^HijW6I9ntJK}RGK=eR$FYp$9p)PtUK&CS zq?IQ!&(U;xgjfCsc!H$1vTENASj21ED6HaVG##i*DgkaoFRXx)*g7FJ0UtYj_NmmG zS*LYdl~%hB_O#~ICETm9V4YaK$vs2&2gp+cpkFUxWwgCTQ^3#yi#V6FP`eccuLNYY z9Hlm7wCfd(5hWs{>78mvpi*Qs9Vj7}GhkpFp+>T(uGcW#J!hz8MHk6vInZ84G?SbY zxj}qmQbv+XWUcr z+FsIGWCvmV^hU|5(;fn1?^_=qn6N9MiFib=>0T1ZUJiSWrwbCciTq0|zm{!YCELgt zk>QhF!8WgcZA#EHK53!PNlky0Fll^JgQb%iPYJBXCzHyZDzV3ox+Watr%zEG=tPM` zEn8x47Em*t!h$KVX~IpK%VPLe;*mN;1zKlnKfAijv=0R>Y_Sjpzxc)4h%V7aWRyT> zsyy$f{XR!1Ot#JJug0zo`70wB^j_WM`z0g@Ev=)Y7diSOC&?|lNHJi4kEkL;f04;Y ztIzOWdh70ba?L7pH4vIL8}mq)o>6{*<+!O3Qp38dQ&K_c^7OLec=v)7A4KW3fd@WI z3g2h>&C}m>dGD`mgl)^Mp(+-vrrDCIdA>5ccx46k`<83X5Mi6OwIXtsvlDLM0UpcU z=NaYJ@!89+LfkQkfw|iD3>_^2@DEzT!9-H!sigS07*lU|H5b$!?)67nwtVTHfUNEp zJzFWs5ZduMOtpLW9L6%Y!6N7EYZ-0tvdEn}kuam(5sRF&ujRCSKu7*c%TaFqjE2W7 za!kFCIoS>%v5$_X)1xe@4h?)(b?)?R@<+l%Axk_kW)5xEfQ?$y z+iyaPCopAi^UFPZIOCit?s&UssXJe;Q0C;F(qir3u2Od3Yc zRF)s?7@L_EO?PUWoYt(C?ef$&Yrf37aQl(jHcvonDY~|0LRZU=tiSlRjm^i}^bEJ& z*G52Vsh2b}8=v_qJLCk>8I>ct>7iozZPro4XR_yo zv)axc3$a21*H-93&yehN(`kFVW{=V`U55(kdBCaHL$jMU`s?R$q4eudL$zkD_=5cB z@!$XL=gG-mj*tKL+2p_fm6t^b1PHaU6>AGaNc;UK*b(?g8rs8Iw| zDRU8hBo?u+k{w;mfBktvI`%$}#Ha2O%;*eL5i&g-wwCo~=Z5{Vq&lYlyPv|R;@9|? zv9AcsmDu`o;OnO67Z&(>ZOg(3y9#lvIb)BAdc`=_#KvQzULnjR`3OBny$+w}s8{zq zN4-8ry*@|1K1aRw&o98n{v7p6o}*qjJkL?D{qr34`s;{#MRj>Y>}w)HTiokD@b%Xe z_o@uB7VNAjWGD1HdCL-l$yTgGYb^mN^QyHd#X1CbrS)bXHzbyJC1)blzC2hP*F;zp zLX@oZUnG*OefUpWLK4>W&g5~AE{RquFJb%bc2XN*aAPLyn)k(TWr+F;G672!&^8+*Q}7D z+{cu;)>bc2g0nc;s8)8^9N<#WikDn3eH1AkU!&zkBVP57BXHHz3ZYHYN?I(K3T6^t zalL;mgqVlYsJb~9sC+>WMz*e&q?MZLWC~oLdPtS7Ytva`%P8RLb_u|s(he_JA08a6 zbR(mzN>?)EkDrl-wT0y>=57SgZY@cb9?fHJ*I78qL07-~Jz2=ERnzl4f2x+`b7%pL zi_RLNNyVtrcBRu*7%UhC!a z`?>%lFQ-Ux^Hz3IfIow|!duQF;=E-=!4yaj4i*?(GEqAY?L}|(qd^~TLMwDTnqbI$ zs50#2p$aFpY|fR=D30o093BG2#T+Y=*IlJ~Q~6bln&ey8sFfE&PazH5N))k9u?$T^ zdEB*f?8n4}#$wm45ltG}Qomh=tZS-H&@oA95qbYwoN^-}w6yA1Cy1ImG|_7&sECSR z7*+~`*ND6qu43h!-!ZX5?Igh@`6VrGnJCG3$}M8K6tuQQeVSI@aYOX1Tle~)pFd9IykgD5H=S8U?15J2Y1H0r zq)ac^d1T%i9$@$c&u>6}?M+mvfG0|l4^%WSqcCwqS>i`u#D_0$d^D(U9Hx&B3lrNe zBAO2wDY=}Xepj-R$UD}yykv@aKG~CTM8kqtkGTWSb(Vjnb(XWV{3B5F2kaU?S!36< zta%ic$dhE)NwtxUyczL3CpsKTOI8J~*yXoLnzDUos4QRG4}+Rjk;>n;dTf97WJ!c9 z$6ajCwb(X1J8H27=L`!o9a>zOQ_b#ay0l{73#-kp)R=YD7PSOtSC`~#Z1V86;##Ru zxIdRQJ_}zREEt1@F51*on8-b~WKUs+24^jSXxBngV-je@T~cNgsapAR?wu9Hr0= zGKixh`wJS!&;pr*3%7;`pDfaAR_*ov)%Duz&i|a@g>_lfmS~)C4@H4hKsNFDL_wu? zTer!o>VG#-kUn*hT?TY%=ey1mWOk){zSwv5`$zjfd{871!Uxe}|(u zq}z9&TAfE>Qt9R#<^-J34LQp(^*SS3j)4W;mVg}5{0a={;Xpm3Ks;Tg&ZN+{&;IcH z-;Y-JT&>(0<-U)q?YSXOI5svQqOxatUhtlIzz5&;qEKWq z5mdHf$Bd)F#I+wH#ykiHOak&th~*O1b`0LjI)Y2|wr)dt8NqFdYJUaW<*+ph_pMBr zTd#H;k0lXbB|UL42}|YB2r3z=5@~2D(HJSoIOIE#QMTZmL^u8b!UaALCSu@|B4{8k zN&I?`VT2FSNTS=*I>{J;W*DxLc@f`Q? zm~jsWh|XV&&wqNJ^zzx0zR8oW1}N^%xC_h-1YUWQQZEng4u9^?t}aQ@GQj_!m70Xqg*EYTmZ8wI0xLT2XC7OdFpsG;2|{ZEfVz zHPe>FfQ+>yRA35htgXle-ovuExQ`pQpm$t$E%u-#d11eLVI+a2j4Pw#d@CI!!ui!! z8}`;yPaMMo7fC6dA$Y<*aHZFp_1o%7Tzw?D`%X1RqIe&bJUu*G;hRJG`|cXP+E@Cv z2ug^F;1U&|k_5h6`i}K1JWJm@9v}Zx@nU&-o*zGi@4MSn`Pd=zIr);RNH+bN3OYxl z)baJzm#9kij;6ym$!$VXi8vEKd-}F6>)t}V1IY~B&xS@TjX7!-EJSzmZD*_EV;Eq zhFY@2wzBoau3A|$yxUgFRr@ z)K+f@eH=$$jQSv=IRj3*l?|PP@SQaG)t5=hG;3=vm;(HX8mZF=%#Hn3#3e9mW!;M| zT82w%9d~#H?&*@yR(7JClJ6H0{#dZ$Fnvm1uMGT+6mW3`B^A)kz!kY+5hbg*AfHb^ z|LkDf5@X02Yy~=0I}moYrMWjg2=;wcFhlg{V(ps4s0%Bd_uy}Q237q|@gp#4_9(xj zOMZ;@PzY}E4D)1D4tvzJW6YPTyV;vRSVjOu-m?c!;z#3};#y$8XBFYpU%Vj4_{$?j zS}FC(!M5Kn!^MYYG--IS6c-;F3YS_Vu5pIs zrqhIpJKjnG0_h!XIoQyGRIdMW{Qm6iKi{0azJT(%qir*{3Y`bi31<>?x4NO@M`C5e z-8{@r0q=D4G90_bs?ir*cIKg5cA{=WU_V&S}{Gb-rj0gtvnpmAGaSox$0 z4}JYLc7mEd$Xo+G$?SpNCdiD7Fl;RCDvrwy&nvX442!jCSVmfN^WZ{hDhia;$dzpM zy=*^_lB&gxq;1(pHTQ6ZKLA7iVbdfv1Z-@D-OStt-|EVK%%1-0Ll#`03}!CNU@~?tJsU$5~d-)hctK@vr zL5C9uy}osfiWObU^#)qpNr!_moC*6gAT}GXVmSWQ|C(Ie_J>rdrsS=;S6p`R-@XO2 zootE!eM9H$AIC2nd)g*UXx26@SL~(F)m8RS7I>gu(oQdA%YTAjyhKVlvB7~YGlDzi z)dp|xZra~Wol}f%(H5l(r)=A{ZQHhOTc>Q>wr$%sPT4lQ?!D=BKmEym`ID9G>^a6a z*BBBn2h4mEHlJPPplARC4TR^0kWLJZe1G&fpwt{p9`qR~JU))rsak0(FG&(Rr?^TU zF#yVNALF2tS^T?Mc;Z!618&MFbQrf3U8_%$N1yOaDH`Q|M z;XLsset}xaO+_JPmv6J$aCZ*EtLgFixc;!){jT-P@p-xC`+no=@q3w_ytufb-TAK5 zAa_%xCyU5ZQkBY2+i9AOxh;uFm2KbZI2%I2dK>xM!@G2^B0cpqkQ*Uk*A;jjNS_d~ z%dh}3J4KS*W zzsc+eJlLPVt_Ipl%E6Iik-=#aV1J?Z?Q+Z^MK=<4@6V*_lUJ zlgts$xRzw_(MlbjQu}joc{Tj@G+fz=b^|LgKOG7YWA^tzWAj$<&gTy%opNtS&lE~2 zo?I6Au-TRcG%{--T(GIU5v5bGb>>h=@rq5W##~*s4%9|j@RI4criRMq>lNV1VF$5K z?#HZCXa^~l#`v2-GYb>k>TQMjr!Y(7vd!&aE<;6Uia<)Yh7?zv%!4_lrvKmre5A)U z4pq-OP3SVT+Fb%l6G=36W2*XCLiTiDbMqi!7A?{f69W&tWhee`v`j9mG$znUn->-#V_YoCW4PgPk{URp+6jV^P%5i zw)T2Y7)+@8w>!so*PrV?FJb6aR`g1HKtYBlgu!e|}@W9m^u?(3G7v zA*fnE`v)i39#)+m{jIFkMDTFhpXfLL=F!cIG?m`V`|__T)T{-y$q|Hn59tgpIG5Z zP=*gQrV_dg_OVx>QF=0MYGzARSG?+!=>fltn!WZrTi^JTXaPSjqAObem{aOgEl(2q z3|^gww#P`^ac0TiAvF&5-FXiqa>w4DR_k(VS{^5U)2Sx64W(8gqxRRnWYr(Mtls(G zgxFpCfWYBuw?Ba0oi<1$pGYC~jnivdG;T+FuVXm^_XjjuxD-%r#KD@kBkgcB_y4>! z@9qFR**vjQ0ihrgdtd7dm+32(e2x8#pASZ3cIP7mzGfUmqSQXl;$*A&XTdzp>L4YuA_Jip%Bgvafcj;CczicEV$ zq|b((HGJ!3OdwPs*$?AMKtjs2LAt$SXL&2$84FbrI-0*y(jg67^%X^tnRu8HO)5Cs z#Kp1;woavy^DI{izFEw+*A7Na5^j-=KD~m+ez;v$^+nbjqaq zz%_&^sg^66fhBv3$IfL8+PksjH=Gp-wqpNrSZfOqb^%QWqsk$ZBXn&vl9oz>ZHi?l9ZbSY6wlb+h{@fl}uJTA^7x>tgE~EAjbSFm}2I7F+Y{S$e5y z^A;*p5AV2mwAG|=eeK|MLR`TauGy)`B97%2O6}WV+!EWIW88hX=QNnE8%ur*`UiC$ z8=377;n{U__A8l~i|rm*9>=plqv~7*9nOO|o6MrytHdr{KesQ&s^osqKA8ybw@Cs+ zYpi=PaqM%r9(FFf$yFO|4xx@sCvm}Og*A4iZxt26>1|>U;$M$X#+slZh+(-1?@uZM zBU`*XNpajOq+Uc`hlvdb5^j->4ObH3*k+T6pGWo8fUC>I9>`1V-+$sCFy1E<=KV&A zW3P=J`%V;nAJX_Cla}FH`&KF}wxQ;;%%Y_5xW0`5h=S$%F0u=Nd3ZC~0#8SZ%*B3t zZ5_mQ@&<|!yf4drt-gcGZ~MvZwzIRv5__KRWy=LpTX&LWG?^^;lwDq;ASn<9kcHvx+A5F4NiEeoecW#2 zZ~5M#Rim~ubtjI)SZ~e#tS8#7p+9XrWX9}U>eiD_{Ff4%KFQ}h_qY{vzcps`JL{e6 zy}=?-7n9?-`Dh@4kn<=h@|Ct0S5{sfAHnR7rr2O0nvXs0V5Fl5Jqg?UNCN-Z>RLQt zGb3VaUzG5G+5&>c_O`~o;XdeyIH@K$$lAf+_~>$?gAzg5diyE0bfe(pnBt9o9SHnl z(0$D#vfWy=3|AiQ@73vyil=+)m#5k!_a~Mm#tc@hE!mJ~JM)#eS?lrZ=SgQsJ0!~v z7ZaOBua?LT!Yqy(SL4PcLn$nz`AAlWLapVru{ymQUu(sjZBiw1ul9gihc;og4`crn z#r&uulV`VktT#u3-Vdn<|(fqWY)lfqU{0F_3*`Oo)CQB=)t{529{?{%iYBUV?hVp`(~ZVOR0@V;NsoM+ zEWcfDx@PZ+5$8Qbt{(W2@Qh*z_{F0X;xOp~t6&VukhZRaiey{_Z#gz-~{8jqRH^_@)Zk1Ksj}U9YAXBQ0>argiD=N^WTZ2kimU?2S zX+<=PgZOVxx*AV^e67Ah%yR?BH#unC+Ih zxCq|D!ZP@)VTzY|#N)a%cW@2F*WFdZp`|7;Up5?t&7? z!xo=Ng^?*2n8il=BDF~c1;^h->66Ybb=U=I!4K^7GWJ+18DmLhXk|SIr< z?DufBW!w$g4hTRW$Zi-t(znQTL|aSLs*yy5Oyzd(ziF7huWV>_He0|8glgwq5B1&- zM|N>1#Zv}CcI6ce9CyO@Sm531_n__}r(24{`TWyDeC&Y)FXgn8sKRZI+jUtRynl&0 z7bHJkn6>``m{_!FCx~qcagssbLLN^y74wBn^Tc-sDD**Y_nntkc`-N4f~Y}82>qXZ z_#(0iPeu=o{u>y@Ke9WyD|B(!3?7?^^C3bLhW& zwE^0d{tmnZg_FSqY#i1~4XxD3>$Mtvx^E^raIc)nlRIp@ZrXXrtQpg7T0&F$Nf;)+ z4eg@VZ#K~k3yGT>{6Vu-W^JSFF1yy3Dv8moA@E5tDOw;yL;67GP&~COe|Fj@x+RHT z*8qCe%jJu#+@)3i5tLRJXY3=5d~0eqpGpYxE+E!3khGyCHwPGG$I$uvc{q_756(XH zg9mDxha9622B1utw68|oCzQY<(LW|01CE!s**DB;dM({GI$RYtPQjO&e$FrMFW0c> zO=Hjti6d(2u!HFGMTghhL;lV9x)JNO*4yDKT6&7Pd1_Mo(yP^#;a^}yyWYs(?DygQ z4#~b>9YLY>r(BGH(;?NKH?PAwh>@J6D`YL02+by73=ipCU?gtv2cCfPZE`H!3vEP< zouQ#>Tgd@g(0_ElgP*g&K}JHqKO~x(>*qJewJ|6I)$~*(l#mbcW#x$>pYTtth50m;pG8-HT!ziZ66>f%|sJ9=` zu@5|2$2ATw#Vzs z4p9NGgJDpo!K7gACQYXlCwkx&QAS$mN-)W5565sW0JS<%LP6KBV4`U*+Qh~a3s{40mLV}3n6H!hj8h2XaK@bW2eUf}e%1wYKkCOQ z_&LPKeSb14{keoz_&(9|XuZyodSGeW*rrw6T6lY+1m~M(m8t+Ik75mppBRxq^nij@ z?VUh=#tCgn5fI<3JNC+@KW$AWo1XM>SVirR;~pEz>>SK^;l4DYyVw0IuxTJs15~0d zb>D60U0LdPq^>yPTyo1ft*BFLiL9RQOYSpdwr9Cc8rim?l1{yU?m`OXu##`bD^*DV z4T?z>$gFk?(;5d50^Rek(j8hY+BG3&zldWx2eHaufgRUqT4eO_M25SCu>cCpk0b#k zP-@_oY)=4G7MHCk<~9$G?4G{(Hy`mH<9RMLZ$*60u$j&qyEqF$GI#YA9rC>~Xkl%c z7!->)flz^Q_!F&Z&yw%6F^LHQkfaL< zhc+mVjqp|A9?^EJsUXx^QZ+oegUYqvKtOf9VZr?RMZ?_I$}{y5&P0kTE{>@ZHW6&x z0;EG@%AB+jV-0;hzVv1QmR?Jy4?9b4$AUf-$Sc*5p@$K@bY*DW{H=UH6sq4Viq*1L z8`&!?o$1Yo$utcw&gJFTF>XCA#qaa3ZBN_rA-2x`D7d+{d3pw!o%?T^YK(YScCjZC z7CfHSb_s4Z-ILY#-bYjz!?G_^4@*tBV<~PkCl#GnUk@|Mt=*A1wQKe`b<8zLjAMi* z{*+?vWk88>fV8tdfRUm=Qd|zf;T>_fzvH>JHQ_(bXfU(=Dr;hE`>5MeNb0hj{x;q( z-3pvB$!Sp%mACtj2#^*#n&Se?Ku=&f=@GK3cG;z#@z>|?%*B*$=n{HubKzNc5DBCD z@Dmqk_Tu-y-F+fj`VnM{S8FHu_uS&aeo_(rERh}BSkHi_1on66>r9&>ifRqvH~Qfg zkvopK`QYpWF}2Oc2e0Pp-kgLAmlDJHlUwAEgQ5t~Xj)jc`Jr2@-k03SA@+pGfM`q* z2R8d2^iz1}oOcdEMGvN&mFTxivuVk$XsHr4RQGCKr17=wA}V%fm8~@tfS?C?CQ9hC zji!xjt|9|R+?3hl`vjIHvN=d5?T7?=mKIz9x=5G&yT^rAa`~}Ba=4KO<=K%;q#i>A zD(K^A%rFvIV=#~yV}ftK!i+FsoIT07BF&D{cp38)xqK`d;1?kJ>MgzJA`*?wEUk#Y zt1fm)-j$6?=|yY{NtpN|5qipqEtF*p$75)tQsUtbQzZh^UT`tsky@6t%6~v=+iGu~|XdD2)@nSy^hc zymYFWE6*Ont1YwWmz-=x(i})Votg!QV~jVhP+*8J}&9>pVf~l7ysF5bzD|cv?Zf9;~R2Sf=Zh)P(fY> zhqefjv#ne_QnmD>_EAkXDUC{eLU@0&?KZYVd)FV`CtX zWss;kH}t$D=LZIKEpThbJ>yloi%jVp9aa_?mf0$qc4v2QJ^EdD=bWuewN*E7j9mai z^)Wr>&1zO~UCpdi2GAHRU;kR{pbqd4O7Z9=+TeMQ!U zhR5pGXBtZ9a>c<~Z9H23)X6l#TdJ0LIiazcKjU21E|)z;l~{i!XCL%1=TA`>nV=>MBT3VWUfB@UaU z#cPmTTiz2XmSsk~N(7P34P$iU{15&MQMsG+B5FOMi1>tpHm8H#*iT7nXJ6hiuP>V1 zIZ{|6CFD!uP*9URfka7AgnGAa-3I<$jsuQ3t26vBz{dU*JufIKohfu%pQqBxeA0ud zTm}Z6GjVC?H3mw`)9O$(H(#+d>)Y|8MKLsGcJo@0A~L9U$ZKNPL1FI5p@dJTQz(Oj z%MWZg-If!-UIJx}qThgQ06qi`Cw%b>Qkjh!wk(=vbgE;hY3-BQEy@+>*tOothWG}L z#I~zRrgoQF^H}97UQa5PwB1K|;~2Vc5VU9ifS^Wf*5E>Nyz2}eTDwyNLdJlVbeXO6 z$6oWF->FV8_0X`pP5c19DCFiA0@$8gv?a)8&J@8JN~MAsBniJ+Ly6W?Oq!s*jspF^FjzIfL8N;gXIY#|K}1*>!rD(n?}Qz zM8)x6;~JHI14%RzuH3?iYw<{jYKNAMFRcqh++C~v(!-bWSJJE?MPn#pLCC%0=8H4`R9r?it6eL{ukLC`%M zzh#(j(-^Dl1g02*W5tTqWgiK>s4NZ`B0?CCDd7{jS&xqv%f^|20PedgFM+ zLt&))6zCFk4Wo##4K4r@bkQyM!HeTDnt>>XTVzL;eQ=JKi3L=0&bWJ*LaO8}J+OQ! zERm9^5Eql)%|FBB%Wf&YfeS!}Te2_jgB!Sr5fNr>$5?*2RcHZf6l{p;=UO zjf(%Z1VIKA6zCH;S{qKJh{TF#1$IEv61AB4qrQ*U@^%^c@(BAe=q>H5BqtN9oMe~aoM_Rw2O1*5bRgc$L?VPJYAZC*MJqI5MfHdiMG$z6+7_@h z`fW@_AYN|!cPwZ?zrsN<4H%jM5k6Jg6vTaV##FJ_nj%A4m*onSoi`$*%{0C~qJV_q zPXLsSPgk5B3>x8v;@#{k>mWC?ygA*`N}rgyo`dpoGU%Cp#MgLxm`qa`<{LpdWw;Y^ z?cSK%?-bspFFBClHjGlh>qvEj*J}4;S3tP@y*tN=myI++jw@na??tLXW0Ulp;^uaq zB8sC`87At^BtnR1W!V^g5Phf`F(#Ym643suO56nD?8He%+60@vicfxwE(`3*-ki8b zyfkJRD041+&q5XUmk?I^=f}UPnJID@+Qd35##OM7~4J!qd9hF_Nr1Jc82h| z;0HQlp-vV@9$}T%0KUz9q%BQ#@T`TEV_S;xW$s>Uo;dl~|r5~+zzAPXG#^UF=xbqrurTy557kZW^^_>(%OyHX7idzPJh3jziHWT~7%bdAc!WXwQ2PsF%tWQC45L*SeldVA+{FS& z3^{2gSnnhnVI?RVVcKKoN!M>L=i#1tqx@;TLY0I1O-}dv)lH}*?RD3Dn^gcHPy(Hl z#XF_lE|G};E4GL5y6N=x{rtY(#X!JH4@*2&Bh(Mk)`W-`7W5yQ-yk>CkR%E9_y+c~ zX9-UfJfbHTWc27e32Z*GA`f{ z`R8k`C<)PwP;^A_D7~vHaRh%Ci5v~%WAjn2)_~c`5d@n3`2hN$h^!hFQ$X$^5{!AY z-v1yxw5omF0Vx${x&?$-N8e2W)YHDDx?AV?Cau`T`%Jxu6?wNCe%O#3mhTP}h%Coa z+P~vuzd8~yX)x-3;UdcrOz>TEoEijJphEk|xZ)kD(MlTcTtTg?Dx}fJ!LbVv*yW@g zKVq=c>Rk08dOJ6^8SF=`i*b{B8sa8W3Ok5F*5F2)S%6as+#|@wUoq^Ox4}1uB)a*% zzJg(ul?n@6OG^}@^9jR|T8JU^@1^bX++4M5q08S?)%&@2+ z13T~yWkVXeQpIyv1nH2gx8y0AD&L&CIW4h8iqvG(+uBPku=+qij;@X`95~M{jG!WQ zT18zQG;8Xrl*olIoNeQZY+zpf<|Iao!az6MHSv{P;dSV87x=M^e8nL4-(7vTTi!KK z{c31usJ9ocT|hVB?uFQBG0*$d`etCKy@$T=Wm2au)@5c33vee0P3hV2TkGqQD84;YNu$%@Z6s|+}Z}4D;^EB`Ql`)hw%sf_l>BXv~8TkP~u^5ohl@5Ko zzkMxm-uEVtKN=|Hb)X_YeaGc8n0U;r-&#V8Uk3+XBSf4rjn6q^Py~p}58`j{atuzD zP3&j!k`UL>H-OjbK30?Gc$H62NMG9G5{HFZCpa5I$>^YU~`>-+3$In z{SN{FV!(_;A_5nif^jQ_aj6DS9ESAp9bve#1~E_TIK-@8La=c+H8urPa!F6?TD4F0 zW9D&LoXa@IIURKN2<(E&ROA0(M5R3-%kUh|HhIfk};py}@L+ zG2HWq9d#pVYvKVr)g(QJ#8}dj=9iLg@wA%D>+LmB;co?r*9>2!Mk-(KYl^8di7!!& z1}!fPfcm_{aK^rrV@@FU{bv*f=izW9+k6mRYk_|RF)QkIaL0f2cE9L^HIaRM(R*fLbu z94FK5C#2}sH+qzCL&CxF5V7i6Lo!zeG5AS z$R#!OZY8&O@>7<-=}SEfWwNW43mpBJ@U%{bUzdHSYdq zsIDq7kL?$bSK!C~B#U^Lm`f{+6^q!p#o5H=Onu&Rz<}p$+nxnmACF)M>ze4%vnv!x zFdc%_h0Zhho`08-awd~hZdW}dFg&BOeq@sB@d7bqctof46}T>anx}p?*C5Y3qpR5joq24 zYUAm?sKcG=GX7Uo%0c<#@_M8|f+#LLPnIbx^0=23a7&Ru5!~k*=YiV+?;(W2p%c5R zvd@0gFS(IpTa1^&!5gOm8)cHi_HsUXS@1ZVmV6J+R;8y{cab9-8FAMprz?@hJ3*VR zNUp-P_A!+v6bP_Gg?K+FJ&S2+yIH*zpL}8KGl~g6%*Q_*+-uFx5pRA~gM-^Ti9aW_ zzI}&3;ho=e_dj7x|<6n)a?S9DDeMj)%`DktAIQ~tt_JQlj<}35(jNsFW zc&Fc4-^H;zJKv~lyNN$A? zQu;1u1ZQ}i{q6t{AQ|^C1#7auJ<_^Y1_I^O#^mrZb2O>qm0FOwONl`>Gq!{MpgU6j ztBIytFgnnKs}RrfGdyDVWwq4p9Nu1$pAbl_>jLG_Dn2cO^Z~;=xYVSkBTtx^eh1(ee!y^K=Hv4tGhh>z1lUg`Z=D>fJj z0-uvxRAyj-54(~XW6|^D@=uU3*B}~UF^Zr$7|%A`6b3X7XZ;=t@fOzcy7Ba$ zyKCvLmFK*R>ziKr%jVy-xd3;N}hGPwz!tjs< zBcEMgso-%#(t4%?AYT5DbX9IwIg&~*1Gz&r=Kv8o*y{cxk9&0scM|DtE?W%uqeSiT zuD%?A-f2-lYD|^`nvRgGS~9sm-@>xQ#sk+>OL0BbP=jC{$=3)9=>yJ>=fQF62kIqa5zL`p+ zW_04q3J_WD8noVpm))FWjkV33Bk;`9nmvrBud{jkh>!QWc*oXJ@t)Q8*{K0fR||C5 z3?y_I=$>tK?Y{q%K%qHf|M%fSXUmn888P5ra94-$uSrKzzd69Ip&r2}28u^PeOT?s zuTJPdgF-yMzB4NPnMupNdB(aco_u_rn`^Z1wRp7lI3p%tzx$6uLtq=k5e<~`iY9je zc2lOh2--jwVAXXNbrPyJzbZo;L{*At1G6$l8QW>yZw!H412i7I$Jp3be$O3>e(2xH z%`VT21JjI;+ySJHwombKp;G>6RH0IJPAU~xv3-b@$&;bUA2|9$cF za;fIS&3yiX(T2!ek6^dGDg4C1)y=8(Pu9AD>&BB~;IvaIdl%3N(=KSK!!ER_j#rfB zZF!1vpj;wF+osYPC^;uJXbVH>!Ud!vUZwy|-jJw)yM#g*&S4|*$c!OtlL+9%Xc{KI z(B{&=q!^qvbgT4|ljACSwH8+Xq6zgD(o$!a-h6U|FJPh8M^LT+GWxhBdOoA5FEb z2ZDv$Ez~}-tEE9ep7SvDr9>(%tfhyEtzzahvuwF)=QOd*@qC0wXz0^$FLQo5t2w7s z0d`1y0{$@7Os@!9_LtDVGw=luYrd3Tt^Kd1w$#wC7!ptQVG9sI+IF z)0umU^GsuTgn+w-gPCXT*5K}mR++su3%emr>*aP6bJ3Te3eKn~0gd(s83B!# zXgOTgnTVfc65{B^+se#5v<;=h{#gsz&C974w1)Jp2-u=_;Tve#2&|sbIJCAn1HXb< z=gL`OJ#RJ{U~$`)$n=LqX1~Ld(rU84=VW+|_6_f8Nx6_c#((SQme*HQ@N<~taIVW> zDGB?%N%wj_bzJhh@=Bjnep+u)-){o{6vO{W`2M5IE|bUNk;hUgN53wIL*{loE&G~U ztj=z=6CV562}{VNPhF-%0_>&V1T3{^*n12XC$;zex_i0fqr>=4Nj7dzulBjS9Q8R- z+#S+W`S#0YKb5`z(HiX6_v_92*=2nDx!DJk{{CsZApPkb|FOIO+4cJVNjAA2g&szL z@Ga`VphJ^ZvXnP7=N$&YG&ng>EVdLxwf$Obs~1;qsGi@&@IdGit2gf zPBW$N^1S&EAceXlK(t8q&aACdo9oGtM|^vCK)fMdf1!$)d^f*YzkOZX;gV0yQJ)yx z9;DHSiEZuM(T9S5|J7PYpa0cb^u6?vgNrlnSr)OpV(r5q0EG|>X_R|53tA*dUCPjj zg3V=h zF|%)TWDy`zm?reAu7z;~83fx6_qiwz=_r2Sx|47$lu|L4|CMTr+1=`0j=}Qw1W!gb z9>@FQ=9{Kg9Cq2!vw;HsU*o}_I~9~x3~S+e@GYc$;PB%UQc0v56V-$PFe@2Fj*wpx zkrSJm>^$yXkQm%j)W;Je);85AH#%cmm&MJPadFXiHt7c=Aa7N7AC?QR{)Atx`yN-N zhGiQ^nQ}Hv@s;)DtEAbnHf%VIGogS*YBk(Y^5|3Oowf*Bmzr9A4KXeLF3wXN9WKiI zrw8KLT^4n)cEozwRW|HvKr45)<+$B|vJ4NuD`pvO@uiD~9k@pBYks&}x+9mT2}JYc zQ{O|ob1$#4xjzqLH`e#6QC*0vYt3p#QyN^86Dorx6vD9gx8JwGE=%;u`~b7`O}Yd26cW`XYH?|Og02R& zy2&xJaU(A@!TS&n0J@(|IQ%Zk>X66KDw648NHu7%vSO1^);+$1HG2DW1tWP^;(^3i zP=r`4Azf@3WnWUFA^Ej;Jb~>_FF%ga2@0cT=1;9@as6C$=qtWIvgc8~)MT~5+G3Ko ziWaDc1R=JTB;6;u9}}8{5@G{~`QOClcx6;68*xPMZt2Cv5Kc~03oBM_Z#mSnSUitb z>FiyS-&P{-@r;WUpEoT*9AUj_!%W+oLv5#DF+|&9HXs~1Lhr~`;$BD|rTuKxnT<$@ zwtkLK^m#_A(Mp|xCR$-_CDmM6SjQW8amu?)y@`s<%7P}E(^9|CSQ|~nd>3QY+F28q z_TWyFTmmV+6-J%})$#J+_jpr8?r@^#7tV2|=c;L4N0?(h7yTr7s1)W5MhrC;7iNIw zCuQh~!I1DBg{~m(X>M55cCSV!!QAQ@On~riwAtsVl$|=E171F=&iE**GQZafe<3ZFh~_6NW5Mz{v!ITn zKWMc5t2^^RS~r9ui%2PXQ#9qS;EExa-jsOM2As! zG+i>}jHex^&hR_iyUzW{@u?f_(g)eps^+r$A;!J;AfZQR%-jM~VOY&+TPuC!MNvcs z@a`*1VMWVTsDjF3yl#4hW%>G*=4pl4N358Jj#su*wVlt>L##5I_XENBn!+)Gt7Gi# z`uyGAYJc6J%c!YHD>K>Ts?)AFK&>fs@ZLTi)J2PgBd2tcnDHw-@&*ex*0%Y{=J~R! zDyj?AMB6&49X9|>{o|zpl`f2b^HdTaZ88N~V=dN9);F}-411&P__E{ieuXBF*NV68 zniEZM2#=bxInSlx&5YH|;mWK+vaa?dJvcL(WFza15{8Xu5_+_mq#3nL{^;r!X9)dIf z%n#7ig+BsW{K!z7HnAOgbang$%i|J=Jjk9PPy1|0< zf)ElZsbu}L*`WkcycMZjj;ixzO#yNv1XAX}LqMsf`qGRWiJts3LC$$J{V9 zmXJcs!>nN#^>-r|fh=0MCY{|wbH2Wl5wa(laY+5o>L#Ii7s$;vE&31DcDOeCEzN{i z6W3Tt@&Cq~*8A&TnLJZPa6$Q0?TTeA#i3)Gua z{?9t;wXUqA(1>Mkt95pr@qw~il$Gj?-8{{LbdDG`ujD)a5E-g`BmrB9CLW6)OoGlU zR+(WWZ0VMxEX1lS{vt54BOoDt^W;l^)ipso z&Q|h8wC7*eDxwFU3aVDnJ`7yiZ6Gc<+_@uIpgT5%D0ZLps3}y$G;4@jideNoE%=dT z-HVM362Hzk)YcDC^W<(>-yDm$#m2vS$Z7q@moU(sQ9+G;GARfyyKic(#ZFZRKI*S| z@4JuiyZIfJ{%1`9iHH1J6R-yVu_mbC+FjPdA)Ducn-!8H{^h*qnP;E=KW_rT^i@~3 zqF6ss-c{(72EN=|)_@%cLA;UFGoGFA+$YDkF=h;8Mq=bbIbve)_e#gqP5#$k_umBT z`pALo!`UmVeNo&E*Mz5q|1cts&0kU-7}(8o_tUR0|JmzO*W23qGPJ?9)ktmSZeRCd zOpkpD3-`;a6Fw@wL$j2^q(uy1XHX=8*)sR{G(oAb;O~0iC3>cGs}#y<>nc8) z1yNL-dCR`w2;re(C2K2dB|&fKxddY*UR}L_ef2+sGnliK63B)jyv0+AE3iCb6Id$5 zL_OQ6$spEbrNXW4s1gv~SAKCAw$)Pfv)<{E0lJ+U|H*}G>6>e4>9hCqgHo@UbmY_| z({Ste3q(RwQ2qO~jwk^jCbbuVJ;mI8CgVLc)G~X4=NPY7gxaWlmH43Rj>1G7&F)?G z@>11*cX#`$Z{L*Jqwst?W`eg`+x0P#&s(-YdXT2mDH5U{FT?{9PF4fO3l)xczV1*t zb;{aQdJ5zfCaeiu7Iyvke}Rtiezq`R4LL2=jpuC0=;-r~fCR~5rDvO08%Wi^JJdBS z>i~hD)3&CDTn;ZUd$i+KbBCghw;+zOksRd^$!t4TvXK+8;^F=NO=Eu7<>oyDJ_@DC zds0>xC(`4qR&7h%pgcSS~yU^ zXR^D1Iu(_b0rRXnaf49MD1Z=KmX(#m*8xgoW(1cksN8`nk5LwXuHIi zLP53`dmU@X0lbEjLW?rpa^h~(&~j?J=J6?I4T_y0yzf;}<{oCnP{9oYmTtv<&UkgD zovH(eS9#FS*Ypbq-(yM#;XHal^MrhmSfvO^YupuzNn_KSyp9S0X6hQ=^E|+7rMT=d z80N6y`J>M_J4*Qm#hFb93>5z1J`S72cBtMDv#P}v#l=O*O(&vt8cOamjA>(i3ai`{ z*7nbhOZ2coZ)$+!daPY^$`iLzEvwA=XZ=g%T7*UA&}Dh`%64u zdVIH~TS+0VpPyU%zH>h9=IDevTj1rpT4&q$|Jg&`YJz6*2b zLV-8qn%7?0vRCZFhP`K2(xva(z)i zCP?-77KOfvq1{xVXVVj4_DGB`&PuJ`f8$M}yA7V%@VZ7$H*tYPwI%)k60p3bY%Mi6 zed;<6NjdD3eV$bhI+}6KJZxAbbb78qW$tg+%o92$qDd{7$ETmR&QoN{WXqWYpd68y z-1h#uggCvNUz^Ou2ynecbmZY!t>Ii_!$5D%tJv!qeC!jzfG7#Xn`AE=ZnwO$6fln+=FO$FKPo#pj9tx91|*_ zSB0!qlR^fzI65U+#4zIR;NT+#{Hsrco&v!?GMfVR7{E%+V!)!7{OlId$Nxr`BaFaC z3d(;NPJeoyBrnO)%+?E0jWq|~>D222YA7@MrIEpUAvJw%Cfv5IQEE&fg8!07;^1gY zl7tg2;4SjTz0)>gyQETIksNdkm0&eMZg#^=t-Q~6F;C8 z5%qY%j8~A$N5E+3<A<_^k%4BDwD8V+`NS<`om<#J0rg<Xkf*Kj%sFD^)a&Xrd|=11}a3Fky$X^MBnvhU!2OUr=~>4Vwo=s7nPYR4=3p{ zKprfzW(DKWhFv%fY1n$j7iEaT!y2}qMiD}&a!R}oz7#rJyW#ec|AdLbUTSkiZEZGMlK8p|p!{AvOj@ zWc;$^u>Uc#F@uis1Np#x;jalMoSnBE|dF^c`mGS zX3x@)*X2!$N~Qq+;h+d!C8=!Xw?Q_?BZG^`;B+r-b1@($|vt@zno0-tmOr zo9Q0z*b>#HB17fYD{U$3ogr&}9h?E^)M$Rpm+u?7t5DEJC_o`{>^5AiXx-u0O za_tc@ae0kg>V3LSq1U7F0Wiz4z!Di!W=fb>*ibwbs?IazFC(X}@VxU;55fwOu>8}D z1T%-xDgt|YqHc3O43O-FV|LM5ncIu*d;-V0PHN#!vlyeB=Pa?)&sJRFme>t`*FYz! zi`3K>rvf=EtDBLN9#f_l@Lo@+7TH)>^0354ofnxC1e_DF$OltHVY(y{7xR$axrYtN zLz`V-0Y^i$o|VVLp^E->Hn*_Vp;9pitvrXCywz3kGqT4mQ$hZrZ+daff6T~x*dn#F z+Y)ZR%1i<}dxF95qPf(ZNFkN`6wh-~34InUlEzsdHB=!kDYFu<$}EtN7A%g_BF-0M z!&tv+lV_g)iU8JeSIs*Lh@uZ+4ds{bDERDQcd8V|)clwOGF5PX`Hspw|pQco*)NE8q z`Q}0x;A4~}Th73al}fxx$h_L-E)sXVNXX1-$pF|?8$iveq{h8seyE^4@%orhi1{D5 zq^^B^>A7z$=3T=nvm@uXDEqWIa!##%`*v!kZsp}t$Y=G;W35nR*iynHr$rBq*_h?t z-k^9F9UAASBl|!^_7)}YBzQy}>U>8ISwsk6wkkpk7pmIXOoaG=_PNngb+8zy%pi+2RQI z@)(}uf&b&3;q%uZxcq~zqc8^Oplyffj}rEeFO>s)h47D!hqK;i3iA=ZW4og;HpgxU z0JRWZp|9~Lq#sT!=h_Teh3~XKB#bXX;m=H<|075bMhnF2Am0XJohl65=R1~X_W*`0 zbh@lQ$0K+NO2B7Dj5+K4bLo3FaFeHL{fw690RDG+1cM=}S||9=9G0&)G#6j21&6iMnR3bBJiKz#8` z)H)?0127a1^5Y|G{|#Ov5KuCq(DNX}wisGUTwceFL^PcjiX?^#%k!V^|34!Xipu*l z&t2{sYuNw0)^MQi|HI*d+5dO(OQ-$i)2GiGf8Z4iu?@}<>fU|24Xz=@eAm7N z*n<-k>#|1?{3ww?TUxJWP>`$HoAfWlnls$o1E(1ouh_jx+B{Sso zp8_CGFJ6LPk+|N|0tNB*=sp?C@@VziEX!x$Bbwll(HRJnr5R(}+9g^Ve>(~p@si4vaffknCc%V&;cx!G zPQ84cexH(E=y0B*q=j>2Q^cME5KhU>B_0d^>^!kQbM-Wv`yihD^8nd+Jj;mxTQow{ zM+}7^6m`lO6H_}vVg>Ro1|v2_KI0Mxfg8-GyoOT0``}#^GPxfm6u>^XibiNVUORjP z2?wA)c#dwN7sP$CDRN#^Z~9(3FgbGN5cK98$l_>y`uKJUV_6;$5F*OA$8%|U;4p!r zfP|P4I(wlRYK~SAbO0Fx>^tbT?+~te_A%kJ;E)@iqXlp(o=P?v=EqXLK8;3wXnUyd zBldECZ*Q0HNpF@;6Gw0S&xDd4G4{z|zm~m8$MDm~x0=1.26-0' + catalog.cattle.io/release-name: "" +apiVersion: v1 +appVersion: 3.2.1.1 +description: A Helm chart for installing Portworx on Kubernetes. +home: https://portworx.com/ +icon: file://assets/icons/portworx.svg +keywords: +- Storage +- ICP +- IKS +- persistent disk +- pvc +- cloud native storage +- persistent storage +- portworx +- amd64 +- Commercial +kubeVersion: '>=1.26-0' +name: portworx +sources: +- https://github.com/portworx/helm +version: 5.1.1 diff --git a/charts/portworx/portworx/5.1.1/README.md b/charts/portworx/portworx/5.1.1/README.md new file mode 100644 index 000000000..e33358f51 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/README.md @@ -0,0 +1,239 @@ +# Portworx + +[Portworx](https://portworx.com/) is a software defined persistent storage solution designed and purpose built for applications deployed as containers, via container orchestrators such as Kubernetes, Marathon and Swarm. It is a clustered block storage solution and provides a Cloud-Native layer from which containerized stateful applications programmatically consume block, file and object storage services directly through the scheduler. + +## Pre-requisites +The helm chart (portworx-helm) deploys Portworx and Stork (https://docs.portworx.com/scheduler/kubernetes/stork.html) on your Kubernetes cluster. The minimum requirements for deploying the helm chart are as follows: + +- Helm has been installed on the client machine from where you would install the chart (https://docs.helm.sh/using_helm/#installing-helm). +- Tiller version 2.9.0 and above is running on the Kubernetes cluster where you wish to deploy Portworx. +- Tiller has been provided with the right RBAC permissions for the chart to be deployed correctly. +- Kubernetes 1.7+ +- All [pre-requisites](https://docs.portworx.com/install-portworx/prerequisites/) for Portworx fulfilled. + +## Upgrading the Chart from an old chart with Daemonset +1. Deploy StorageCluster CRD. +Helm does not handle CRD upgrade, let's manually deploy it. +``` +kubectl apply -f ./charts/portworx/crds/core_v1_storagecluster_crd.yaml +``` +2. Run helm upgrade with the original values.yaml that was used to deploy the Daemonset chart. +``` +helm upgrade [RELEASE] [CHART] -f values.yaml +``` +3. Review the StorageCluster spec. If any value is not expected, change values.yaml and run `helm upgrade` to update it. +``` +kubectl -n kube-system describe storagecluster +``` +4. Approve the migration +``` +kubectl -n kube-system annotate storagecluster --all --overwrite portworx.io/migration-approved='true' +``` +5. Wait for migration to complete +Describe the StorageCluster to see event `Migration completed successfully`. If migration fails, there is corresponding event about the failure. +``` +kubectl -n kube-system describe storagecluster +``` +6. Rollback to Daemonset (Unsupported) + +Use `helm rollback` to rollback to Daemonset install is not supported, if there is any issue during migration please try to update values.yaml and perform `helm upgrade`. + +## Installing the Chart + +To install the chart with the release name `my-release` run the following commands substituting relevant values for your setup: + +##### NOTE: +Upgrading from previous versions to 4.0.0 is not backward compatible. Please review the values configuration for the latest release. + +`internalKVDB` is set to `true` by default. Portworx would by default use internal kvdb. + +To use an external KVDB, you can provide the `etcdEndPoint` +If the etcd cluster being used is a secured etcd (SSL/TLS) then please follow instructions to create a kubernetes secret with the certs. https://docs.portworx.com/scheduler/kubernetes/etcd-certs-using-secrets.html#create-kubernetes-secret + + +`clusterName` should be a unique name identifying your Portworx cluster. The default value is `mycluster`, but it is suggested to update it with your naming scheme. + +For eg: +``` +git clone https://github.com/portworx/helm.git +helm install --debug --name my-release --set clusterName=$(uuidgen) ./helm/charts/portworx/ +``` + +## Configuration + +The following tables lists the configurable parameters of the Portworx chart and their default values. + +| Parameter | Description | Default +|--------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------| +| `imageVersion` | Version of the PX image | 3.2.1.1 | +| `pxOperatorImageVersion` | Version of the PX operator image | 24.2.1 | +| `openshiftInstall` | Installing on Openshift? | false | +| `nonDisruptivek8sUpgrade` | Used to disable or enable smart and parallel kubetnetes node upgrades. By default, S&P upgrades are disabled. To enable them, set this to true | false | +| `skipHealthChecks` | Used to skip health checks. By default, health checks are enabled. Set this to true to disable health checks | false | +| `pksInstall` | Installing on Pivotal Container service? | false | +| `EKSInstall` | Installing EKS (Amazon Elastic Container service) | false | +| `AKSInstall` | Installing on AKS (Azure Kubernetes service) | false | +| `GKEInstall` | Installing on GKE (Google Kubernetes Engine) | false | +| `etcdEndPoint` | (REQUIRED) etcd endpoint for PX to function properly in the form "etcd:http://". Multiple Urls should be semi-colon seperated example: etcd:http://;etcd:http:// | "" | +| `clusterName` | Portworx Cluster Name | "mycluster" | +| `usefileSystemDrive` | Should Portworx use an unmounted drive even with a filesystem ? | false | +| `usedrivesAndPartitions` | Should Portworx use the drives as well as partitions on the disk ? | false | +| `drives` | Semi-colon seperated list of drives to be used for storage (example: "/dev/sda;/dev/sdb") | "none" | +| `provider` | Specifies the cloud provider name, such as: pure, azure, aws, gce, vsphere, if using cloud storage. | "" | +| `journalDevice` | Journal device for Portworx metadata | "" | +| `cacheDevices` | semi-colon seperated list of cache devices Portworx should use. | "" | +| `maxStorageNodesPerZone` | Indicates the maximum number of storage nodes per zone. If this number is reached, and a new node is added to the zone, Portworx doesn't provision drives for the new node. Instead, Portworx starts the node as a compute-only node | 0 | +| `maxStorageNodes` | Specifies the maximum number of storage nodes. If this number is reached, and a new node is added, Portworx doesn't provision drives for the new node. Instead, Portworx starts the node as a compute-only node. As a best practice, it is recommended to use the `maxStorageNodesPerZone` field | 0 | +| `systemMetadataDevice` | Specifies the device Portworx uses to store metadata. | "" | +| `secretType` | Secrets store to be used can be AWS KMS/KVDB/Vault/K8s/IBM Key Protect | k8s | +| `dataInterface` | Name of the interface | "none" | +| `managementInterface` | Name of the interface | "none" | +| `serviceType` | Kubernetes service type for services deployed by the Operator. Direct Values like 'LoadBalancer', 'NodePort' will change all services. To change the types of specific services, value can be specified as 'portworx-service:LoadBalancer;portworx-api:ClusterIP'| "none" | +| `runtimeOptions` | semi-colon seperated list of key-value pairs that overwrite the runtime options.| "" | +| `featureGates` | semi-colon seperated list of key-value specifying which Portworx features should be enabled or disabled | "" | +| `security.enabled` | Enables or disables Security at any given time | false | +| `security.auth.guestAccess` | Determines how the guest role will be updated in your cluster. Options are Enabled, Disabled, or Managed | "Enabled" | +| `security.auth.selfSigned.tokenLifetime` | Time till operator-generated tokens will be alive until being refreshed | "" | +| `security.auth.selfSigned.issuer` | The issuer name to be used when configuring PX-Security | "" | +| `security.auth.selfSigned.sharedSecret` | The Kubernetes secret name for retrieving and storing your shared secret. | "" | +| `resources` | Configure Portworx container usage such as memory and CPU usage.| {} | +| `customMetadata.annotations.pod.storage` | Custom annotations for Portworx pods | "" | +| `customMetadata.annotations.service.portworxApi` | Custom annotations for portwork-api service | "" | +| `customMetadata.annotations.service.portworxService` | Custom annotations for portwork-service | "" | +| `customMetadata.annotations.service.portworxKVDBService` | Custom annotations for portworx-kvdb-service | "" | +| `customMetadata.labels.service.portworxApi` | Custom labels for portwork-api service. Currently, custom labels are only supported on the portworx-api service | "" | +| `envVars` | semi-colon-separated list of environment variables that will be exported to portworx. (example: MYENV1=val1;MYENV2=val2) ( Depricated : use `envs` to set environment variables) | "none" | +| `envs` | Add environment variables to the Portworx container in all Kubernetes-supported formats | [] | +| `disableStorageClass` | Disable installation of default Portworx StorageClasses. | false | +| `stork.enabled` | [Storage Orchestration for Hyperconvergence](https://github.com/libopenstorage/stork). | true | +| `stork.storkVersion` | The version of stork | "" | +| `stork.args` | Pass arguments to Stork container | "" | +| `stork.volumes` | Add volumes to Stork container | [] | +| `stork.env` | List of Kubernetes like environment variables passed to Stork | [] | +| `customRegistryURL` | Custom Docker registry | "" | +| `registrySecret` | Registry secret | "" | +| `monitoring.prometheus.enabled` | Enable or disable Prometheus | false | +| `monitoring.prometheus.exportMetrics` | Expose the Portworx metrics to an external or operator deployed Prometheus | false | +| `monitoring.prometheus.alertManager` | Enable or disable alertmanager | false | +| `monitoring.prometheus.resources` | Configure stork container resources such memory and cpu | {} | +| `monitoring.prometheus.replicas` | Number of prometheus replicas that will be deployed | 1 | +| `monitoring.prometheus.retention` | Time period for which prometheus retains historical matrics | "24h" | +| `monitoring.prometheus.retentionSize` | Maximum amount of disk space that Prometheus can use to store historical metrics. Example: "10GiB","50MiB" | "" | +| `monitoring.prometheus.storage` | Storage type that Prometheus will use for storing data | {} | +| `monitoring.prometheus.volumes` | Additional volumes for the prometheus statefulSet | [] | +| `monitoring.prometheus.volumeMounts` | Additional VolumeMounts for the Prometheus StatefulSet | [] | +| `monitoring.prometheus.securityContext.runAsNonRoot` | Enable prometheus container run as a non-root user | false | +| `monitoring.telemetry` | Enable or disable telemetry | true | +| `monitoring.grafana` | Enable or disable grafana | false | +| `csi.enabled` | Enables CSI | true | +| `csi.topology.enabled` | Enable CSI topology feature gate | false | +| `csi.installSnapshotController` | Install CSI Snapshot Controller | false | +| `autopilot.enabled` | Enable AutoPilot | true | +| `autopilot.image` | Specify AutoPilot image | "" | +| `autopilot.lockImage` | Enables locking AutoPilot to the given image | false | +| `autopilot.args` | semicolon sperated list to Override or add new AutoPilot arguments | "" | +| `autopilot.env` | List of Kubernetes like environment variables passed to Autopilot | [] | +| `internalKVDB` | Internal KVDB store | true | +| `kvdbDevice` | specify a separate device to store KVDB data, only used when internalKVDB is set to true | "" | +| `kvdb.authSecretName` | Name of the secret for configuring secure KVDB (https://docs.portworx.com/portworx-enterprise/operations/kvdb-for-portworx/external-kvdb#secure-your-etcd-communication)| "none" | +| `etcd.credentials` | Username and password for etcd authentication in the form user:password (Depricated : use `kvdb.authSecretName`) | "none":"none" | +| `etcd.certPath` | Base path where the certificates are placed. (example: if the certificates ca,.crt and the .key are in /etc/pwx/etcdcerts the value should be provided as /etc/pwx/etcdcerts Refer: https://docs.portworx.com/scheduler/kubernetes/etcd-certs-using-secrets.html) (Depricated : use `kvdb.authSecretName`) | "none" | +| `etcd.ca` | Location of CA file for etcd authentication. Should be /path/to/server.ca (Depricated : use `kvdb.authSecretName`)| "none" | +| `etcd.cert` | Location of certificate for etcd authentication. Should be /path/to/server.crt (Depricated : use `kvdb.authSecretName`) | "none" | +| `etcd.key` | Location of certificate key for etcd authentication Should be /path/to/servery.key (Depricated : use `kvdb.authSecretName`)| "none" | +| `consul.token` | ACL token value used for Consul authentication. (example: 398073a8-5091-4d9c-871a-bbbeb030d1f6) (Depricated : use `kvdb.authSecretName`) (Depricated : use `kvdb.authSecretName`) | +| `volumes` | Specifies volumes for Portworx by defining a name, mount path, mount propagation (None, HostToContainer, Bidirectional), and whether the volume is read-only. For secrets, provide the secret name and map specific keys to paths. Supported volume types include Host, Secret, and ConfigMap | [] | +| `tolerations` | Specifies tolerations for scheduling Portworx pods. | [] | +| `nodeAffinity` | Specifies node affinity rules for Portworx pods. | {} | +| `nodesConfiguration` | Override certain cluster-level configurations for individual or groups of nodes, including network, storage, environment variables, and runtime options. | [] | +| `clusterToken.create` | Determines whether a cluster token should be created. | false | +| `clusterToken.secretName` | Name of the Kubernetes secret to be created for the cluster token. Requires clusterToken.create to be true. | "px-vol-encryption" | +| `clusterToken.serviceAccountName` | Service account name to use for the post-install hook to create the cluster token. | "px-create-cluster-token" | +| `deleteStrategy.type` | Optional: Specifies the delete strategy for the Portworx cluster. Valid values: Uninstall, UninstallAndWipe | "" | +| `updateStrategy.type` | Specifies the update strategy for the Portworx cluster. Supported values: RollingUpdate, OnDelete | "" | +| `updateStrategy.maxUnavailable` | Maximum number of nodes that can be unavailable during a rolling update | 1 | +| `updateStrategy.minReadySeconds` | Minimum number of seconds that a pod should be ready before the next batch of pods is updated during a rolling update | 1 | +| `updateStrategy.disruption.allow` | This field is used to enable or disable smart and parallel upgrade. Smart upgrade is disabled by default, Enable it by setting to false. we can use the `maxUnavailable` field to control the maximum number of Portworx nodes that can be upgraded at a time | None | +| `updateStrategy.autoUpdateComponents` | Specifies the update strategy for the component images. Valid values: None, Once, Always | None | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +## Cloud installs + +#### Installing on AKS + +Details are [here](https://docs.portworx.com/portworx-install-with-kubernetes/cloud/azure/aks/2-deploy-px/). + +> **Tip**: In this case the chart is located at `./helm/charts/portworx`, do change it as per your setup. +``` +helm install --name my-release --set imageVersion=1.2.12.0,etcdEndPoint=etcd:http://192.168.70.90:2379 ./helm/charts/portworx/ +``` + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, +``` +helm install --name my-release -f ./helm/charts/portworx/values.yaml ./helm/charts/portworx +``` +> **Tip**: You can use the default [values.yaml](values.yaml) and make changes as per your requirement + +#### Installing on IKS [ IBM Cloud ] + +Refer the IBM charts [here](https://github.com/IBM/charts/tree/master/community/portworx) + +> **Tip**: You will need to add the IBM charts repo with the repo path set to rawgithub +``` +helm repo add ibm-porx https://raw.githubusercontent.com/IBM/charts/master/repo/community +``` + +## Upgrading Portworx Install + +You can update the `imageVersion` value in the YAML file that specifies the values for the parameters used while installing the chart. +``` +helm upgrade my-release -f ./helm/charts/portworx/values.yaml ./helm/charts/portworx +``` + +Alternatively, you can also use the `--set` directive to do the same. For example, +``` +helm upgrade my-release --set imageVersion=,etcdEndPoint=,clusterName= -f ./helm/charts/portworx/values.yaml ./helm/charts/portworx +``` + +> **Tip**: You can check the upgrade with the new values took effect using. Check the reference for upgrade [here](https://v2.helm.sh/docs/using_helm/#helm-upgrade-and-helm-rollback-upgrading-a-release-and-recovering-on-failure) +``` +helm get values my-release +``` + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: +The chart would follow the process as outlined here. (https://docs.portworx.com/scheduler/kubernetes/install.html#uninstall) + +> **Tip** > The Portworx configuration files under `/etc/pwx/` directory are preserved, and will not be deleted. + +``` +helm delete my-release +``` +The command removes all the Kubernetes components associated with the chart and deletes the release. + +### Basic troubleshooting + +#### Helm install errors with `no available release name found` + +``` +helm install --dry-run --debug --set etcdEndPoint=etcd:http://192.168.70.90:2379,clusterName=$(uuidgen) ./helm/charts/portworx/ +[debug] Created tunnel using local port: '37304' +[debug] SERVER: "127.0.0.1:37304" +[debug] Original chart version: "" +[debug] CHART PATH: /root/helm/charts/portworx + +Error: no available release name found +``` +This most likely indicates that Tiller doesn't have the right RBAC permissions. +You can verify the tiller logs +``` +[storage/driver] 2018/02/07 06:00:13 get: failed to get "singing-bison.v1": configmaps "singing-bison.v1" is forbidden: User "system:serviceaccount:kube-system:default" cannot get configmaps in the namespace "kube-system" +[tiller] 2018/02/07 06:00:13 info: generated name singing-bison is taken. Searching again. +[tiller] 2018/02/07 06:00:13 warning: No available release names found after 5 tries +[tiller] 2018/02/07 06:00:13 failed install prepare step: no available release name found +``` + + + diff --git a/charts/portworx/portworx/5.1.1/app-readme.md b/charts/portworx/portworx/5.1.1/app-readme.md new file mode 100644 index 000000000..ae8bbe96f --- /dev/null +++ b/charts/portworx/portworx/5.1.1/app-readme.md @@ -0,0 +1,8 @@ +# Portworx + +[Portworx](https://portworx.com/) is a software defined storage overlay that allows you to + + * Run containerized stateful applications that are highly-available (HA) across multiple nodes, cloud instances, regions, data centers or even clouds + * Migrate workflows between multiple clusters running across same or hybrid clouds + * Run hyperconverged workloads where the data resides on the same host as the applications + * Have programmatic control on your storage resources \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/crds/core_v1_storagecluster_crd.yaml b/charts/portworx/portworx/5.1.1/crds/core_v1_storagecluster_crd.yaml new file mode 100644 index 000000000..1d42f9aaa --- /dev/null +++ b/charts/portworx/portworx/5.1.1/crds/core_v1_storagecluster_crd.yaml @@ -0,0 +1,4247 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: storageclusters.core.libopenstorage.org +spec: + group: core.libopenstorage.org + names: + kind: StorageCluster + listKind: StorageClusterList + plural: storageclusters + singular: storagecluster + shortNames: + - stc + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - description: The unique ID of the storage cluster + jsonPath: .status.clusterUid + name: Cluster UUID + type: string + - description: The status of the storage cluster + jsonPath: .status.phase + name: Status + type: string + - description: The version of the storage cluster + jsonPath: .spec.version + name: Version + type: string + - description: The age of the storage cluster + jsonPath: .metadata.creationTimestamp + name: Age + type: date + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + description: The desired behavior of the storage cluster. + properties: + priorityClassName: + type: string + description: Priority class name that the operator will pass to the portworx storage pods to be scheduled accordingly. + metadata: + type: object + description: Metadata contains metadata for different storage cluster components. + properties: + annotations: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + The annotations section of spec is a map of map to pass custom annotations to different + storage cluster components. The key specifies component in format of "kind/component", + e.g. "deployment/stork" to pass custom annotations to stork deployment. The value is a map of + string that contains custom annotation key and value pairs. + labels: + type: object + x-kubernetes-preserve-unknown-fields: true + description: >- + The labels section of spec is a map of map to pass custom labels to different storage cluster + components. The key specifies component in format of "kind/component", e.g. "service/portworx-api" + to pass custom labels to portworx-api service. The value is a map of string that contains custom + label key and value pairs. + resources: + type: object + description: Specifies the compute resource requirements for the storage pod. + properties: + requests: + type: object + description: Requested resources for the storage pod. + properties: + memory: + type: string + description: Specifies the memory that the Portworx container requests; for example, "4Gi" + cpu: + type: string + description: Specifies the cpu that the Portworx container requests; for example, `"4000m"` + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the storage pod. + properties: + memory: + type: string + description: Maximum memory for the storage pod. + cpu: + type: string + description: Maximum cpu for the storage pod. + image: + type: string + description: Specifies the Portworx monitor image. + version: + type: string + description: Version of the storage driver. This field is read-only. + imagePullPolicy: + type: string + description: Specifies the image pull policy for all the images deployed by the operator. The possible values can be `Always` or `IfNotPresent`. Default value is `Always`. + imagePullSecret: + type: string + description: If Portworx pulls images from a secure repository, you can use this field to pass it the name of the secret. Note that the secret should be in the same namespace as the `StorageCluster` object. + customImageRegistry: + type: string + description: The custom container registry server Portworx uses to fetch the Docker images. You may include the repository as well. + preserveFullCustomImageRegistry: + type: boolean + description: >- + Setting this to true this stops part of the image tag being swallowed when setting a + customImageRegistry with a / in it. When set to false using a customImageRegistry of + `example.io/public` and an image of `portworx/oci-monitor` the full image path is is + `example.io/public/oci-monitor`, setting to true gives you + `example.io/public/portworx/oci-monitor`. Defaults to false + secretsProvider: + type: string + description: The name of the secrets provider Portworx uses to store your credentials. To use features like cloud snapshots or volume encryption, you must configure a secret store provider. Refer to the [Secret store management page](/portworx-enterprise/operations/key-management)for more details. + startPort: + type: integer + format: int32 + minimum: 0 + description: Start port is the starting port in the range of ports used by the cluster. + autoUpdateComponents: + type: string + description: Indicates the update strategy for the component images (such as Stork, Autopilot, Prometheus, and so on). Portworx supports the following auto update strategies for the component images:
  • `None`- Updates the component images only when the Portworx image changes in `StorageCluster.spec.image`.
  • `Once`- Updates the component images once even if the Portworx image does not change. This is useful when the component images on the manifest server change due to bug fixes.
  • `Always`- Regularly checks for the updates on the manifest server, and updates the component images if required.
+ updateStrategy: + type: object + description: An update strategy to replace existing StorageCluster pods with new pods. + properties: + type: + type: string + description: Indicates the update strategy. Currently, Portworx supports the following update strategies- `RollingUpdate` and `OnDelete`. Default value is `RollingUpdate`. + enum: + - RollingUpdate + - OnDelete + rollingUpdate: + type: object + description: Spec to control the desired behavior of storage cluster rolling update. + properties: + minReadySeconds: + description: During rolling updates, this flag will wait for all pods to be ready for at least `minReadySeconds` before updating the next batch of pods, where the size of the pod batch is specified through the `spec.updateStrategy.rollingUpdate.maxUnavailable` flag. Default value is `1`. + format: int32 + type: integer + maxUnavailable: + x-kubernetes-int-or-string: true + description: >- + The maximum number of StorageCluster pods that can be unavailable + during the update. Value can be an absolute number (ex: 5) or a percentage of + total number of StorageCluster pods at the start of the update (ex: 10%). + Absolute number is calculated from percentage by rounding up. This cannot be 0. + Default value is 1. Example: when this is set to 30%, at most 30% of the total + number of nodes that should be running the storage pod can have their pods + stopped for an update at any given time. The update starts by stopping at most + 30% of those StorageCluster pods and then brings up new StorageCluster pods in + their place. Once the new pods are available, it then proceeds onto other + StorageCluster pods, thus ensuring that at least 70% of original number of + StorageCluster pods are available at all times during the update. + disruption: + type: object + description: >- + The default behavior is non-disruptive upgrades. This setting disables the default + non-disruptive upgrades and reverts to the previous behavior of upgrading nodes in + parallel without worrying about disruption. + properties: + allow: + type: boolean + description: Flag indicates whether updates are non-disruptive or disruptive. + deleteStrategy: + type: object + description: Indicates what happens when the Portworx `StorageCluster` object is deleted. By default, there is no delete strategy, which means only the Kubernetes components deployed by the operator are removed. The Portworx `systemd` service continues to run, and the Kubernetes applications using the Portworx volumes are not affected. Portworx supports the following delete strategies.
- `Uninstall` - Removes all Portworx components from the system and leaves the devices and KVDB intact.
- `UninstallAndWipe` - Removes all Portworx components from the system and wipes the devices and metadata from KVDB. + properties: + type: + type: string + description: Type of storage cluster delete. Can be Uninstall or UninstallAndWipe. + There is no default delete strategy. When no delete strategy only objects managed + by the StorageCluster controller and owned by the StorageCluster object are deleted. + The storage driver will be left in a state where it will not be managed by any object. + Uninstall strategy ensures that the cluster is completely uninstalled even from the + storage driver perspective. UninstallAndWipe strategy ensures that the cluster is + completely uninstalled as well as the storage devices and metadata are wiped for + reuse. This may result in data loss. + enum: + - Uninstall + - UninstallAndWipe + revisionHistoryLimit: + type: integer + format: int32 + description: The number of old history to retain to allow rollback. This is a pointer + to distinguish between an explicit zero and not specified. Defaults to 10. + featureGates: + type: object + x-kubernetes-preserve-unknown-fields: true + description: A collection of key-value pairs specifying which Portworx features should be enabled or disabled. + runtimeOptions: + type: object + x-kubernetes-preserve-unknown-fields: true + description: A collection of key-value pairs that overwrites the runtime options. + placement: + type: object + description: Describes placement configuration for the storage cluster pods. + properties: + nodeAffinity: + type: object + description: Use this field to restrict Portwox on certain nodes. It works similarly to the [Kubernetes node affinity](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L2692) feature. + properties: + requiredDuringSchedulingIgnoredDuringExecution: + type: object + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - nodeSelectorTerms + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + properties: + weight: + type: integer + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - preference + - weight + tolerations: + type: array + description: Specifies a list of tolerations that will be applied to Portworx pods so that they can run on nodes with matching taints. + items: + type: object + properties: + effect: + type: string + description: Effect indicates the taint effect to match. Empty means match + all taint effects. When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + key: + type: string + description: Key is the taint key that the toleration applies to. Empty means + match all taint keys. If the key is empty, operator must be Exists; this + combination means to match all values and all keys. + operator: + type: string + description: "Operator represents a key's relationship to the value. Valid + operators are Exists and Equal. Defaults to Equal. Exists is equivalent to + wildcard for value, so that a pod can tolerate all taints of a particular category." + value: + type: string + description: Value is the taint value the toleration matches to. If the operator + is Exists, the value should be empty, otherwise just a regular string. + tolerationSeconds: + type: integer + description: TolerationSeconds represents the period of time the toleration + (which must be of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + kvdb: + type: object + description: Details of KVDB that the storage driver will use. + properties: + internal: + type: boolean + description: Specifies if Portworx starts with the [internal KVDB](/portworx-enterprise/operations/kvdb-for-portworx/internal-kvdb). Default value is `true`. + endpoints: + type: array + description: A list of endpoints for your external key-value database like ETCD. This field takes precedence over the spec.kvdb.internal field. That is, if you specify the endpoints, Portworx ignores the spec.kvdb.internal field and it uses the external KVDB.A list of endpoints for your external key-value database like ETCD. This field takes precedence over the `spec.kvdb.internal field`. That is, if you specify the endpoints, Portworx ignores the `spec.kvdb.internal` field and it uses the external KVDB. + items: + type: string + authSecret: + type: string + description: Indicates the name of the secret Portworx uses to authenticate against your KVDB. The secret must be placed in the same namespace as the `StorageCluster` object. The secret should provide the following information.
- `username` (optional)
- `password` (optional)
- `kvdb-ca.crt` (the CA certificate)
- `kvdb.key` (certificate key)
- `kvdb.crt` (etcd certificate)
- `acl-token` (optional)
For example, create a directory called etcd-secrets, copy the files into it and create a secret with `kubectl -n kube-system create secret generic px-kvdb-auth --from-file=etcd-secrets/` + resources: + type: object + description: Specifies the compute resource requirements for the kvdb pod. + properties: + requests: + type: object + description: Requested resources for the kvdb pod. + properties: + memory: + type: string + description: Requested memory for the kvdb pod. + cpu: + type: string + description: Requested cpu for the kvdb pod. + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the kvdb pod. + properties: + memory: + type: string + description: Maximum memory for the kvdb pod. + cpu: + type: string + description: Maximum cpu for the kvdb pod. + storage: + type: object + description: Specify storage configuration for the selected nodes, similar to the one [specified at cluster level](#storage-configuration). If some of the config is left empty, the cluster level storage values are passed to the nodes. If you don't want to use a cluster level value and set the field to empty, then explicitly set an empty value for it so no value is passed to the nodes. For instance, set `spec.nodes[0].storage.kvdbDevice:""`, to prevent using the KVDB device for the selected nodes. + properties: + useAll: + type: boolean + description: If set to `true`, Portworx uses all available, unformatted, and unpartitioned devices. Default value is `true`. + useAllWithPartitions: + type: boolean + description: If set to `true`, Portworx uses all the available and unformatted devices. Default value is `false`. + forceUseDisks: + type: boolean + description: If set to `true`, Portworx uses a device even if there's a file system on it. Note that Portworx may wipe the drive before using it. Default value is `false`. + devices: + type: array + description: Specifies the list of devices Portworx should use. + items: + type: string + cacheDevices: + type: array + description: Specifies the list of cache devices Portworx should use. + items: + type: string + journalDevice: + type: string + description: Specifies the device Portworx uses for journaling. + systemMetadataDevice: + type: string + description: Indicates the device Portworx uses to store metadata. For better performance, specify a system metadata device when using Portworx with the internal KVDB. + kvdbDevice: + type: string + description: Specifies the device Portworx uses to store internal KVDB data. + cloudStorage: + type: object + description: Details of storage used in cloud environment. + properties: + provider: + type: string + description: Specifies the cloud provider name. Example - azure, aws, gce, vsphere. + maxStorageNodes: + type: integer + format: int32 + minimum: 0 + description: Specifies the maximum number of storage nodes. If this number is reached, and a new node is added, Portworx doesn't provision drives for the new node. Instead, Portworx starts the node as a compute-only node. As a best practice, it is recommended to use the `maxStorageNodesPerZone` field. + maxStorageNodesPerZone: + type: integer + format: int32 + minimum: 0 + description: Indicates the maximum number of storage nodes per zone. If this number is reached, and a new node is added to the zone, Portworx doesn't provision drives for the new node. Instead, Portworx starts the node as a compute-only node. + maxStorageNodesPerZonePerNodeGroup: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes in every zone in every node group that will have storage + in the cluster. + nodePoolLabel: + type: string + description: Kubernetes node label key with which nodes are grouped into node pools + for storage distribution in cloud environment. + deviceSpecs: + type: array + description: List of storage device specs. A cloud storage device will be created + for every spec in the list. The specs will be applied to all nodes in the cluster + up to spec.cloudStorage.maxStorageNodes or spec.cloudStorage.maxStorageNodesPerZone + or spec.cloudStorage.maxStorageNodesPerZonePerNodeGroup. + This will be ignored if spec.cloudStorage.capacitySpecs is present. + items: + type: string + capacitySpecs: + type: array + description: List of cluster wide storage types and their capacities. A single + capacity spec identifies a storage pool with a set of minimum requested IOPS + and size. Based on the cloud provider, the total storage capacity will get + divided amongst the nodes. The nodes bearing storage themselves will get + uniformly distributed across all the zones. + items: + type: object + properties: + minIOPS: + type: integer + format: int64 + minimum: 0 + description: Minimum IOPS expected from the cloud drive. + minCapacityInGiB: + type: integer + format: int64 + minimum: 0 + description: Minimum capacity for this storage cluster. The total capacity + of devices created by this capacity spec should not be less than this + number for the entire cluster. + maxCapacityInGiB: + type: integer + format: int64 + minimum: 0 + description: Maximum capacity for this storage cluster. The total capacity + of devices created by this capacity spec should not be greater than this + number for the entire cluster. + options: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Additional options required to provision the drive in cloud. + journalDeviceSpec: + type: string + description: Specifies the cloud device Portworx uses for journaling. + systemMetadataDeviceSpec: + type: string + description: Indicates the cloud device Portworx uses for metadata. For performance, specify a system metadata device when using Portworx with the internal KVDB. + kvdbDeviceSpec: + type: string + description: Specifies the cloud device Portworx uses for an internal KVDB. + network: + type: object + description: Specify network configuration for the selected nodes, similar to the one [specified at cluster level](#network-configuration). If this network configuration is empty, then cluster level values are used. + properties: + dataInterface: + type: string + description: Specifies the network interface Portworx uses for data traffic. + mgmtInterface: + type: string + description: Indicates the network interface Portworx uses for control plane traffic. + stork: + type: object + description: Contains STORK related spec. + properties: + enabled: + type: boolean + description: Enables or disables Stork at any given time. Default value is `true`. + lockImage: + type: boolean + description: Enables locking Stork to the given image. When set to false, the Portworx Operator will overwrite the Stork image to a recommended image for given Portworx version. Default value is `false`. + image: + type: string + description: Specifies the Stork image. + hostNetwork: + type: boolean + description: Flag indicating if Stork pods should run in host network. + args: + type: object + x-kubernetes-preserve-unknown-fields: true + description: A collection of key-value pairs that overrides the default Stork arguments or adds new arguments. + env: + type: array + description: A list of [Kubernetes like environment variables](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L1826) passed to Stork. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + description: A list of volumes passed to Stork pods. The schema is similar to the [top-level volumes](#volume-configuration). + items: + type: object + properties: + name: + type: string + readOnly: + type: boolean + mountPath: + type: string + mountPropagation: + type: string + hostPath: + type: object + properties: + path: + type: string + type: + type: string + secret: + type: object + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + resources: + type: object + description: Specifies the resource requirements for stork and stork scheduler. + properties: + requests: + type: object + description: Requested resources. + properties: + memory: + type: string + description: Requested memory. + cpu: + type: string + description: Requested cpu. + limits: + type: object + description: Resource limit. + properties: + memory: + type: string + description: Memory limit. + cpu: + type: string + description: CPU limit. + userInterface: + type: object + description: Contains spec of a user interface for the storage driver. + properties: + enabled: + type: boolean + description: Flag indicating whether the user interface needs to be enabled. + lockImage: + type: boolean + description: Flag indicating if the user interface image needs to be locked to the given + image. If the image is not locked, it can be updated by the storage driver during upgrades. + image: + type: string + description: Docker image of the user interface container. + env: + type: array + description: List of environment variables used by the UI components. This is an array + of Kubernetes EnvVar where the value can be given directly or from a source like field, + config map or secret. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + autopilot: + type: object + description: Contains spec of autopilot component for storage driver. + properties: + enabled: + type: boolean + description: Enables or disables Autopilot at any given time. Default value is `false`. + lockImage: + type: boolean + description: Enables locking Autopilot to the given image. When set to false, the Portworx Operator will overwrite the Autopilot image to a recommended image for given Portworx version. Default value is `false`. + image: + type: string + description: Specifies the Autopilot image. + args: + type: object + x-kubernetes-preserve-unknown-fields: true + description: A collection of key-value pairs that overrides the default Autopilot arguments or adds new arguments. + env: + type: array + description: A list of [Kubernetes like environment variables](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L1826) passed to Autopilot. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + description: A list of volumes passed to Autopilot pods. The schema is similar to the [top-level volumes](#volume-configuration). + items: + type: object + properties: + name: + type: string + readOnly: + type: boolean + mountPath: + type: string + mountPropagation: + type: string + hostPath: + type: object + properties: + path: + type: string + type: + type: string + secret: + type: object + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + providers: + type: array + description: List of data providers for Autopilot. + items: + type: object + properties: + name: + type: string + description: Unique name of the data provider. + type: + type: string + description: Type of the data provider. For instance - prometheus + params: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Map of key-value params for the provider. + resources: + type: object + description: Specifies the resource requirements for the autopilot pod. + properties: + requests: + type: object + description: Requested resources. + properties: + memory: + type: string + description: Requested memory. + cpu: + type: string + description: Requested cpu. + limits: + type: object + description: Resource limit. + properties: + memory: + type: string + description: Memory limit. + cpu: + type: string + description: CPU limit. + monitoring: + type: object + description: Contains monitoring configuration for the storage cluster. + properties: + enableMetrics: + type: boolean + description: "If this flag is enabled it will expose the storage cluster metrics to external + monitoring solutions like Prometheus. DEPRECATED - use prometheus.exportMetrics instead" + prometheus: + type: object + description: Contains configuration of Prometheus to monitor the storage cluster. + properties: + resources: + type: object + description: Provides the ability to configure Prometheus resource usage such as memory and CPU usage. Default limits - `CPU 1, memory 800M, and ephemeral storage 5G` + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + securityContext: + description: 'Security options the pod should run with. More + info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by + the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent to + root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to + use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root + filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a + non-root user. If true, the Kubelet will validate the + image at runtime to ensure that it does not run as UID + 0 (root) and fail to start the container if it does. If + unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both + SecurityContext and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a + random SELinux context for each container. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: string + type: object + type: object + exportMetrics: + type: boolean + description: Expose the Portworx metrics to an external or operator deployed Prometheus. Default value is `false`. + enabled: + type: boolean + description: Enables or disables a Prometheus cluster. Default value is `false`. + remoteWriteEndpoint: + type: string + description: Specifies the remote write endpoint for Prometheus. + alertManager: + type: object + description: Contains configuration of AlertManager for the storage cluster. + properties: + enabled: + type: boolean + description: Enables or disables Prometheus Alertmanager. + replicas: + description: Specifies the number of Prometheus replicas that will be deployed. Default value is `1`. + format: int32 + type: integer + retention: + description: Specifies the time period for which Prometheus retains historical metrics. Default + is '24h' if retentionSize is not set, and must match the regular + expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes + hours days weeks years). + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + retentionSize: + description: Maximum amount of disk space used by blocks. + pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ + type: string + storage: + description: Specifies the storage type that Prometheus will use for storing data. If you set the storage type to PVCs, do not set the `runAsGroup` or `fsGroup` option for the `spec.monitoring.prometheus.securityContext` flag. + properties: + disableMountSubPath: + description: 'Deprecated: subPath usage will be disabled by default + in a future release, this option will become unnecessary. DisableMountSubPath + allows to remove any subPath usage in volume mounts.' + type: boolean + emptyDir: + description: 'EmptyDirVolumeSource to be used by the StatefulSet. + If specified, used in place of any volumeClaimTemplate. More + info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which means + to use the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage on memory + medium EmptyDir would be the minimum value between the SizeLimit + specified here and the sum of memory limits of all containers + in a pod. The default is nil which means that the limit + is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: 'EphemeralVolumeSource to be used by the StatefulSet. + This is a beta field in k8s 1.21, for lower versions, starting + with k8s 1.19, it requires enabling the GenericEphemeralVolume + feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes' + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC will + be deleted together with the pod. The name of the PVC will + be `-` where `` is the + name from the `PodSpec.Volumes` array entry. Pod validation + will reject the pod if the concatenated name is not valid + for a PVC (for example, too long). \n An existing PVC with + that name that is not owned by the pod will *not* be used + for the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated PVC + is removed. If such a pre-created PVC is meant to be used + by the pod, the PVC has to updated with an owner reference + to the pod once the pod exists. Normally this should not + be necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no changes + will be made by Kubernetes to the PVC after it has been + created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified data + source. When the AnyVolumeDataSource feature gate + is enabled, dataSource contents will be copied to + dataSourceRef, and dataSourceRef contents will be + copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a + non-empty API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, when namespace isn''t specified + in dataSourceRef, both fields (dataSource and dataSourceRef) + will be set to the same value automatically if one + of them is empty and the other is non-empty. When + namespace is specified in dataSourceRef, dataSource + isn''t set to the same value and must be empty. + There are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is + specified. * While dataSource only allows local + objects, dataSourceRef allows objects in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled. (Alpha) Using the namespace + field of dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace + is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept the + reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires the + CrossNamespaceVolumeDataSource feature gate + to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used + by this container. \n This is an alpha field + and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of + one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes + that resource available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the + StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is + implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + volumeClaimTemplate: + description: A PVC spec to be used by the StatefulSet. The easiest + way to use a volume that cannot be automatically provisioned + (for whatever reason) is to use a label selector alongside manually + created PersistentVolumes. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: EmbeddedMetadata contains metadata relevant to + an EmbeddedResource. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value + map stored with a resource that may be set by external + tools to store and retrieve arbitrary metadata. They + are not queryable and should be preserved when modifying + objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be + used to organize and categorize (scope and select) objects. + May match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + name: + description: 'Name must be unique within a namespace. + Is required when creating resources, although some resources + may allow a client to request the generation of an appropriate + name automatically. Name is primarily intended for creation + idempotence and configuration definition. Cannot be + updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + type: object + spec: + description: 'Spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. When the AnyVolumeDataSource + feature gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, then + dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty + API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic provisioner. + This field will replace the functionality of the dataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + when namespace isn''t specified in dataSourceRef, both + fields (dataSource and dataSourceRef) will be set to + the same value automatically if one of them is empty + and the other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the same + value and must be empty. There are three important differences + between dataSource and dataSourceRef: * While dataSource + only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed values + (dropping them), dataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + * While dataSource only allows local objects, dataSourceRef + allows objects in any namespaces. (Beta) Using this + field requires the AnyVolumeDataSource feature gate + to be enabled. (Alpha) Using the namespace field of + dataSourceRef requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace is specified, + a gateway.networking.k8s.io/ReferenceGrant object + is required in the referent namespace to allow that + namespace's owner to accept the reference. See the + ReferenceGrant documentation for details. (Alpha) + This field requires the CrossNamespaceVolumeDataSource + feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used by + this container. \n This is an alpha field and requires + enabling the DynamicResourceAllocation feature gate. + \n This field is immutable." + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one + entry in pod.spec.resourceClaims of the Pod + where this field is used. It makes that resource + available inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes to + consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is + required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources is the storage resource + within AllocatedResources tracks the capacity allocated + to a PVC. It may be larger than the actual capacity + when a volume expansion operation is requested. For + storage quota, the larger value from allocatedResources + and PVC.spec.resources is used. If allocatedResources + is not set, PVC.spec.resources alone is used for quota + calculation. If a volume expansion capacity request + is lowered, allocatedResources is only lowered if there + are no expansion operations in progress and if the actual + volume capacity is equal or lower than the requested + capacity. This is an alpha field and requires enabling + RecoverVolumeExpansionFailure feature. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources + of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of persistent + volume claim. If underlying persistent volume is being + resized then the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed + the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the + condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message + indicating details about last transition. + type: string + reason: + description: reason is a unique, this should be + a short, machine understandable string that gives + the reason for condition's last transition. If + it reports "ResizeStarted" that means the underlying + persistent volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + resizeStatus: + description: resizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by resize + controller or kubelet. This is an alpha field and requires + enabling RecoverVolumeExpansionFailure feature. + type: string + type: object + type: object + type: object + volumes: + description: Specifies additional volumes to the output Prometheus StatefulSet. These specified volumes will be appended to other volumes generated as a result of the `spec.monitoring.prometheus.storage` spec. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume that + you want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent disk + resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must be + a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to + be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used to + set permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair in + the Data field of the referenced ConfigMap will be projected + into the volume as a file whose name is the key and content + is the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in + the ConfigMap, the volume setup will error unless it is + marked optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, + the volume defaultMode will be used. This might + be in conflict with other options that affect the + file mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which means + to use the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage on memory + medium EmptyDir would be the minimum value between the + SizeLimit specified here and the sum of memory limits + of all containers in a pod. The default is nil which means + that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use this + if: a) the volume is only needed while the pod runs, b) features + of normal volumes like restoring from snapshot or capacity + tracking are needed, c) the storage driver is specified through + a storage class, and d) the storage driver supports dynamic + volume provisioning through a PersistentVolumeClaim (see EphemeralVolumeSource + for more information on the connection between this volume + type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n Use + CSI for light-weight local ephemeral volumes if the CSI driver + is meant to be used that way - see the documentation of the + driver for more information. \n A pod can use both types of + ephemeral volumes and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of the + PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). \n + An existing PVC with that name that is not owned by the + pod will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC + is meant to be used by the pod, the PVC has to updated + with an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may be useful + when manually reconstructing a broken cluster. \n This + field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, must + not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No other + fields are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be copied + to dataSourceRef, and dataSourceRef contents will + be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, if + a non-empty volume is desired. This may be any + object from a non-empty API group (non core object) + or a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed + if the type of the specified object matches some + installed volume populator or dynamic provisioner. + This field will replace the functionality of the + dataSource field and as such if both fields are + non-empty, they must have the same value. For + backwards compatibility, when namespace isn''t + specified in dataSourceRef, both fields (dataSource + and dataSourceRef) will be set to the same value + automatically if one of them is empty and the + other is non-empty. When namespace is specified + in dataSourceRef, dataSource isn''t set to the + same value and must be empty. There are three + important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types + of objects, dataSourceRef allows any non-core + object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping + them), dataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + * While dataSource only allows local objects, + dataSourceRef allows objects in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled. (Alpha) Using the + namespace field of dataSourceRef requires the + CrossNamespaceVolumeDataSource feature gate to + be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: Namespace is the namespace of resource + being referenced Note that when a namespace + is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires the + CrossNamespaceVolumeDataSource feature gate + to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are used + by this container. \n This is an alpha field + and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If Requests + is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'storageClassName is the name of the + StorageClass required by the claim. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. TODO: how do we prevent errors in the + filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends + on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if no + secret object is specified. If the secret object contains + more than one secret, all secrets are passed to the plugin + scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored as + metadata -> name on the dataset for Flocker should be + considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume that + you want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume that + you want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource in + GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir into + the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, the + volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that is + attached to a kubelet''s host machine and then exposed to + the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that uses + an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. The + portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: targetPortal is iSCSI Target Portal. The Portal + is either an IP or ip_addr:port if the port is other than + default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and unique + within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the host that shares + a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export to + be mounted with read-only permissions. Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents a + reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting in + VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Directories within the path are + not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along with + other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified which + is not present in the ConfigMap, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 and + 0777 or a decimal value between 0 and + 511. YAML accepts both octal and decimal + values, JSON requires decimal values for + mode bits. If not specified, the volume + defaultMode will be used. This might be + in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be + an absolute path. May not contain the + path element '..'. May not start with + the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to + set permissions on this file, must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu + and requests.memory) are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified which + is not present in the Secret, the volume setup + will error unless it is marked optional. Paths + must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 and + 0777 or a decimal value between 0 and + 511. YAML accepts both octal and decimal + values, JSON requires decimal values for + mode bits. If not specified, the volume + defaultMode will be used. This might be + in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of + the file to map the key to. May not be + an absolute path. May not contain the + path element '..'. May not start with + the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must identify + itself with an identifier specified in the audience + of the token, and otherwise should reject the + token. The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, the + kubelet volume plugin will proactively rotate + the service account token. The kubelet will + start trying to rotate the token if the token + is older than 80 percent of its time to live + or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is no + group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple Quobyte + Registry services specified as a string as host:port pair + (multiple entries are separated with commas) which acts + as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in the + Backend Used with dynamically provisioned Quobyte volumes, + value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to serivceaccount + user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication secret + for RBDUser. If provided overrides keyring. Default is + nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'user is the rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not provided, + Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage for + a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used to + set permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair in + the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and content + is the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in + the Secret, the volume setup will error unless it is marked + optional. Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in the + pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for obtaining + the StorageOS API credentials. If not specified, default + values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: volumeName is the human-readable name of the + StorageOS volume. Volume names are only unique within + a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name to + override the default behaviour. Set to "default" if you + are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must be + a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + volumeMounts: + description: Specifies additional VolumeMounts on the output Prometheus StatefulSet definition. These specified VolumeMounts specified will be appended to other VolumeMounts in the Prometheus container generated as a result of `spec.monitoring.prometheus.storage` spec. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume should + be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are propagated + from the host to container and the other way around. When + not set, MountPropagationNone is used. This field is beta + in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which the + container's volume should be mounted. Behaves similarly to + SubPath but environment variable references $(VAR_NAME) are + expanded using the container's environment. Defaults to "" + (volume's root). SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + grafana: + type: object + description: Contains configuration of Grafana to monitor the storage cluster. + properties: + enabled: + type: boolean + description: Enables or disables a Grafana instance with Portworx dashboards. Default value is `false`. + telemetry: + type: object + description: Contains telemetry configuration for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicates if telemetry component needs to be enabled + image: + type: string + description: Docker image of the telemetry container. + logUploaderImage: + type: string + description: Docker image of the log-upload-service container. + security: + type: object + description: An object for specifying PX-Security configurations. Refer to the [Operator Security page](/portworx-enterprise/operations/operate-kubernetes/authorization/enable) for more details. + properties: + enabled: + type: boolean + description: Enables or disables Security at any given time. Default value is `false`. + auth: + type: object + description: Authorization configurations for a RBAC enabled storage cluster + properties: + guestAccess: + type: string + description: Determines how the guest role will be updated in your cluster. The options are Enabled, Disabled, or Managed. Managed will cause the operator to ignore updating the system.guest role. Enabled and Disabled will allow or disable guest role access, respectively. Default value is `Enabled`. + selfSigned: + type: object + description: Configuration for self signed authentication + properties: + issuer: + type: string + description: The issuer name to be used when configuring PX-Security. This field maps to the `PORTWORX_AUTH_JWT_ISSUER` environment variable in the Portworx Daemonset. Default value is `operator.portworx.io`. + tokenLifetime: + type: string + description: The length operator-generated tokens will be alive until being refreshed. Default value is `24h`. + sharedSecret: + type: string + description: The Kubernetes secret name for retrieving and storing your shared secret. This field can be used to add a pre-existing shared secret or for customizing which secret name the operator will use for its auto-generated shared secret key. This field maps to the `PORTWORX_AUTH_JWT_SHAREDSECRET` environment variable in the Portworx Daemonset. Default value is `px-shared-secret`. + csi: + type: object + description: Contains CSI configuration for the storage cluster. + properties: + enabled: + type: boolean + description: Flag indicating whether CSI needs to be installed for the storage cluster. Default value is `true`. + installSnapshotController: + type: boolean + description: Flag indicating whether CSI Snapshot Controller needs to be installed for the storage cluster. Default value is `false`. + topology: + type: object + description: Contains CSI topology configurations. + properties: + enabled: + type: boolean + description: Flag indicating whether CSI topology feature gate is enabled. + placement: + type: object + description: Describes placement configuration for the CSI sidecar pods. + properties: + nodeAffinity: + type: object + description: Describes node affinity scheduling rules for the CSI sidecar pods. + This is exactly the same object as Kubernetes node affinity for pods. + properties: + requiredDuringSchedulingIgnoredDuringExecution: + type: object + properties: + nodeSelectorTerms: + type: array + items: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - nodeSelectorTerms + preferredDuringSchedulingIgnoredDuringExecution: + type: array + items: + type: object + properties: + weight: + type: integer + preference: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + matchFields: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + values: + type: array + items: + type: string + required: + - key + - operator + required: + - preference + - weight + tolerations: + type: array + description: Tolerations for the CSI sidecar pods. + The pod with this toleration attached will tolerate any taint that matches the + triple using the matching operator . + items: + type: object + properties: + effect: + type: string + description: Effect indicates the taint effect to match. Empty means match + all taint effects. When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + key: + type: string + description: Key is the taint key that the toleration applies to. Empty means + match all taint keys. If the key is empty, operator must be Exists; this + combination means to match all values and all keys. + operator: + type: string + description: "Operator represents a key's relationship to the value. Valid + operators are Exists and Equal. Defaults to Equal. Exists is equivalent to + wildcard for value, so that a pod can tolerate all taints of a particular category." + value: + type: string + description: Value is the taint value the toleration matches to. If the operator + is Exists, the value should be empty, otherwise just a regular string. + tolerationSeconds: + type: integer + description: TolerationSeconds represents the period of time the toleration + (which must be of effect NoExecute, otherwise this field is ignored) tolerates + the taint. By default, it is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will be treated as 0 (evict + immediately) by the system. + nodeDriverRegistrar: + type: object + description: Defines configuration for the CSI node driver registrar. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi node driver registar container. + properties: + requests: + type: object + description: Requested resources for the csi node driver registrar container. + properties: + memory: + type: string + description: Requested memory for the csi node driver registrar container. + cpu: + type: string + description: Requested cpu for the csi node driver registrar container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi node driver registrar container. + properties: + memory: + type: string + description: Maximum memory for the csi node driver registrar container. + cpu: + type: string + description: Maximum cpu for the csi node driver registrar container. + externalProvisioner: + type: object + description: Defines configuration for the CSI provisioner. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi provisioner container. + properties: + requests: + type: object + description: Requested resources for the csi provisioner container. + properties: + memory: + type: string + description: Requested memory for the csi provisioner container. + cpu: + type: string + description: Requested cpu for the csi provisioner container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi provisioner container. + properties: + memory: + type: string + description: Maximum memory for the csi provisioner container. + cpu: + type: string + description: Maximum cpu for the csi provisioner container. + snapshotter: + type: object + description: Defines configuration for the CSI snapshotter. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi snapshotter container. + properties: + requests: + type: object + description: Requested resources for the csi snapshotter container. + properties: + memory: + type: string + description: Requested memory for the csi snapshotter container. + cpu: + type: string + description: Requested cpu for the csi snapshotter container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi snapshotter container. + properties: + memory: + type: string + description: Maximum memory for the csi snapshotter container. + cpu: + type: string + description: Maximum cpu for the csi snapshotter container. + resizer: + type: object + description: Defines configuration for the CSI resizer. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi resizer container. + properties: + requests: + type: object + description: Requested resources for the csi resizer container. + properties: + memory: + type: string + description: Requested memory for the csi resizer container. + cpu: + type: string + description: Requested cpu for the csi resizer container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi resizer container. + properties: + memory: + type: string + description: Maximum memory for the csi resizer container. + cpu: + type: string + description: Maximum cpu for the csi resizer container. + snapshotController: + type: object + description: Defines configuration for the CSI snapshot controller. + properties: + resources: + type: object + description: Specifies the resource requirements for the csi snapshot controller container. + properties: + requests: + type: object + description: Requested resources for the csi snapshot controller container. + properties: + memory: + type: string + description: Requested memory for the csi snapshot controller container. + cpu: + type: string + description: Requested cpu for the csi snapshot controller container. + limits: + type: object + description: Limits describes the maximum amount of resources allowed for the csi snapshot controller container. + properties: + memory: + type: string + description: Maximum memory for the csi snapshot controller container. + cpu: + type: string + description: Maximum cpu for the csi snapshot controller container. + portworxApi: + type: object + description: Contains a spec to configure the Portworx API component. + properties: + resources: + type: object + description: Specifies the compute resource requirements for the portworx api container. + properties: + requests: + type: object + description: Requested resources for the portworx api container. + properties: + memory: + type: string + description: Requested memory for the portworx api container. + cpu: + type: string + description: Requested cpu for the portworx api container. + limits: + type: object + description: Limits describes the maximum amount of compute resources allowed for the portworx api container. + properties: + memory: + type: string + description: Maximum memory for the portworx api container. + cpu: + type: string + description: Maximum cpu for the portworx api container. + env: + type: array + description: A list of [Kubernetes like environment variables](https://github.com/kubernetes/api/blob/master/core/v1/types.go#L1826). Similar to how environment variables are provided in Kubernetes, you can directly provide values to Portworx or import them from a source like a Secret, ConfigMap, etc. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + volumes: + type: array + items: + type: object + properties: + name: + type: string + description: Unique name for the volume. + readOnly: + type: boolean + description: Volume is mounted read-only if true, read-write otherwise. Default value is `false`. + mountPath: + type: string + description: Path within the Portworx container at which the volume should be mounted. Must not contain the ':' character. + mountPropagation: + type: string + description: Determines how mounts are propagated from the host to container and the other way around. + hostPath: + type: object + description: Specifies the location and type of the mounted volume. This is similar to the VolumeSource schema of a Kubernetes pod volume. + properties: + path: + type: string + type: + type: string + secret: + type: object + description: Specifies the location and type of the mounted volume. This is similar to the VolumeSource schema of a Kubernetes pod volume. + properties: + secretName: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + description: Specifies the location and type of the mounted volume. This is similar to the VolumeSource schema of a Kubernetes pod volume. + properties: + name: + type: string + defaultMode: + type: integer + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + projected: + type: object + properties: + defaultMode: + type: integer + sources: + type: array + items: + type: object + properties: + secret: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + configMap: + type: object + properties: + name: + type: string + optional: + type: boolean + items: + type: array + items: + type: object + properties: + key: + type: string + path: + type: string + mode: + type: integer + nodes: + type: array + description: Node level configurations that will override the configuration at cluster level. + These configurations can be for individual nodes or can be grouped to override configuration + of multiple nodes based on label selectors. + items: + type: object + properties: + selector: + type: object + description: Selector for the node(s) to which the configuration in this section will be applied. + properties: + nodeName: + type: string + description: Name of the node to which this configuration will be applied. Node name takes precedence over `selector.labelSelector`. + labelSelector: + type: object + description: The [Kubernetes style label selector](https://github.com/kubernetes/apimachinery/blob/master/pkg/apis/meta/v1/types.go) for nodes to which this configuration will be applied. + properties: + matchLabels: + type: object + x-kubernetes-preserve-unknown-fields: true + description: It is a map of key-value pairs. A single key-value in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + matchExpressions: + type: array + description: It is a list of label selector requirements. The requirements are ANDed. + items: + type: object + properties: + key: + type: string + description: It is the label key that the selector applies to. + operator: + type: string + description: "It represents a key's relationship to a set of values. Valid operators + are In, NotIn, Exists and DoesNotExist." + values: + type: array + description: It is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. + items: + type: string + storage: + type: object + description: Specify storage configuration for the selected nodes, similar to the one [specified at cluster level](#storage-configuration). If some of the config is left empty, the cluster level storage values are passed to the nodes. If you don't want to use a cluster level value and set the field to empty, then explicitly set an empty value for it so no value is passed to the nodes. For instance, set `spec.nodes[0].storage.kvdbDevice:""`, to prevent using the KVDB device for the selected nodes. + properties: + useAll: + type: boolean + description: Use all available, unformatted, unpartitioned devices. This will be + ignored if spec.storage.devices is not empty. + useAllWithPartitions: + type: boolean + description: Use all available unformatted devices. This will be + ignored if spec.storage.devices is not empty. + forceUseDisks: + type: boolean + description: Flag indicating to use the devices even if there is file system present + on it. Note that the devices may be wiped before using. + devices: + type: array + description: Specifies the list of devices Portworx should use. + items: + type: string + cacheDevices: + type: array + description: Specifies the list of cache devices Portworx should use. + items: + type: string + journalDevice: + type: string + description: Specifies the device Portworx uses for journaling. + systemMetadataDevice: + type: string + description: Indicates the device Portworx uses to store metadata. For better performance, specify a system metadata device when using Portworx with the internal KVDB. + kvdbDevice: + type: string + description: Specifies the device Portworx uses to store internal KVDB data. + cloudStorage: + type: object + description: Details of storage used in cloud environment. + properties: + deviceSpecs: + type: array + description: List of storage device specs. A cloud storage device will be created + for every spec in the list. The specs will be applied to all nodes in the cluster + that match the node group selector. The number of nodes that will come up with + storage are constrained by maxStorageNodes, maxStorageNodesPerZone and + maxStorageNodesPerZonePerNodeGroup. + items: + type: string + journalDeviceSpec: + type: string + description: Specifies the cloud device Portworx uses for journaling. + systemMetadataDeviceSpec: + type: string + description: Device spec for the metadata device. This device will be used to store + system metadata by the driver. + kvdbDeviceSpec: + type: string + description: Specifies the cloud device Portworx uses for an internal KVDB. + maxStorageNodesPerZonePerNodeGroup: + type: integer + format: int32 + minimum: 0 + description: Maximum nodes in every zone in every node group that will have storage + in the cluster. + network: + type: object + description: Specify network configuration for the selected nodes, similar to the one [specified at cluster level](#network-configuration). If this network configuration is empty, then cluster level values are used. + properties: + dataInterface: + type: string + description: Specifies the network interface Portworx uses for data traffic. + mgmtInterface: + type: string + description: Name of the network interface used by the storage driver for + management traffic. + runtimeOptions: + type: object + x-kubernetes-preserve-unknown-fields: true + description: This is map of any runtime options that need to be sent to the storage + driver. The value is a string. If runtime options are present here at node level, + they will override the ones from cluster configuration. + env: + type: array + description: List of environment variables used by the driver. This is an array + of Kubernetes EnvVar where the value can be given directly or from a source + like field, config map or secret. Environment variables specified here at the + node level will be merged with the ones present in cluster configuration and + sent to the nodes. If there is duplicate, the node level value will take precedence. + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + configMapKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + fieldRef: + type: object + properties: + apiVersion: + type: string + fieldPath: + type: string + resourceFieldRef: + type: object + properties: + containerName: + type: string + divisor: + type: string + resource: + type: string + secretKeyRef: + type: object + properties: + key: + type: string + name: + type: string + optional: + type: boolean + status: + type: object + description: Most recently observed status of the storage cluster. This data may not be up to date. + properties: + clusterName: + type: string + description: Name of the storage cluster. + version: + type: string + description: Version of the storage driver. + clusterUid: + type: string + description: Unique ID of the storage cluster. + phase: + type: string + description: Phase of the StorageCluster is a simple, high-level summary of where the + StorageCluster is in its lifecycle. The condition array contains more detailed + information about the state of the cluster. + reason: + type: string + description: CamelCase messages split with commas indicating details about why the StorageCluster is in this state. + collisionCount: + type: integer + format: int32 + description: Count of hash collisions for the StorageCluster. The StorageCluster controller + uses this field as a collision avoidance mechanism when it needs to create the name of + the newest ControllerRevision. + storage: + type: object + description: Contains details of storage in the cluster. + properties: + storageNodesPerZone: + type: integer + format: int64 + description: The number of storage nodes per zone in the cluster. + desiredImages: + type: object + description: Represents all the desired images of various components. + properties: + stork: + type: string + description: Desired image for stork. + userInterface: + type: string + description: Desired image for user interface. + autopilot: + type: string + description: Desired image for autopilot. + csiNodeDriverRegistrar: + type: string + description: Desired image for CSI node driver registrar. + csiDriverRegistrar: + type: string + description: Desired image for CSI driver registrar. + csiProvisioner: + type: string + description: Desired image for CSI provisioner. + csiAttacher: + type: string + description: Desired image for CSI attacher. + csiResizer: + type: string + description: Desired image for CSI resizer. + csiSnapshotter: + type: string + description: Desired image for CSI snapshotter. + csiSnapshotController: + type: string + description: Desired image for CSI snapshot controller. + csiHealthMonitorController: + type: string + description: Desired image for CSI health monitor controller. + prometheusOperator: + type: string + description: Desired image for Prometheus operator. + prometheusConfigMapReload: + type: string + description: Desired image for Prometheus config map reload. + prometheusConfigReloader: + type: string + description: Desired image for Prometheus config reloader. + prometheus: + type: string + description: Desired image for Prometheus. + grafana: + type: string + description: Desired image for Grafana. + alertManager: + type: string + description: Desired image for AlertManager. + telemetry: + type: string + description: Desired image for telemetry. + metricsCollector: + type: string + description: Desired image for metrics collector. + metricsCollectorProxy: + type: string + description: Desired image for metrics collector proxy. + telemetryProxy: + type: string + description: Desired image for telemetry proxy. + logUploader: + type: string + description: Desired image for log uploader. + kubeScheduler: + type: string + description: Desired image for kubernetes scheduler. + kubeControllerManager: + type: string + description: Desired image for kubernetes controller manager. + pause: + type: string + description: Desired image for pause image. + dynamicPlugin: + type: string + description: Desired image for dynamic plugin image. + dynamicPluginProxy: + type: string + description: Desired image for nginx proxy image. + conditions: + type: array + description: Contains details for the current condition of this cluster. + items: + type: object + properties: + source: + type: string + description: Name of the component. + type: + type: string + description: Type of the condition. + status: + type: string + description: Status of the condition. + message: + type: string + description: Message is human readable message indicating details about the current + state of the cluster. + lastTransitionTime: + type: string + format: date-time + description: Time at which the condition changed. + - name: v1alpha1 + served: false + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/charts/portworx/portworx/5.1.1/crds/core_v1_storagenode_crd.yaml b/charts/portworx/portworx/5.1.1/crds/core_v1_storagenode_crd.yaml new file mode 100644 index 000000000..829969abf --- /dev/null +++ b/charts/portworx/portworx/5.1.1/crds/core_v1_storagenode_crd.yaml @@ -0,0 +1,191 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: storagenodes.core.libopenstorage.org +spec: + group: core.libopenstorage.org + names: + kind: StorageNode + listKind: StorageNodeList + plural: storagenodes + singular: storagenode + shortNames: + - sn + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: ID + type: string + description: The corresponding Kubernetes node name for the storage node + jsonPath: .status.nodeUid + - name: Status + type: string + description: The status of the storage node + jsonPath: .status.phase + - name: Version + type: string + description: The version of the storage node + jsonPath: .spec.version + - name: Age + type: date + description: The age of the storage cluster + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + description: The desired behavior of the storage node. Currently changing the spec does + not affect the actual storage node in the cluster. Eventually spec in StorageNode will + override the spec from StorageCluster so that configuration can be overridden at node + level. + properties: + version: + type: string + description: Version of the storage driver on the node. + cloudStorage: + type: object + description: Details of storage on the node for cloud environments. + properties: + driveConfigs: + type: array + description: List of cloud drive configs for the storage node. + items: + type: object + properties: + type: + type: string + description: Type of cloud drive. + sizeInGiB: + type: integer + format: int64 + minimum: 0 + description: Size of cloud drive in GiB. + iops: + type: integer + format: int64 + minimum: 0 + description: IOPS required from the cloud drive. + options: + type: object + x-kubernetes-preserve-unknown-fields: true + description: Additional options for the cloud drive. + status: + type: object + description: Most recently observed status of the storage node. The data may not be up + to date. + properties: + nodeUid: + type: string + description: Unique ID of the storage node. + phase: + type: string + description: Phase of the StorageNode is a simple, high-level summary of where + the StorageNode is in its lifecycle. The condition array contains more detailed + information about the state of the node. + network: + type: object + description: Contains network information used by the storage node + properties: + dataIP: + type: string + description: IP address used by the storage driver for data traffic. + mgmtIP: + type: string + description: IP address used by the storage driver for management traffic. + storage: + type: object + description: Contains details of the status of storage for the node + properties: + totalSize: + type: string + description: Cumulative total size of all storage pools on the node. + usedSize: + type: string + description: Cumulative used size of all storage pools on the node. + conditions: + type: array + description: Contains details for the current condition of this storage node. + items: + type: object + properties: + type: + type: string + description: Type of the condition. + status: + type: string + description: Status of the condition. + reason: + type: string + description: Reason is a unique one-word reason about the current state + of the cluster. + message: + type: string + description: Message is the human readable message indicating details about the + current state of the cluster. + lastTransitionTime: + type: string + format: date-time + description: Time at which the condition changed. + checks: + type: array + description: Contains list of pre or post flight checks that are performed by the Operator + items: + type: object + properties: + type: + type: string + description: Type of the check. + reason: + type: string + description: Reason for success or failure of the check + success: + type: boolean + description: If true, the check was successful + result: + type: string + description: Result of the check fatal, warning, success + geography: + type: object + description: Contains topology information for the storage node. + properties: + region: + type: string + description: Region in which the storage node is placed. + zone: + type: string + description: Zone in which the storage node is placed. + rack: + type: string + description: Rack on which the storage node is placed. + operatingSystem: + type: string + description: Operating system of the underlying host. + kernelVersion: + type: string + description: Kernel version of the underlying host. + nodeAttributes: + type: object + description: Attributes of the storage node. + properties: + storage: + type: boolean + description: Indicates whether the node is a storage node or not. + kvdb: + type: boolean + description: Indicates whether the node is a kvdb node or not. + - name: v1alpha1 + served: false + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/charts/portworx/portworx/5.1.1/files/portworx-cluster-dashboard.json b/charts/portworx/portworx/5.1.1/files/portworx-cluster-dashboard.json new file mode 100644 index 000000000..28275070d --- /dev/null +++ b/charts/portworx/portworx/5.1.1/files/portworx-cluster-dashboard.json @@ -0,0 +1,975 @@ + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0-beta1" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1541099197839, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 54, + "panels": [], + "repeat": null, + "title": "Portworx Cluster \"[[Cluster]]\"", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#fce2de", + "#eab839", + "#bf1b00" + ], + "datasource": "prometheus", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 56, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(px_cluster_disk_utilized_bytes{cluster=~\"[[Cluster]]\"})/sum(px_cluster_disk_total_bytes{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80,90", + "title": "Usage Meter", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 63, + "interval": null, + "links": [ + { + "dashUri": "db/portworx-volume-dashboard", + "dashboard": "Portworx Volume Dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": true, + "title": "Portworx Volume Dashboard", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(px_cluster_disk_utilized_bytes{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Capacity Used", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 61, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(px_cluster_cpu_percent{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Avg. Cluster CPU", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 62, + "interval": null, + "links": [ + { + "dashUri": "db/portworx-node-dashboard", + "dashboard": "Portworx Node Dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": true, + "title": "Portworx Node Dashboard", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_cluster_size{cluster=~\"[[Cluster]]\"}) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "# Nodes (total)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "#629e51", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "hideTimeOverride": true, + "id": 81, + "interval": "15s", + "links": [], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "2", + "text": "Quorum Unhealthy", + "to": "1000" + }, + { + "from": "0", + "text": "Quorum Healthy", + "to": "1.99" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_cluster_size{cluster=~\"[[Cluster]]\"})/sum(px_cluster_status_cluster_quorum{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "1.1,1.9", + "timeFrom": "1m", + "title": "Quorum", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 85, + "interval": null, + "links": [ + { + "dashUri": "db/portworx-node-dashboard", + "dashboard": "Portworx Node Dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": true, + "title": "Portworx Node Dashboard", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_nodes_online{cluster=~\"[[Cluster]]\"}) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "# Nodes online", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_storage_nodes_online{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Storage Providers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "0", + "text": "All members online", + "to": "0" + }, + { + "from": "1", + "text": "Members offline", + "to": "1" + }, + { + "from": "2", + "text": "Members offline", + "to": "1000" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_cluster_size{cluster=\"[[Cluster]]\"})-min(px_cluster_status_storage_nodes_online{cluster=\"[[Cluster]]\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0.5,1", + "title": "", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "", + "value": "0" + }, + { + "op": "=", + "text": "", + "value": "" + } + ], + "valueName": "current" + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "prometheus", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "heatmap": {}, + "highlightCards": true, + "id": 78, + "legend": { + "show": true + }, + "links": [ + { + "dashUri": "db/portworx-node-dashboard", + "dashboard": "Portworx Node Dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": true, + "title": "Portworx Node Dashboard", + "type": "dashboard" + } + ], + "targets": [ + { + "expr": "px_cluster_cpu_percent{cluster =~ \"[[Cluster]]\"}", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU utilization heat map", + "tooltip": { + "show": true, + "showHistogram": true + }, + "transparent": false, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "percent", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "prometheus", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "heatmap": {}, + "highlightCards": true, + "id": 79, + "legend": { + "show": true + }, + "links": [ + { + "dashUri": "db/portworx-node-dashboard", + "dashboard": "Portworx Node Dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": true, + "title": "Node Drilldown", + "type": "dashboard" + } + ], + "repeat": null, + "repeatDirection": "h", + "targets": [ + { + "expr": "px_cluster_memory_utilized_percent{cluster =~ \"[[Cluster]]\"}", + "format": "time_series", + "interval": "5m", + "intervalFactor": 1, + "legendFormat": "{{cluster}}", + "refId": "A" + } + ], + "title": "Memory utilization heat map", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "percent", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketNumber": null, + "yBucketSize": null + } + ], + "refresh": "30s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "hide": 1, + "includeAll": false, + "label": "", + "multi": false, + "name": "Cluster", + "options": [], + "query": "px_cluster_cpu_percent", + "refresh": 1, + "regex": "/.*cluster=\"([^\"]*).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Portworx Cluster Dashboard", + "uid": "xLgt8oTik", + "version": 6 + } \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/files/portworx-etcd-dashboard.json b/charts/portworx/portworx/5.1.1/files/portworx-etcd-dashboard.json new file mode 100644 index 000000000..adb5c0be8 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/files/portworx-etcd-dashboard.json @@ -0,0 +1,1189 @@ + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0-beta1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(etcd_server_has_leader)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 10, + "x": 6, + "y": 0 + }, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{grpc_type=\"unary\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_debugging_mvcc_db_total_size_in_bytes", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 5, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 5, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total[1d])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Etcd Dashboard", + "uid": "CRWfZ_tiz", + "version": 15 + } \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/files/portworx-node-dashboard.json b/charts/portworx/portworx/5.1.1/files/portworx-node-dashboard.json new file mode 100644 index 000000000..e39076bcf --- /dev/null +++ b/charts/portworx/portworx/5.1.1/files/portworx-node-dashboard.json @@ -0,0 +1,1529 @@ + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0-beta1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1541099248953, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 72, + "panels": [], + "title": "", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 62, + "interval": null, + "links": [ + { + "dashUri": "db/portworx-cluster-dashboard", + "dashboard": "Portworx Cluster Dashboard", + "includeVars": true, + "keepTime": true, + "title": "Porworx Cluster Dashboard", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_cluster_size{cluster=~\"[[Cluster]]\"}) ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Members", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(px_cluster_status_storage_nodes_online{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Storage Providers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 14, + "x": 10, + "y": 1 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Cluster Disk Total", + "dsType": "influxdb", + "expr": "px_cluster_disk_total_bytes{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_total_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Cluster Disk Used", + "dsType": "influxdb", + "expr": "px_cluster_disk_utilized_bytes{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_utilized_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PWX Disk Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 65, + "panels": [], + "repeat": null, + "title": "Instances \"[[Instances]]\"", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "CPU Usage: $tag_cluster - $tag_node", + "dsType": "influxdb", + "expr": "px_cluster_cpu_percent{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "node" + ], + "type": "tag" + }, + { + "params": [ + "cluster" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_cpu_percent", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Memory Usage: $tag_node", + "dsType": "influxdb", + "expr": "px_cluster_memory_utilized_percent{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "node" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_memory_utilized_percent", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "Memory Usage" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PWX Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "IO Pending", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "IO Bytes TX", + "dsType": "influxdb", + "expr": "px_network_io_bytessent{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Bytes TX", + "measurement": "px_network_io_bytessent", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "IO Bytes RX", + "dsType": "influxdb", + "expr": "px_network_io_received_bytes{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Bytes RX", + "measurement": "px_network_io_received_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network TX/RX", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "IO Pending", + "color": "#890F02" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "IO Pending", + "dsType": "influxdb", + "expr": "irate(px_disk_stats_read_seconds {cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Read {{cluster}} {{instance}}", + "measurement": "px_cluster_pendingio", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "IO Pending", + "dsType": "influxdb", + "expr": "irate(px_disk_stats_write_seconds {cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Write {{cluster}} {{instance}}", + "measurement": "px_cluster_pendingio", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PWX Disk IO", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 86, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Cluster Disk Total", + "dsType": "influxdb", + "expr": "irate(px_disk_stats_read_bytes{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_total_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Cluster Disk Used", + "dsType": "influxdb", + "expr": "irate(px_disk_stats_write_bytes{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}[5m])", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_utilized_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PWX Disk throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 13 + }, + "id": 87, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Cluster Disk Total", + "dsType": "influxdb", + "expr": "px_disk_stats_read_latency_seconds{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_total_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Cluster Disk Used", + "dsType": "influxdb", + "expr": "px_disk_stats_write_latency_seconds{cluster=~\"[[Cluster]]\",instance=~\"[[Instances]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{cluster}}{{instance}}", + "measurement": "px_cluster_disk_utilized_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PWX Disk Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "hide": 1, + "includeAll": false, + "label": "", + "multi": false, + "name": "Cluster", + "options": [], + "query": "px_cluster_cpu_percent", + "refresh": 1, + "regex": "/.*cluster=\"([^\"]*).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "hide": 1, + "includeAll": false, + "label": null, + "multi": true, + "name": "Instances", + "options": [], + "query": "px_cluster_cpu_percent", + "refresh": 1, + "regex": "/.*instance=\"([^\"]*).*/", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Portworx Node Dashboard", + "uid": "Vhz18oTik", + "version": 3 + } \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/files/portworx-volume-dashboard.json b/charts/portworx/portworx/5.1.1/files/portworx-volume-dashboard.json new file mode 100644 index 000000000..620d9e590 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/files/portworx-volume-dashboard.json @@ -0,0 +1,2247 @@ + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0-beta1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1541099296083, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "panels": [], + "title": "All Volumes", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "decimals": null, + "format": "dtdurations", + "gauge": { + "maxValue": 30, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 77, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "Value", + "targets": [ + { + "expr": "avg(px_volume_vol_read_latency_seconds{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "15,25", + "title": "Avg Read Latency (1m)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 5, + "y": 1 + }, + "id": 82, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk (10,(px_volume_usage_bytes{cluster=~\"[[Cluster]]\"}/px_volume_capacity_bytes{cluster=~\"[[Cluster]]\"})*100)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{volumename}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Top n Volumes by Capacity", + "tooltip": { + "shared": false, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 9, + "x": 15, + "y": 1 + }, + "id": 83, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,px_volume_depth_io{cluster=\"[[Cluster]]\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{volumename}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Top n Volumes by IO depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "decimals": null, + "format": "dtdurations", + "gauge": { + "maxValue": 30, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 0, + "y": 4 + }, + "id": 78, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "Value", + "targets": [ + { + "expr": "avg(px_volume_vol_write_latency_seconds{cluster=~\"[[Cluster]]\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": "15,25", + "title": "Avg Write Latency (1m)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 55, + "panels": [], + "repeat": "Volumes", + "title": "Volume: [[Volumes]]", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 3, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "px_volume_currhalevel{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "{{volumename}}", + "measurement": "px_volume_halevel", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": "", + "title": "Current Replication Level (HA)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 3, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 8 + }, + "id": 61, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "px_volume_iopriority{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "{{volumename}}", + "measurement": "px_volume_halevel", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": "", + "title": "I/O Priority", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "High", + "value": "1" + }, + { + "op": "=", + "text": "Medium", + "value": "2" + }, + { + "op": "=", + "text": "Low", + "value": "3" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "decimals": null, + "format": "short", + "gauge": { + "maxValue": 30, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 8 + }, + "id": 79, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "Value", + "targets": [ + { + "expr": "avg(px_volume_vol_read_latency_seconds{cluster=~\"[[Cluster]]\",volumename=~\"[[Volumes]]\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "15,25", + "title": "Avg Read Latency", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "prometheus", + "decimals": null, + "format": "short", + "gauge": { + "maxValue": 30, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 8 + }, + "id": 80, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "Value", + "targets": [ + { + "expr": "avg(px_volume_vol_write_latency_seconds{cluster=~\"[[Cluster]]\",volumename=~\"[[Volumes]]\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 4, + "legendFormat": "", + "refId": "B" + } + ], + "thresholds": "15,25", + "title": "Avg Write Latency", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 0, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Disk Capacity", + "dsType": "influxdb", + "expr": "px_volume_capacity_bytes{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": " Capacity", + "measurement": "px_volume_capacity_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Disk Usage", + "dsType": "influxdb", + "expr": "px_volume_usage_bytes{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": " Usage", + "measurement": "px_volume_usage_bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus", + "format": "none", + "gauge": { + "maxValue": 3, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "px_volume_halevel{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "{{volumename}}", + "measurement": "px_volume_halevel", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": "", + "title": "Configured Replication Level (HA)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Bytes Written Throughput", + "dsType": "influxdb", + "expr": "px_volume_num_long_reads{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "No. of Reads ", + "measurement": "px_volume_writethroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Bytes Read Throughput", + "dsType": "influxdb", + "expr": "px_volume_num_long_writes{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "No. of Writes ", + "measurement": "px_volume_readthroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume total reads/writes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Bytes Written Throughput", + "dsType": "influxdb", + "expr": "px_volume_iops{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "IOPs", + "measurement": "px_volume_writethroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume IOPs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Bytes Written Throughput", + "dsType": "influxdb", + "expr": "px_volume_read_bytes{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Bytes Read ", + "measurement": "px_volume_writethroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Bytes Read Throughput", + "dsType": "influxdb", + "expr": "px_volume_written_bytes{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "Bytes Written ", + "measurement": "px_volume_readthroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Read/Written Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Read Latency", + "dsType": "influxdb", + "expr": "px_volume_vol_read_latency_seconds{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Read Latency", + "measurement": "px_volume_vol_read_latency_seconds", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Write Latency", + "dsType": "influxdb", + "expr": "px_volume_vol_write_latency_seconds{volumename=~\"[[Volumes]]\"}\n", + "format": "time_series", + "groupBy": [ + { + "params": [ + "10s" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Write Latency", + "measurement": "px_volume_vol_write_latency_seconds", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "dtdurations", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "hideTimeOverride": true, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "px_volume_depth_io{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "I/O Depth", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": "1h", + "timeShift": null, + "title": "Volume IO Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prometheus", + "decimals": 2, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Bytes Written Throughput", + "dsType": "influxdb", + "expr": "px_volume_readthroughput{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "intervalFactor": 1, + "legendFormat": "Bytes Read Throughput", + "measurement": "px_volume_writethroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + }, + { + "alias": "Bytes Read Throughput", + "dsType": "influxdb", + "expr": "px_volume_writethroughput{volumename=~\"[[Volumes]]\"}", + "format": "time_series", + "groupBy": [ + { + "params": [ + "30s" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 1, + "legendFormat": "Bytes Written Throughput", + "measurement": "px_volume_readthroughput", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "gauge" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "volumename", + "operator": "=~", + "value": "/^$Volumes$/" + }, + { + "condition": "AND", + "key": "cluster", + "operator": "=~", + "value": "/^$Cluster$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Throughput", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "Cluster", + "options": [], + "query": "px_cluster_cpu_percent", + "refresh": 1, + "regex": "/.*cluster=\"([^\"]*).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "hide": 0, + "includeAll": false, + "label": "Volume Name", + "multi": true, + "name": "Volumes", + "options": [], + "query": "px_volume_depth_io {cluster=\"[[Cluster]]\"}", + "refresh": 1, + "regex": "/.*volumename=\"([^\"]*).*/", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Portworx Volume Dashboard", + "uid": "LAgtVanmz", + "version": 82 + } \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/questions.yml b/charts/portworx/portworx/5.1.1/questions.yml new file mode 100644 index 000000000..ff44a2c67 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/questions.yml @@ -0,0 +1,43 @@ +questions: + +################################## Basic Settings ############################ +- variable: imageVersion + default: "3.2.1.1" + type: string + label: Portworx version to be deployed. + group: "Basic Settings" + +- variable: clusterName + type: string + label: Portworx cluster name + default: mycluster + group: "Basic Settings" + +############ Storage Parameters ############## +- variable: usedrivesAndPartitions + label: "Use unmounted drives and partitions" + descrition: "Use unmounted disks even if they have a partition or filesystem on it. PX will never use a drive or partition that is mounted." + type: boolean + default: false + group: "Storage Parameters" + +################################### Registry settings options ################################ +- variable: customRegistry + label: "Use a custom container registry?" + type: boolean + default: false + group: "Container Registry Parameters" + +- variable: registrySecret + show_if: "customRegistry=true" + description: "Specify a custom Kubernetes secret that will be used to authenticate with a container registry. Must be defined in kube-system namespace. (example: regcred)" + type: string + label: "Registry Kubernetes Secret" + group: "Container Registry Parameters" + +- variable: customRegistryURL + show_if: "customRegistry=true" + description: "Specify a custom container registry server (including repository) that will be used instead of index.docker.io to download Docker images. (example: dockerhub.acme.net:5443 or myregistry.com/myrepository/)" + label: "Custom Registry URL" + type: string + group: "Container Registry Parameters" \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/templates/NOTES.txt b/charts/portworx/portworx/5.1.1/templates/NOTES.txt new file mode 100644 index 000000000..85d288ba1 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/NOTES.txt @@ -0,0 +1,24 @@ + +Your Release is named {{ .Release.Name | quote }} +Portworx Pods should be running on each node in your cluster. + +Portworx would create a unified pool of the disks attached to your Kubernetes nodes. +No further action should be required and you are ready to consume Portworx Volumes as part of your application data requirements. + +For further information on usage of the Portworx in creating Volumes please refer + https://docs.portworx.com/portworx-enterprise/operations/operate-kubernetes/storage-operations/create-pvcs + +Explore the fundamental concepts of Portworx volumes refer + https://docs.portworx.com/portworx-enterprise/concepts + +Want to use Storage Orchestration for hyperconvergence, Please look at Stork here. + https://docs.portworx.com/portworx-enterprise/operations/operate-kubernetes/storage-operations/stork + +Track the health and performance of your storage cluster using Portworx’s monitoring features. + https://docs.portworx.com/portworx-enterprise/operations/operate-kubernetes/monitoring + +Refer to the install various stateful applications with Portworx. + https://docs.portworx.com/portworx-enterprise/operations/operate-kubernetes/application-install-with-kubernetes + + +For options that you could provide while installing Portworx on your cluster head over to the README.md \ No newline at end of file diff --git a/charts/portworx/portworx/5.1.1/templates/_helpers.tpl b/charts/portworx/portworx/5.1.1/templates/_helpers.tpl new file mode 100644 index 000000000..21b3623b4 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/_helpers.tpl @@ -0,0 +1,287 @@ +{{/* Gets the correct API Version based on the version of the cluster +*/}} + +{{- define "rbac.apiVersion" -}} +{{$version := .Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+" | trimPrefix "v"}} +{{- if semverCompare ">= 1.8" $version -}} +"rbac.authorization.k8s.io/v1" +{{- else -}} +"rbac.authorization.k8s.io/v1beta1" +{{- end -}} +{{- end -}} + +{{- define "px.labels" -}} +chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +heritage: {{ .Release.Service | quote }} +release: {{ .Release.Name | quote }} +{{- end -}} + +{{- define "driveOpts" }} +{{ $v := .Values.installOptions.drives | split "," }} +{{$v._0}} +{{- end -}} + +{{- define "px.kubernetesVersion" -}} +{{$version := .Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+"}}{{$version}} +{{- end -}} + +{{- define "px.kubectlImageTag" -}} +{{$version := .Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+" | trimPrefix "v" | split "."}} +{{- $major := index $version "_0" -}} +{{- $minor := index $version "_1" -}} +{{printf "%s.%s" $major $minor }} +{{- end -}} + +{{- define "px.getPxOperatorImage" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ cat (trim .Values.customRegistryURL) "/px-operator" | replace " " ""}} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/portworx/px-operator" | replace " " ""}} + {{- end -}} +{{- else -}} + {{ "portworx/px-operator" }} +{{- end -}} +{{- end -}} + +{{- define "px.getImage" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{- if .Values.openshiftInstall -}} + {{ cat (trim .Values.customRegistryURL) "/px-monitor" | replace " " ""}} + {{- else -}} + {{ cat (trim .Values.customRegistryURL) "/oci-monitor" | replace " " ""}} + {{- end -}} + {{- else -}} + {{- if .Values.openshiftInstall -}} + {{cat (trim .Values.customRegistryURL) "/portworx/px-monitor" | replace " " ""}} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/portworx/oci-monitor" | replace " " ""}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if .Values.openshiftInstall -}} + {{ "registry.connect.redhat.com/portworx/px-monitor" }} + {{- else -}} + {{ "portworx/oci-monitor" }} + {{- end -}} +{{- end -}} +{{- end -}} + +{{- define "px.getStorkImage" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ cat (trim .Values.customRegistryURL) "/stork" | replace " " ""}} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/openstorage/stork" | replace " " ""}} + {{- end -}} +{{- else -}} + {{ "openstorage/stork" }} +{{- end -}} +{{- end -}} + +{{- define "px.getk8sImages" -}} +{{- $version := .Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+" | trimPrefix "v" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ trim .Values.customRegistryURL }} + {{- else -}} + {{- if or (or (and (semverCompare ">= 1.16.14" $version ) (semverCompare "<=1.17.0" $version)) (and (semverCompare ">= 1.17.10" $version) (semverCompare "<=1.18.0" $version ))) (semverCompare ">=1.18.7" $version) -}} + {{cat (trim .Values.customRegistryURL) "/k8s.gcr.io" | replace " " ""}} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/gcr.io/google_containers" | replace " " ""}} + {{- end -}} + {{- end -}} +{{- else -}} + {{- if or (or (and (semverCompare ">= 1.16.14" $version ) (semverCompare "<=1.17.0" $version)) (and (semverCompare ">= 1.17.10" $version) (semverCompare "<=1.18.0" $version ))) (semverCompare ">=1.18.7" $version) -}} + {{ "k8s.gcr.io" }} + {{- else -}} + {{ "gcr.io/google_containers" }} + {{- end -}} +{{- end -}} +{{- end -}} + + +{{- define "px.getPauseImage" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ trim .Values.customRegistryURL }} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/k8s.gcr.io" | replace " " ""}} + {{- end -}} +{{- else -}} + {{ "k8s.gcr.io" }} +{{- end -}} +{{- end -}} + +{{- define "px.getcsiImages" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ trim .Values.customRegistryURL }} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/quay.io/k8scsi" | replace " " ""}} + {{- end -}} +{{- else -}} + {{ "quay.io/k8scsi" }} +{{- end -}} +{{- end -}} + +{{- define "px.getLighthouseImages" -}} +{{- if (.Values.customRegistryURL) -}} + {{- if (eq "/" (.Values.customRegistryURL | regexFind "/")) -}} + {{ trim .Values.customRegistryURL }} + {{- else -}} + {{cat (trim .Values.customRegistryURL) "/portworx" | replace " " ""}} + {{- end -}} +{{- else -}} + {{ "portworx" }} +{{- end -}} +{{- end -}} + +{{- define "px.registryConfigType" -}} +{{- $version := .Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+" | trimPrefix "v" -}} +{{- if semverCompare ">=1.9" $version -}} +".dockerconfigjson" +{{- else -}} +".dockercfg" +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use for hooks +*/}} +{{- define "px.hookServiceAccount" -}} +{{- if .Values.serviceAccount.hook.create -}} + {{- printf "%s-hook" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{ default "default" .Values.serviceAccount.hook.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the cluster role to use for hooks +*/}} +{{- define "px.hookClusterRole" -}} +{{- if .Values.serviceAccount.hook.create -}} + {{- printf "%s-hook" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{ default "default" .Values.serviceAccount.hook.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the cluster role binding to use for hooks +*/}} +{{- define "px.hookClusterRoleBinding" -}} +{{- if .Values.serviceAccount.hook.create -}} + {{- printf "%s-hook" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{ default "default" .Values.serviceAccount.hook.name }} +{{- end -}} +{{- end -}} + + +{{/* +Create the name of the role to use for hooks +*/}} +{{- define "px.hookRole" -}} +{{- if .Values.serviceAccount.hook.create -}} + {{- printf "%s-hook" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{ default "default" .Values.serviceAccount.hook.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the role binding to use for hooks +*/}} +{{- define "px.hookRoleBinding" -}} +{{- if .Values.serviceAccount.hook.create -}} + {{- printf "%s-hook" .Chart.Name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- else -}} + {{ default "default" .Values.serviceAccount.hook.name }} +{{- end -}} +{{- end -}} + +{{/* +Generate a random token for storage provisioning +*/}} + +{{- define "portworx-cluster-key" -}} +{{- randAlphaNum 16 | nospace | b64enc -}} +{{- end -}} + + +{{- define "px.affinityPxEnabledOperator" -}} +{{- if .Values.requirePxEnabledTag -}} + {{- "In" }} +{{- else -}} + {{ "NotIn" }} +{{- end -}} +{{- end -}} + + +{{- define "px.getDeploymentNamespace" -}} +{{- if (.Release.Namespace) -}} + {{- if (eq "default" .Release.Namespace) -}} + {{- printf "portworx" -}} + {{- else -}} + {{- printf "%s" .Release.Namespace -}} + {{- end -}} +{{- end -}} +{{- end -}} + + + +{{- define "px.affinityPxEnabledValue" -}} +{{- if .Values.requirePxEnabledTag -}} + {{- "true" | quote }} +{{- else -}} + {{ "false" | quote }} +{{- end -}} +{{- end -}} + +{{- define "px.deprecatedKvdbArgs" }} +{{- $result := "" }} +{{- if ne .Values.etcd.credentials "none:none" }} + {{- $result = printf "%s -userpwd %s" $result .Values.etcd.credentials }} +{{- end }} +{{- if ne .Values.etcd.ca "none" }} + {{- $result = printf "%s -ca %s" $result .Values.etcd.ca }} +{{- end }} +{{- if ne .Values.etcd.cert "none" }} + {{- $result = printf "%s -cert %s" $result .Values.etcd.cert }} +{{- end }} +{{- if ne .Values.etcd.key "none" }} + {{- $result = printf "%s -key %s" $result .Values.etcd.key }} +{{- end }} +{{- if ne .Values.consul.token "none" }} + {{- $result = printf "%s -acltoken %s" $result .Values.consul.token }} +{{- end }} +{{- trim $result }} +{{- end }} + +{{- define "px.miscArgs" }} +{{- $result := "" }} +{{- if (include "px.deprecatedKvdbArgs" .) }} + {{- $result = printf "%s %s" $result (include "px.deprecatedKvdbArgs" .) }} +{{- end }} +{{- if ne .Values.miscArgs "none" }} + {{- $result = printf "%s %s" $result .Values.miscArgs }} +{{- end }} +{{- trim $result }} +{{- end }} + +{{- define "px.volumesPresent" }} +{{- $result := false }} +{{- if (default false .Values.isTargetOSCoreOS) }} + {{- $result = true }} +{{- end }} +{{- if ne (default "none" .Values.etcd.certPath) "none" }} + {{- $result = true }} +{{- end }} +{{- if .Values.volumes }} + {{- $result = true }} +{{- end }} +{{- $result }} +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/hooks/post-install/px-create-cluster-token.yaml b/charts/portworx/portworx/5.1.1/templates/hooks/post-install/px-create-cluster-token.yaml new file mode 100644 index 000000000..b943fdbc9 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/hooks/post-install/px-create-cluster-token.yaml @@ -0,0 +1,134 @@ +{{- if (.Values.clusterToken.create) }} + {{- $customRegistryURL := .Values.customRegistryURL | default "none" }} + {{- $registrySecret := .Values.registrySecret | default "none" }} + +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ template "px.getDeploymentNamespace" . }} + name: px-set-cluster-token + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + template: + spec: + {{- if not (eq $registrySecret "none") }} + imagePullSecrets: + - name: {{ $registrySecret }} + {{- end }} + restartPolicy: Never + serviceAccountName: {{ .Values.clusterToken.serviceAccountName }} + containers: + - name: post-install-job + {{- if eq $customRegistryURL "none" }} + image: "bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- else}} + image: "{{ $customRegistryURL }}/bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- end }} + env: + - name: NS + value: {{ template "px.getDeploymentNamespace" . }} + - name: KEY + value: cluster-wide-secret-key + command: ['/bin/bash', '-c'] + args: + - | + readyPortworxPod=$(kubectl -n $NS get pods -lname=portworx -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{" "}{.metadata.name}{"\n"}{end}' | grep True | cut -d' ' -f2 | head -n1) + kubectl -n $NS exec -c portworx $readyPortworxPod -- /opt/pwx/bin/pxctl secrets set-cluster-key --secret $KEY + initContainers: + - name: post-install-job-init + {{- if eq $customRegistryURL "none" }} + image: "bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- else}} + image: "{{ $customRegistryURL }}/bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- end }} + env: + - name: NS + value: {{ template "px.getDeploymentNamespace" . }} + command: ['/bin/bash', '-c'] + args: + - | + # Get ready state of all Portworx pods (True or False) + output=$(kubectl -n $NS get pods -l name=portworx -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') + # Wait until there is at least one pod and at least one of them is ready + while [[ $(echo -n $output | wc -c) -eq 0 ]] || [[ $(echo -n $output | grep -v True) ]]; do + sleep 5 + output=$(kubectl -n $NS get pods -l name=portworx -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') + done +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.clusterToken.serviceAccountName }} + namespace: {{ template "px.getDeploymentNamespace" . }} + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + "helm.sh/hook": "post-install" + labels: + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" +--- +kind: Role +apiVersion: {{ template "rbac.apiVersion" . }} +metadata: + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + "helm.sh/hook": post-install + name: {{ .Values.clusterToken.serviceAccountName }}-role + namespace: {{ template "px.getDeploymentNamespace" . }} +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list" ] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +kind: RoleBinding +apiVersion: {{ template "rbac.apiVersion" . }} +metadata: + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed + "helm.sh/hook": post-install + name: {{ .Values.clusterToken.serviceAccountName }}-binding + namespace: {{ template "px.getDeploymentNamespace" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.clusterToken.serviceAccountName }} + namespace: {{ template "px.getDeploymentNamespace" . }} +roleRef: + kind: Role + name: {{ .Values.clusterToken.serviceAccountName }}-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Secret +metadata: + annotations: + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook": post-install + name: {{ .Values.clusterToken.secretName }} + namespace: portworx + labels: + name: {{ .Values.clusterToken.secretName }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" +type: "Opaque" +data: + cluster-wide-secret-key: {{ template "portworx-cluster-key" }} +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/hooks/pre-delete/delete-storagecluster.yaml b/charts/portworx/portworx/5.1.1/templates/hooks/pre-delete/delete-storagecluster.yaml new file mode 100644 index 000000000..a46325fb8 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/hooks/pre-delete/delete-storagecluster.yaml @@ -0,0 +1,41 @@ +{{- if .Values.deployCluster }} +{{- $customRegistryURL := .Values.customRegistryURL | default "none" }} +{{- $registrySecret := .Values.registrySecret | default "none" }} +{{- $clusterName := .Values.clusterName }} + +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ template "px.getDeploymentNamespace" . }} + name: px-hook-delete-storagecluster + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + backoffLimit: 0 + template: + spec: + {{- if not (eq $registrySecret "none") }} + imagePullSecrets: + - name: {{ $registrySecret }} + {{- end }} + serviceAccountName: {{ template "px.hookServiceAccount" . }} + restartPolicy: Never + containers: + - name: delete-storagecluster + {{- if eq $customRegistryURL "none" }} + image: "bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- else}} + image: "{{ $customRegistryURL }}/bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- end}} + command: ['/bin/sh', + '-c', + 'kubectl -n {{ template "px.getDeploymentNamespace" . }} delete storagecluster {{ $clusterName }} --ignore-not-found'] +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/hooks/pre-upgrade/retain-daemonset-install.yaml b/charts/portworx/portworx/5.1.1/templates/hooks/pre-upgrade/retain-daemonset-install.yaml new file mode 100644 index 000000000..69f21fdc9 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/hooks/pre-upgrade/retain-daemonset-install.yaml @@ -0,0 +1,97 @@ +{{- $customRegistryURL := .Values.customRegistryURL | default "none" }} +{{- $registrySecret := .Values.registrySecret | default "none" }} + +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ template "px.getDeploymentNamespace" . }} + name: px-hook-retain-daemonset-resources + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "10" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + {{- if not (eq $registrySecret "none") }} + imagePullSecrets: + - name: {{ $registrySecret }} + {{- end }} + serviceAccountName: {{ template "px.hookServiceAccount" . }} + restartPolicy: Never + containers: + - name: retain-px-daemonset + {{- if eq $customRegistryURL "none" }} + image: "bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- else}} + image: "{{ $customRegistryURL }}/bitnami/kubectl:{{ template "px.kubectlImageTag" . }}" + {{- end}} + command: ['/bin/sh', + '-c', + 'kubectl -n kube-system annotate DaemonSet portworx-api helm.sh/resource-policy=keep --overwrite; + kubectl -n kube-system annotate DaemonSet portworx helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service stork-service helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service prometheus helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service portworx-service helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service autopilot helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service grafana helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service alertmanager-portworx helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service px-csi-service helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Service portworx-api helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment stork-scheduler helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment px-csi-ext helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment autopilot helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment grafana helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment stork helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Deployment prometheus-operator helm.sh/resource-policy=keep --overwrite || true; + + kubectl -n kube-system annotate RoleBinding px-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Role px-role helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding stork-scheduler-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding stork-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding node-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding prometheus helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding px-csi-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding autopilot-role-binding helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRoleBinding prometheus-operator helm.sh/resource-policy=keep --overwrite || true; + + kubectl annotate ClusterRole stork-scheduler-role helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole autopilot-role helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole prometheus helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole prometheus-operator helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole node-get-put-list-role helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole px-csi-role helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate ClusterRole stork-role helm.sh/resource-policy=keep --overwrite || true; + + kubectl annotate StorageClass stork-snapshot-sc helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate StorageClass portworx-shared-sc helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate StorageClass portworx-db2-sc helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate StorageClass portworx-null-sc helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate StorageClass portworx-db-sc helm.sh/resource-policy=keep --overwrite || true; + + kubectl -n kube-system annotate ConfigMap grafana-dashboard-config helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ConfigMap autopilot-config helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ConfigMap grafana-dashboards helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ConfigMap grafana-source-config helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ConfigMap stork-config helm.sh/resource-policy=keep --overwrite || true; + + kubectl -n kube-system annotate ServiceAccount stork-scheduler-account helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount px-account helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount prometheus-operator helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount px-csi-account helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount stork-account helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount prometheus helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceAccount autopilot-account helm.sh/resource-policy=keep --overwrite || true; + + kubectl -n kube-system annotate Alertmanager portworx helm.sh/resource-policy=keep --overwrite || true; + kubectl annotate CSIDriver pxd.portworx.com helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate Prometheus prometheus helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate PrometheusRule prometheus-portworx-rules-portworx.rules.yaml helm.sh/resource-policy=keep --overwrite || true; + kubectl -n kube-system annotate ServiceMonitor portworx-prometheus-sm helm.sh/resource-policy=keep --overwrite || true; + '] diff --git a/charts/portworx/portworx/5.1.1/templates/portworx-k8s-secrets.yaml b/charts/portworx/portworx/5.1.1/templates/portworx-k8s-secrets.yaml new file mode 100644 index 000000000..f42823c4a --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/portworx-k8s-secrets.yaml @@ -0,0 +1,8 @@ +{{- if and (eq .Release.Namespace "default") (not (lookup "v1" "Namespace" "portworx" "portworx")) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ template "px.getDeploymentNamespace" . }} + annotations: + "helm.sh/hook": pre-install +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/portworx-operator.yaml b/charts/portworx/portworx/5.1.1/templates/portworx-operator.yaml new file mode 100644 index 000000000..137b6ef2c --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/portworx-operator.yaml @@ -0,0 +1,103 @@ +{{- if .Values.deployOperator }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: portworx-operator + namespace: {{ template "px.getDeploymentNamespace" . }} +{{- if (semverCompare "<1.25" (.Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+"| trimPrefix "v"))}} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: px-operator +spec: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + volumes: + - secret + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' +{{- end }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: portworx-operator +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] + {{- if (semverCompare "<1.25" (.Capabilities.KubeVersion.GitVersion | regexFind "^v\\d+\\.\\d+\\.\\d+"| trimPrefix "v"))}} + - apiGroups: ["policy"] + resources: ["podsecuritypolicies"] + resourceNames: ["px-operator"] + verbs: ["use"] + {{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: portworx-operator +subjects: + - kind: ServiceAccount + name: portworx-operator + namespace: {{ template "px.getDeploymentNamespace" . }} +roleRef: + kind: ClusterRole + name: portworx-operator + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: portworx-operator + namespace: {{ template "px.getDeploymentNamespace" . }} +spec: + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + name: portworx-operator + template: + metadata: + labels: + name: portworx-operator + spec: + containers: + - name: portworx-operator + imagePullPolicy: Always + image: {{ template "px.getPxOperatorImage" . }}:{{ required "A valid Image tag is required in the SemVer format" .Values.pxOperatorImageVersion }} + command: + - /operator + - --verbose + - --driver=portworx + - --leader-elect=true + env: + - name: OPERATOR_NAME + value: portworx-operator + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "name" + operator: In + values: + - portworx-operator + topologyKey: "kubernetes.io/hostname" + serviceAccountName: portworx-operator +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/serviceaccount-hook.yaml b/charts/portworx/portworx/5.1.1/templates/serviceaccount-hook.yaml new file mode 100644 index 000000000..114528e26 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/serviceaccount-hook.yaml @@ -0,0 +1,48 @@ + +{{- if (.Values.serviceAccount.hook.create) }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "px.hookServiceAccount" . }} + namespace: {{ template "px.getDeploymentNamespace" . }} + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook": "post-install,pre-upgrade,pre-delete" + labels: + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" +--- +kind: ClusterRole +apiVersion: {{ template "rbac.apiVersion" . }} +metadata: + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook": "post-install,pre-upgrade,pre-delete" + name: {{ template "px.hookClusterRole" . }} +rules: + # for daemonset to operator migration, we need hooks for all resources deployed by daemonset, due to resources are + # different in different helm charts (GCP, IKS, Rancher and portworx), we use wild card here. After daemonset + # migration is finished for all customers we shall change this back to limited access. + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: {{ template "rbac.apiVersion" . }} +metadata: + annotations: + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + "helm.sh/hook": "post-install,pre-upgrade,pre-delete" + name: {{ template "px.hookClusterRoleBinding" . }} +subjects: + - kind: ServiceAccount + name: {{ template "px.hookServiceAccount" . }} + namespace: {{ template "px.getDeploymentNamespace" . }} +roleRef: + kind: ClusterRole + name: {{ template "px.hookClusterRole" . }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/templates/storage-cluster.yaml b/charts/portworx/portworx/5.1.1/templates/storage-cluster.yaml new file mode 100644 index 000000000..73a60c5ad --- /dev/null +++ b/charts/portworx/portworx/5.1.1/templates/storage-cluster.yaml @@ -0,0 +1,606 @@ +{{- if and (.Values.deployCluster) (or (not (lookup "apps/v1" "DaemonSet" "kube-system" "portworx")) (default true .Values.generateStorageClusterForMigration)) }} + + {{- $isCoreOS := .Values.isTargetOSCoreOS | default false }} + {{- $internalKVDB := .Values.internalKVDB | default false }} + {{- $etcdEndPoints := .Values.etcdEndPoint }} + {{- $etcdCertPath := .Values.etcd.certPath | default "none" }} + {{- $etcdCert := .Values.etcd.cert | default "none" }} + {{- $etcdKey := .Values.etcd.key | default "none" }} + {{- $kvdbAuthSecret := .Values.kvdb.authSecretName | default "none" }} + {{- $openshiftInstall := .Values.openshiftInstall | default false }} + {{- $EKSInstall := .Values.EKSInstall | default false }} + {{- $GKEInstall := .Values.GKEInstall | default false }} + {{- $pksInstall := .Values.pksInstall | default false }} + {{- $AKSInstall := .Values.AKSInstall | default false }} + {{- $OKEInstall := .Values.OKEInstall | default false }} + {{- $usefileSystemDrive := .Values.usefileSystemDrive | default false }} + {{- $usedrivesAndPartitions := .Values.usedrivesAndPartitions | default false }} + {{- $secretType := .Values.secretType | default "k8s" }} + {{- $deployEnvironmentIKS := .Capabilities.KubeVersion.GitVersion | regexMatch "IKS" }} + {{- $drives := .Values.drives | default "none" }} + {{- $dataInterface := .Values.dataInterface | default "none" }} + {{- $managementInterface := .Values.managementInterface | default "none" }} + {{- $envVars := .Values.envVars | default "none" }} + {{- $customRegistryURL := .Values.customRegistryURL | default "none" }} + {{- $registrySecret := .Values.registrySecret | default "none" }} + {{- $licenseSecret := .Values.licenseSecret | default "none" }} + {{- $kvdbDevice := .Values.kvdbDevice | default "none" }} + {{- $journalDevice := .Values.journalDevice | default "none" }} + {{- $cacheDevices := .Values.cacheDevices | default "none"}} + {{- $systemMetadataDevice:= .Values.systemMetadataDevice | default "none"}} + {{- $miscArgs := include "px.miscArgs" . }} + {{- $volumesPresent := include "px.volumesPresent" . }} + {{- $storagePodAnnotations := .Values.customMetadata.annotations.pod.storage }} + {{- $portworxApiServiceAnnotations := .Values.customMetadata.annotations.service.portworxApi }} + {{- $portworxServiceAnnotations := .Values.customMetadata.annotations.service.portworxService }} + {{- $kvdbServiceAnnotations := .Values.customMetadata.annotations.service.portworxKVDBService }} + {{- $portworxApiServiceLabels := .Values.customMetadata.labels.service.portworxApi}} + + {{- $prometheus := .Values.monitoring.prometheus}} + {{- $prometheousRetention := .Values.monitoring.prometheus.retention | default "none"}} + {{- $prometheousRetentionSize := .Values.monitoring.prometheus.retentionSize | default "none"}} + +kind: StorageCluster +apiVersion: core.libopenstorage.org/v1 +metadata: + name: "{{ required "Clustername cannot be empty" .Values.clusterName }}" + namespace: {{ template "px.getDeploymentNamespace" . }} + annotations: + {{- if eq $openshiftInstall true }} + portworx.io/is-openshift: "true" + {{- end }} + {{- if eq $pksInstall true }} + portworx.io/is-pks: "true" + {{- end }} + {{- if eq $EKSInstall true }} + portworx.io/is-eks: "true" + {{- end }} + {{- if eq $GKEInstall true }} + portworx.io/is-gke: "true" + {{- end }} + {{- if eq $AKSInstall true }} + portworx.io/is-aks: "true" + {{- end }} + {{- if eq $OKEInstall true }} + portworx.io/is-oke: "true" + {{- end }} + {{- if $miscArgs }} + portworx.io/misc-args: {{ $miscArgs | quote }} + {{- end }} + {{- if eq .Values.disableStorageClass true }} + portworx.io/disable-storage-class: "true" + {{- end }} + {{- if and (ne .Values.serviceType "") (ne .Values.serviceType "none") }} + portworx.io/service-type: {{ .Values.serviceType | quote }} + {{- end }} + {{- if (lookup "apps/v1" "DaemonSet" "kube-system" "portworx") }} + portworx.io/migration-approved: "false" + {{- end }} + {{- if eq .Values.nonDisruptivek8sUpgrade true }} + portworx.io/disable-non-disruptive-upgrade: "false" + {{- end }} + {{- if eq .Values.skipHealthChecks true }} + portworx.io/health-check: "skip" + {{- end }} + labels: + heritage: {{.Release.Service | quote }} + release: {{.Release.Name | quote }} + chart: "{{.Chart.Name}}-{{.Chart.Version}}" + app.kubernetes.io/managed-by: {{.Release.Service | quote }} + app.kubernetes.io/instance: {{.Release.Name | quote }} +spec: + image: portworx/oci-monitor:{{ required "A valid Image tag is required in the SemVer format" .Values.imageVersion }} + imagePullPolicy: Always + {{- if not (eq $customRegistryURL "none") }} + customImageRegistry: {{ $customRegistryURL }} + {{- end }} + {{- if not (eq $registrySecret "none") }} + imagePullSecret: {{ $registrySecret }} + {{- end }} + + {{- if (or (eq (toString .Values.updateStrategy.type) "RollingUpdate") (eq (toString .Values.updateStrategy.type) "OnDelete")) }} + updateStrategy: + type: {{ .Values.updateStrategy.type }} + + {{- if eq .Values.updateStrategy.type "RollingUpdate"}} + rollingUpdate: + {{- if .Values.updateStrategy.maxUnavailable}} + maxUnavailable: {{.Values.updateStrategy.maxUnavailable}} + {{- end}} + {{- if .Values.updateStrategy.minReadySeconds}} + minReadySeconds: {{.Values.updateStrategy.minReadySeconds}} + {{- end}} + {{- if and (not (eq .Values.updateStrategy.disruption.allow nil)) (or (eq .Values.updateStrategy.disruption.allow true) (eq .Values.updateStrategy.disruption.allow false)) }} + disruption: + allow: {{.Values.updateStrategy.disruption.allow}} + {{- end}} + {{- end}} + {{- end }} + + + {{- if .Values.updateStrategy.autoUpdateComponents}} + autoUpdateComponents: {{.Values.updateStrategy.autoUpdateComponents}} + {{- end}} + + + {{- $hasAnnotations := or ($storagePodAnnotations) ($portworxApiServiceAnnotations) ($portworxServiceAnnotations ) ($kvdbServiceAnnotations) }} + {{- $hasLabels := $portworxApiServiceLabels }} + {{- $hasMetadata := or $hasAnnotations $hasLabels }} + + {{- if $hasMetadata}} + metadata: + {{- if $hasLabels}} + labels: + {{- if $portworxApiServiceLabels }} + service/portworx-api: + {{- $labels := $portworxApiServiceLabels | split ";" }} + {{- range $key, $val := $labels }} + {{- $label := $val | split "=" }} + {{ $label._0 | trim }}: {{ $label._1 | trim | quote -}} + {{- end }} + {{- end }} + {{- end}} + {{- if $hasAnnotations}} + annotations: + {{- if $storagePodAnnotations }} + pod/storage: + {{- $annotations := $storagePodAnnotations | split ";" }} + {{- range $key, $val := $annotations }} + {{- $annotation := $val | split "=" }} + {{ $annotation._0 | trim }}: {{ $annotation._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- if $portworxApiServiceAnnotations }} + service/portworx-api: + {{- $annotations := $portworxApiServiceAnnotations | split ";" }} + {{- range $key, $val := $annotations }} + {{- $annotation := $val | split "=" }} + {{ $annotation._0 | trim }}: {{ $annotation._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- if $portworxServiceAnnotations }} + service/portworx-service: + {{- $annotations := $portworxServiceAnnotations | split ";" }} + {{- range $key, $val := $annotations }} + {{- $annotation := $val | split "=" }} + {{ $annotation._0 | trim }}: {{ $annotation._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- if $kvdbServiceAnnotations }} + service/portworx-kvdb-service: + {{- $annotations := $kvdbServiceAnnotations | split ";" }} + {{- range $key, $val := $annotations }} + {{- $annotation := $val | split "=" }} + {{ $annotation._0 | trim }}: {{ $annotation._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- end}} + {{- end }} + + kvdb: + {{- if and (eq $internalKVDB true) (empty $etcdEndPoints) }} + internal: true + {{- else }} + internal: false + {{- if empty $etcdEndPoints }} + "{{ required "A valid ETCD url in the format etcd:http:// is required. Verify that the key is correct and there isnt any typo in specifying that, also ensure it is accessible from all node of your kubernetes cluster" .etcdEndPoint}}" + {{- else }} + endpoints: + {{- $endpoints := $etcdEndPoints | split ";" }} + {{- range $key, $val := $endpoints }} + - {{$val}} + {{- end }} + {{- end }} + {{- if ne $kvdbAuthSecret "none" }} + authSecret: {{ .Values.kvdb.authSecretName }} + {{- end }} + {{- end }} + +{{- if and (ne $drives "none") (not (hasPrefix "/" $drives)) }} + cloudStorage: + {{- if .Values.provider}} + provider: {{ .Values.provider }} + {{- end}} + deviceSpecs: + {{- $driveNames := $drives | split ";" }} + {{- range $index, $name := $driveNames }} + - {{ $name }} + {{- end }} + {{- if and (ne $kvdbDevice "none") (not (hasPrefix "/" $kvdbDevice)) }} + kvdbDeviceSpec: {{ $kvdbDevice }} + {{- end }} + {{- if ne $systemMetadataDevice "none"}} + systemMetadataDeviceSpec: {{ $systemMetadataDevice }} + {{- end }} + {{- if and (ne $journalDevice "none") (not (hasPrefix "/" $journalDevice)) }} + journalDeviceSpec: {{ $journalDevice }} + {{- end }} + {{- if ne (.Values.maxStorageNodesPerZone | int) 0 }} + maxStorageNodesPerZone: {{ .Values.maxStorageNodesPerZone }} + {{- end }} + {{- if ne (.Values.maxStorageNodes | int) 0 }} + maxStorageNodes: {{ .Values.maxStorageNodes }} + {{- end }} +{{- else }} + {{- if $AKSInstall }} + cloudStorage: + {{- if .Values.provider}} + provider: {{ .Values.provider }} + {{- end}} + deviceSpecs: + - type=Premium_LRS,size=150 + {{- if and (ne $kvdbDevice "none") (not (hasPrefix "/" $kvdbDevice)) }} + kvdbDeviceSpec: {{ $kvdbDevice }} + {{- end }} + {{- if ne $systemMetadataDevice "none"}} + systemMetadataDeviceSpec: {{ $systemMetadataDevice }} + {{- end }} + {{- if and (ne $journalDevice "none") (not (hasPrefix "/" $journalDevice)) }} + journalDeviceSpec: {{ $journalDevice }} + {{- end }} + {{- if gt (.Values.maxStorageNodesPerZone | int) 0 }} + maxStorageNodesPerZone: {{ .Values.maxStorageNodesPerZone }} + {{- end }} + {{- if ne (.Values.maxStorageNodes | int) 0 }} + maxStorageNodes: {{ .Values.maxStorageNodes }} + {{- end }} + {{- else }} + storage: + {{- if hasPrefix "/" $drives }} + devices: + {{- $driveNames := $drives | split ";" }} + {{- range $index, $name := $driveNames }} + - {{ $name }} + {{- end }} + {{- end }} + {{- if ne $cacheDevices "none"}} + cacheDevices: + {{- $devices := $cacheDevices | split ";" }} + {{- range $index, $device := $devices }} + - {{ $device }} + {{- end }} + {{- end}} + {{- if or $usefileSystemDrive $deployEnvironmentIKS }} + forceUseDisks: true + {{- end }} + {{- if eq $usedrivesAndPartitions true }} + useAllWithPartitions: true + {{- end}} + {{- if and (eq $usedrivesAndPartitions false) (eq $drives "none")}} + useAll: true + {{- end }} + {{- if and (ne $kvdbDevice "none") (hasPrefix "/" $kvdbDevice) }} + kvdbDevice: {{ $kvdbDevice }} + {{- end }} + {{- if ne $systemMetadataDevice "none"}} + systemMetadataDevice: {{ $systemMetadataDevice }} + {{- end }} + {{- if and (ne $journalDevice "none") (or (hasPrefix "/" $journalDevice) (eq "auto" $journalDevice)) }} + journalDevice: {{ $journalDevice }} + {{- end }} + {{- end }} +{{- end }} + + {{- if or (ne $dataInterface "none") (ne $managementInterface "none") }} + network: + {{- if ne $dataInterface "none" }} + dataInterface: {{ $dataInterface }} + {{- end }} + {{- if ne $managementInterface "none" }} + mgmtInterface: {{ $managementInterface }} + {{- end }} + {{- end }} + + {{- if ne $secretType "none" }} + secretsProvider: {{$secretType}} + {{- else }} + {{- if $deployEnvironmentIKS }} + secretsProvider: ibm-kp + {{- end }} + {{- end }} + + {{- $isLicenseSecretAdded := ne $licenseSecret "none" }} + {{- $shouldRenderEnv := or (ne $envVars "none") .Values.envs $AKSInstall $OKEInstall $isLicenseSecretAdded }} + {{- if $shouldRenderEnv }} + env: + {{- with .Values.envs }} + {{- toYaml . | nindent 2 }} + {{- end -}} + {{- if not (eq $envVars "none") }} + {{- $vars := $envVars | split ";" }} + {{- range $key, $val := $vars }} + {{- $envVariable := $val | split "=" }} + - name: {{ $envVariable._0 | trim | quote }} + value: {{ $envVariable._1 | trim | quote }} + {{- end }} + {{- end }} + {{- if $isLicenseSecretAdded }} + - name: SAAS_ACCOUNT_KEY_STRING + valueFrom: + secretKeyRef: + name: "{{ $licenseSecret }}" + key: accountKey + {{- end }} + {{- if $AKSInstall }} + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: px-azure + key: AZURE_CLIENT_SECRET + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: px-azure + key: AZURE_CLIENT_ID + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: px-azure + key: AZURE_TENANT_ID + {{- end }} + {{- if $OKEInstall }} + - name: PX_ORACLE_user_ocid + valueFrom: + secretKeyRef: + name: ociapikey + key: PX_ORACLE_user_ocid + - name: PX_ORACLE_fingerprint + valueFrom: + secretKeyRef: + name: ociapikey + key: PX_ORACLE_fingerprint + - name: "PX_ORACLE_private_key_path" + value: "/etc/pwx/oci_key/oci_api_key.pem" + {{- end }} + {{- end }} + + stork: + {{- if (and (.Values.stork.enabled) (eq .Values.stork.enabled true))}} + enabled: true + {{- if .Values.stork.storkVersion }} + image: {{ template "px.getStorkImage" . }}:{{ required "A valid Image tag is required in the SemVer format" .Values.stork.storkVersion }} + {{- end }} + + {{- with .Values.stork.env }} + env: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.stork.volumes }} + volumes: + {{- toYaml . | nindent 4 }} + {{- end -}} + {{- if .Values.stork.args }} + args: + {{- $args := .Values.stork.args | split ";" }} + {{- range $key, $val := $args }} + {{- $arg := $val | split "=" }} + {{ $arg._0 | trim }}: {{ $arg._1 | trim | quote -}} + {{- end }} + {{- end }} + {{- else }} + enabled: false + {{- end }} + + {{- if or ( eq $volumesPresent "true") ( eq .Values.OKEInstall true) }} + volumes: + {{- if .Values.OKEInstall }} + - name: ociapikey + mountPath: /etc/pwx/oci_key + secret: + secretName: ociapikey + items: + - key: oci_api_key.pem + path: oci_api_key.pem + {{- end }} + {{- if eq $isCoreOS true}} + - name: src + mountPath: /lib/modules + hostPath: + path: /lib/modules + type: Directory + {{- end }} + {{- if ne $etcdCertPath "none" }} + - name: etcdcerts + mountPath: /etc/pwx/etcdcerts + secret: + secretName: px-etcd-certs + items: + - key: ca.pem + path: ca.pem + {{- if ne $etcdCert "none" }} + - key: client.pem + path: client.pem + {{- end -}} + {{- if ne $etcdKey "none" }} + - key: client-key.pem + path: client-key.key + {{- end -}} + {{- end}} + {{- with .Values.volumes }} + {{- toYaml . | nindent 2 }} + {{- end}} + {{- end }} + + {{- $promethusEnabled := or $prometheus.exportMetrics $prometheus.enabled}} + {{- $monitoringEnabled := or $promethusEnabled .Values.monitoring.telemetry $prometheus.enabled .Values.grafana }} + {{- if $monitoringEnabled }} + monitoring: + {{- if $promethusEnabled }} + prometheus: + {{- if $prometheus.enabled }} + enabled: true + {{- end}} + {{- if $prometheus.exportMetrics }} + exportMetrics: true + {{- end}} + {{- if $prometheus.alertManager }} + alertManager: + enabled: true + {{- end}} + {{- if and ($prometheus.enabled) (ge (int $prometheus.replicas) 1) }} + replicas: {{ $prometheus.replicas }} + {{- end }} + {{- if and (ne $prometheousRetention "none") ($prometheus.enabled)}} + retention: {{ $prometheousRetention }} + {{- end}} + {{- if ne $prometheousRetentionSize "none"}} + retentionSize: {{ $prometheousRetentionSize }} + {{- end}} + {{- with $prometheus.resources }} + resources: + {{- toYaml . | nindent 8 }} + {{- end}} + {{- with $prometheus.storage }} + storage: + {{- toYaml . | nindent 8 }} + {{- end}} + {{- with $prometheus.volumes }} + volumes: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with $prometheus.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 6 }} + {{- end}} + {{- if $prometheus.securityContext.runAsNonRoot }} + securityContext: + runAsNonRoot: true + {{- end}} + {{- end }} + + {{- if .Values.monitoring.telemetry }} + telemetry: + enabled: true + {{- end }} + + {{- if .Values.monitoring.grafana }} + grafana: + enabled: true + {{- end }} + {{- end }} + + {{- $csiEnabled := eq .Values.csi.enabled true }} + {{- if or (eq .Values.csi.enabled true) (eq .Values.csi.enabled false) }} + csi: + enabled: {{ .Values.csi.enabled }} + {{- if $csiEnabled}} + {{- if eq .Values.csi.topology.enabled true }} + topology: + enabled: true + {{- end}} + {{- if eq .Values.csi.installSnapshotController true }} + installSnapshotController: true + {{- end}} + {{- end}} + {{- end }} + + + {{- if (and (.Values.autopilot.enabled) (eq .Values.autopilot.enabled true))}} + autopilot: + enabled: true + {{- if ne .Values.autopilot.image ""}} + image: {{.Values.autopilot.image}} + {{- end }} + {{- if eq .Values.autopilot.lockImage true}} + lockImage: true + {{- end }} + {{- if .Values.autopilot.args }} + args: + {{- $args := .Values.autopilot.args | split ";" }} + {{- range $key, $val := $args }} + {{- $arg := $val | split "=" }} + {{ $arg._0 | trim }}: {{ $arg._1 | trim | quote -}} + {{- end }} + {{- end }} + {{- with .Values.autopilot.env }} + env: + {{- toYaml . | nindent 4 }} + {{- end }} + + {{- with .Values.autopilot.volumes }} + volumes: + {{- toYaml . | nindent 4}} + {{- end }} + {{- with .Values.autopilot.providers }} + providers: + {{- range . }} + - name: "{{ .name }}" + type: "{{ .type }}" + params: + {{- $params := splitList ";" .params }} + {{- range $params }} + {{- $pair := splitList "=" . }} + {{- if eq (len $pair) 2 }} + {{- $key := index $pair 0 }} + {{- $value := index $pair 1 }} + {{ $key }}: "{{ $value }}" + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{- if .Values.runtimeOptions }} + runtimeOptions: + {{- $args := .Values.runtimeOptions | split ";" }} + {{- range $key, $val := $args }} + {{- $arg := $val | split "=" }} + {{ $arg._0 | trim }}: {{ $arg._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- if .Values.featureGates }} + featureGates: + {{- $args := .Values.featureGates | split ";" }} + {{- range $key, $val := $args }} + {{- $arg := $val | split "=" }} + {{ $arg._0 | trim }}: {{ $arg._1 | trim | quote -}} + {{- end }} + {{- end }} + + {{- if eq .Values.security.enabled true}} + security: + enabled: true + auth: + {{- if (or (eq .Values.security.auth.guestAccess "Enabled") (eq .Values.security.auth.guestAccess "Disabled") (eq .Values.security.auth.guestAccess "Managed"))}} + guestAccess: {{.Values.security.auth.guestAccess}} + {{- end}} + {{- if and .Values.security.auth.selfSigned.tokenLifetime .Values.security.auth.selfSigned.issuer .Values.security.auth.selfSigned.sharedSecret }} + selfSigned: + tokenLifetime: {{.Values.security.auth.selfSigned.tokenLifetime}} + issuer: {{.Values.security.auth.selfSigned.issuer}} + sharedSecret: {{.Values.security.auth.selfSigned.sharedSecret}} + {{- end}} + {{- end}} + + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 4 }} + {{- end}} + + {{- if or .Values.tolerations .Values.nodeAffinity}} + placement: + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.nodeAffinity }} + nodeAffinity: + {{- toYaml . | nindent 5 }} + {{- end}} + {{- end}} + + {{- with .Values.nodesConfiguration }} + nodes: + {{- toYaml . | nindent 2 }} + {{- end}} + + {{- if .Values.deleteStrategy.type }} + deleteStrategy: + type: {{ .Values.deleteStrategy.type }} + {{- end }} +{{- end }} diff --git a/charts/portworx/portworx/5.1.1/values.yaml b/charts/portworx/portworx/5.1.1/values.yaml new file mode 100644 index 000000000..d326caef4 --- /dev/null +++ b/charts/portworx/portworx/5.1.1/values.yaml @@ -0,0 +1,269 @@ +# Please uncomment and specify values for these options as per your requirements. + +deployOperator: true # Deploy the Portworx operator +deployCluster: true # Deploy the Portworx cluster + +imageVersion: 3.2.1.1 # Version of the PX Image. +pxOperatorImageVersion: 24.2.1 # Version of the PX operator image. + +openshiftInstall: false # Defaults to false for installing Portworx on Openshift . +isTargetOSCoreOS: false # Is your target OS CoreOS? Defaults to false. + # Used to disable or enable smart and parallel kubetnetes node upgrades. +nonDisruptivek8sUpgrade: false # By default, S&P upgrades are disabled. To enable them, set this to true +skipHealthChecks: false # Used to skip health checks. By default, health checks are enabled. Set this to true to disable health checks + +pksInstall: false # installation on PKS (Pivotal Container Service) +EKSInstall: false # installation on EKS. +GKEInstall: false # installation on GKE. +AKSInstall: false # installation on AKS +OKEInstall: false # installation on OKE +etcdEndPoint: # The ETCD endpoint. Should be in the format etcd:http://:2379. If there are multiple etcd endpoints they need to be ";" seperated. + # the default value is empty since it requires to be explicity set using either the --set option of -f values.yaml. +clusterName: mycluster # This is the default. please change it to your cluster name. +usefileSystemDrive: false # true/false Instructs PX to use an unmounted Drive even if it has a filesystem. +usedrivesAndPartitions: false # Defaults to false. Change to true and PX will use unmounted drives and partitions. +drives: none # NOTE: This is a ";" seperated list of drives. For eg: "/dev/sda;/dev/sdb;/dev/sdc" or + # "type=gp2,size=200;type=gp3,size=500". Defaults to use -A switch. +provider: # If we are using cloudstorage, it specifies the cloud provider name, such as: pure, azure, aws, gce, vsphere. + +journalDevice: +cacheDevices: # Specifies the list of cache devices Portworx should use. If there are multiple devices they need to be ";" seperated. + +maxStorageNodesPerZone: 0 # The maximum number of storage nodes desired per zone, in case of cloud drive provisioning + +maxStorageNodes: 0 # Specifies the maximum number of storage nodes. If this number is reached, and a new node is added, Portworx doesn't provision drives for the new node. + # Instead, Portworx starts the node as a compute-only node. As a best practice, it is recommended to use the maxStorageNodesPerZone field. + +systemMetadataDevice: # Indicates the cloud device Portworx uses for metadata. For performance, specify a system metadata device when using Portworx with the internal KVDB. + + +secretType: k8s # Defaults to k8s, but can be kvdb/k8s/aws-kms/vault/ibm-kp. It is autopopulated to ibm-kp + # if the environment is IKS. + +dataInterface: none # Name of the interface +managementInterface: none # Name of the interface +serviceType: none # Kubernetes service type for services deployed by the Operator. Direct Values like + # 'LoadBalancer', 'NodePort' will change all services. To change the types of specific + # services, value can be specified as 'portworx-service:LoadBalancer;portworx-api:ClusterIP' + +runtimeOptions: # A collection of key-value pairs that overwrites the runtime options. ex num_io_threads=10;-c=px-cluster +featureGates: # A collection of key-value pairs specifying which Portworx features should be enabled or disabled. + +security: + enabled: false # Enables or disables Security at any given time. + auth: + guestAccess: Enabled # Determines how the guest role will be updated in your cluster. The options are Enabled, Disabled, or Managed. + selfSigned: + tokenLifetime: # The length operator-generated tokens will be alive until being refreshed. + issuer: # The issuer name to be used when configuring PX-Security. This field maps to the PORTWORX_AUTH_JWT_ISSUER environment variable in the Portworx Daemonset. + sharedSecret: # The Kubernetes secret name for retrieving and storing your shared secret. + +resources: # Configure Portworx container usage such as memory and CPU usage + # requests: + # memory: "64Mi" + # cpu: "250m" + +customMetadata: # Configure custom labels and annotation for specific pod and services + # Pass labels and annotation with ";" sperated list. Example: application=my-app;environment=production + annotations: # Currently, custom annotations are supported on following types of components: + pod: + storage: "" + service: + portworxApi: "" + portworxService: "" + portworxKVDBService: "" + labels: # Currently, custom labels are only supported on the portworx-api service + service: + portworxApi: "" + + +envVars: none # DEPRECATED: Use envs section to set env variables + # NOTE: This is a ";" seperated list of environment variables. + # For eg: MYENV1=myvalue1;MYENV2=myvalue2 + +envs: # Add environment variables to the Portworx container in all Kubernetes supported formats + # - name: AWS_CA_BUNDLE + # value: "/etc/pwx/objectstore-cert/objectstore.pem" + # - name: AWS_ACCESS_KEY_ID + # valueFrom: + # secretKeyRef: + # name: aws-creds + # key: access-key + +miscArgs: none # Miscellaneous arguments that will be passed to portworx verbatim. Only use this if there is + # no equivalent way to specify these options directly via a StorageCluster spec field. + +disableStorageClass: false # Instructs Operator to not install the default Portworx StorageClasses. + +stork: # Use Stork https://docs.portworx.com/portworx-backup-on-prem/concepts/stork for hyperconvergence. + enabled: true + storkVersion: # version of Stork. For eg: 2.7.0, when it's empty Portworx operator will pick up + # version according to Portworx version. + + args: # Pass arguments to Stork container. Example: verbose=true;webhook-controller=false + volumes: # Add volumes to Stork container. Refer the top level volumes for schema. + env: # A list of Kubernetes like environment variables passed to Stork. + +customRegistryURL: +registrySecret: +licenseSecret: + +monitoring: + prometheus: + enabled: false # Enable prometheus + exportMetrics: false # Expose the Portworx metrics to an external or operator deployed Prometheus. + alertManager: false # Enable prometheus alert manager + resources: # Configure Prometheus resource usage such as memory and CPU usage + # requests: + # memory: "64Mi" + # cpu: "250m" + # limits: + # memory: "128Mi" + # cpu: "500m" + replicas: 1 # Number of prometheus replicas that will be deployed. Default is 1 + retention: "24h" # Time period for which prometheus retains historical matrics. default is 24h. + retentionSize: "" # Maximum amount of disk space that Prometheus can use to store historical metrics. Example: "10GiB","50MiB" + storage: # storage type that Prometheus will use for storing data + # volumeClaimTemplate: + # spec: + # storageClassName: px-csi-db + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 2Gi + # + volumes: # Additional volumes for the prometheus statefulSet + # - name: additional-volume + # emptyDir: {} + volumeMounts: # Additional VolumeMounts for the Prometheus StatefulSet + # - mountPath: /test + # name: additional-volume + securityContext: # Enable prometheus container run as a non-root user. + runAsNonRoot: false + telemetry: true # Enable telemetry + grafana: false # Enable grafana + + +deployOnMaster: false # For POC only + +csi: + enabled: true # Enable CSI + topology: + enabled: false # Enable CSI topology feature gate + installSnapshotController: false # Enable CSI Snapshot Controller + +autopilot: + enabled: true # Enable AutoPilot + image: "" # Autopilot image + lockImage: false # Enables locking Autopilot to the given image. + # When set to false, the Portworx Operator will overwrite the Autopilot image to a recommended image for given Portworx version. + + args: # overrides the default Autopilot arguments or adds new arguments.Example: min_poll_interval=4;log-level=info + env: # A list of Kubernetes like environment variables passed to Autopilot + # - name: PX_SHARED_SECRET + # valueFrom: + # secretKeyRef: + # key: apps-secret + # name: px-system-secrets + volumes: # Add volumes to autopilot container. Refer the top level volumes for schema. + providers: + # - name: "" # Name is the unique name for the provider + # type: "" # Type is the type of data provider. For instance, prometheus + # params: "" # Pass params to provider. Example: url=http://datadog:9090;auth=foobar + + +internalKVDB: true # internal KVDB +kvdbDevice: none # specify a separate device to store KVDB data, only used when internalKVDB is set to true + +etcd: # DEPRECATED: Use kvdb.authSecretName for configuring secure etcd + credentials: none:none # Username and password for ETCD authentication in the form user:password + certPath: none # Base path where the certificates are placed. (example: if the certificates ca,crt and the key are in /etc/pwx/etcdcerts the value should be provided as /etc/pwx/etcdcerts) + ca: none # Location of CA file for ETCD authentication. Should be /path/to/server.ca + cert: none # Location of certificate for ETCD authentication. Should be /path/to/server.crt + key: none # Location of certificate key for ETCD authentication Should be /path/to/servery.key + +consul: # DEPRECATED: Use kvdb.authSecretName for configuring secure consul + token: none # ACL token value used for Consul authentication. (example: 398073a8-5091-4d9c-871a-bbbeb030d1f6) + +kvdb: + authSecretName: none # Refer https://docs.portworx.com/portworx-enterprise/operations/kvdb-for-portworx/external-kvdb#secure-your-etcd-communication to + # create a kvdb secret and specify the name of the secret here + +volumes: # Add volumes to Portworx container. Supported volume types: Host, Secret, ConfigMap + # - name: objectstore-cert + # mountPath: /etc/pwx/objectstore-cert + # mountPropagation: None|HostToContainer|Bidirectional + # readOnly: true + # secret: + # secretName: objectstore-cert + # items: + # - key: objectstore.pem + # path: objectstore.pem + +tolerations: # Add tolerations + # - key: "key" + # operator: "Equal|Exists" + # value: "value" + # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + + +nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: px-schedule + # operator: NotIn + # values: + # - "false" + +nodesConfiguration: # override certain cluster level configuration for individual or group of nodes. +# - selector: # Selector for the node(s) to which the configuration in this section will be applied +# nodeName: # Name of the node to which this configuration will be applied. Node name takes precedence over selector.labelSelector +# labelSelector: # selector for nodes to which this configuration will be applied. +# network: # Specify network configuration for the selected nodes, similar to the one specified at cluster level. +# # If this network configuration is empty, then cluster level (dataInterface,managementInterface) values are used. +# dataInterface: +# mgmtInterface: +# storage: # Specify storage configuration for the selected nodes, similar to the one specified at cluster level. If some of the config is left empty, +# # the cluster level storage values are passed to the nodes. +# env: # Specify extra environment variables for the selected nodes. Cluster level environment variables are combined with these and sent to the selected nodes. +# # If same variable is present at cluster level, then the node level variable takes precedence. +# - name: AWS_CA_BUNDLE +# value: "/etc/pwx/objectstore-cert/objectstore.pem" +# - name: AWS_ACCESS_KEY_ID +# valueFrom: +# secretKeyRef: +# name: aws-creds +# key: access-key +# runtimeOptions: # Specify runtime options for the selected nodes. If specified, cluster level options are ignored and only these runtime options are passed to the nodes. + + +serviceAccount: + hook: + create: true + name: + +clusterToken: + create: false # Create cluster token + secretName: px-vol-encryption # Name of kubernetes secret to be created. Requires clusterToken.create to be true. + serviceAccountName: px-create-cluster-token # Service account name to use for post-install hook to create cluster token + +#requirePxEnabledTag: true # if set to true, portworx will only install on nodes with px/enabled: true label. Not required in most scenarios. + +deleteStrategy: # Optional: Delete strategy for the portworx cluster + type: # Valid values: Uninstall, UninstallAndWipe + +updateStrategy: + type: # portworx supports the following update strategies- RollingUpdate and OnDelete + + # maxUnavailable and minReadySeconds applicable only on selecting type RollingUpdate + disruption: + allow: # This field is used to enable smart and parallel upgrade. Smart upgrade is disabled by default, + # Enable it by setting to false. we can use the `maxUnavailable` field to control the maximum number of Portworx nodes that can be upgraded at a time. + + maxUnavailable: # Similarly to how Kubernetes rolling update strategies work, this field specifies how many nodes can be down at any given time. + minReadySeconds: # During rolling updates, this flag will wait for all pods to be ready for at least minReadySeconds before updating the next batch of pods, + # where the size of the pod batch is specified through the spec.updateStrategy.rollingUpdate.maxUnavailable flag. + + autoUpdateComponents: # Indicates the update strategy for the component images + # valid values None,Once,Always \ No newline at end of file diff --git a/index.yaml b/index.yaml index f2ee02312..0d72f56fb 100644 --- a/index.yaml +++ b/index.yaml @@ -34205,6 +34205,37 @@ entries: urls: - assets/fairwinds/polaris-5.11.1.tgz version: 5.11.1 + portworx: + - annotations: + catalog.cattle.io/certified: partner + catalog.cattle.io/display-name: Portworx Enterprise + catalog.cattle.io/kube-version: '>=1.26-0' + catalog.cattle.io/release-name: "" + apiVersion: v1 + appVersion: 3.2.1.1 + created: "2025-01-13T09:55:27.983116+05:30" + description: A Helm chart for installing Portworx on Kubernetes. + digest: a9485781d5246aeb35585fac3c1f0758bd929d4f7ed5c20bc6d75484f2ae7845 + home: https://portworx.com/ + icon: file://assets/icons/portworx.svg + keywords: + - Storage + - ICP + - IKS + - persistent disk + - pvc + - cloud native storage + - persistent storage + - portworx + - amd64 + - Commercial + kubeVersion: '>=1.26-0' + name: portworx + sources: + - https://github.com/portworx/helm + urls: + - assets/portworx/portworx-5.1.1.tgz + version: 5.1.1 psmdb-db: - annotations: catalog.cattle.io/certified: partner @@ -47716,4 +47747,4 @@ entries: urls: - assets/netfoundry/ziti-host-1.5.1.tgz version: 1.5.1 -generated: "2025-01-12T00:01:48.181240315Z" +generated: "2025-01-13T09:55:23.131093+05:30" diff --git a/packages/portworx/portworx/overlay/app-readme.md b/packages/portworx/portworx/overlay/app-readme.md new file mode 100644 index 000000000..ae8bbe96f --- /dev/null +++ b/packages/portworx/portworx/overlay/app-readme.md @@ -0,0 +1,8 @@ +# Portworx + +[Portworx](https://portworx.com/) is a software defined storage overlay that allows you to + + * Run containerized stateful applications that are highly-available (HA) across multiple nodes, cloud instances, regions, data centers or even clouds + * Migrate workflows between multiple clusters running across same or hybrid clouds + * Run hyperconverged workloads where the data resides on the same host as the applications + * Have programmatic control on your storage resources \ No newline at end of file diff --git a/packages/portworx/portworx/overlay/questions.yml b/packages/portworx/portworx/overlay/questions.yml new file mode 100644 index 000000000..ff44a2c67 --- /dev/null +++ b/packages/portworx/portworx/overlay/questions.yml @@ -0,0 +1,43 @@ +questions: + +################################## Basic Settings ############################ +- variable: imageVersion + default: "3.2.1.1" + type: string + label: Portworx version to be deployed. + group: "Basic Settings" + +- variable: clusterName + type: string + label: Portworx cluster name + default: mycluster + group: "Basic Settings" + +############ Storage Parameters ############## +- variable: usedrivesAndPartitions + label: "Use unmounted drives and partitions" + descrition: "Use unmounted disks even if they have a partition or filesystem on it. PX will never use a drive or partition that is mounted." + type: boolean + default: false + group: "Storage Parameters" + +################################### Registry settings options ################################ +- variable: customRegistry + label: "Use a custom container registry?" + type: boolean + default: false + group: "Container Registry Parameters" + +- variable: registrySecret + show_if: "customRegistry=true" + description: "Specify a custom Kubernetes secret that will be used to authenticate with a container registry. Must be defined in kube-system namespace. (example: regcred)" + type: string + label: "Registry Kubernetes Secret" + group: "Container Registry Parameters" + +- variable: customRegistryURL + show_if: "customRegistry=true" + description: "Specify a custom container registry server (including repository) that will be used instead of index.docker.io to download Docker images. (example: dockerhub.acme.net:5443 or myregistry.com/myrepository/)" + label: "Custom Registry URL" + type: string + group: "Container Registry Parameters" \ No newline at end of file diff --git a/packages/portworx/portworx/upstream.yaml b/packages/portworx/portworx/upstream.yaml new file mode 100644 index 000000000..31f0ac53b --- /dev/null +++ b/packages/portworx/portworx/upstream.yaml @@ -0,0 +1,8 @@ +GitRepo: https://github.com/portworx/helm.git +GitBranch: master +GitSubdirectory: charts/portworx +Vendor: Portworx +DisplayName: Portworx Enterprise +ChartMetadata: + kubeVersion: '>=1.26-0' + icon: https://central.portworx.com/assets/images/logo/small.svg \ No newline at end of file